Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Progress measure #359

Open
wants to merge 28 commits into
base: 2.4.0-dev
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
28 commits
Select commit Hold shift + click to select a range
5850251
add progress measure testing file
MaiaPelletier Jul 18, 2022
44027ae
add tests
MaiaPelletier Jul 25, 2022
5135477
Update progress measure names
MaiaPelletier Jul 27, 2022
749d6d3
docstrings and polishing
MaiaPelletier Aug 3, 2022
c3dbbed
Final touches after last tests
MaiaPelletier Aug 4, 2022
27ccb2e
reverts
MaiaPelletier Aug 4, 2022
1f82b44
final
MaiaPelletier Aug 4, 2022
a7410d2
Change progress label
MaiaPelletier Jan 20, 2023
24d94cf
Merge branch 'open-sdg:2.3.0-dev' into maia-progress-calculation
MaiaPelletier May 9, 2023
6d3d128
Methodology changes
MaiaPelletier Jun 8, 2023
10da7f0
Merge pull request #1 from MaiaPelletier/maia-progress-calculation
tristanmenard Sep 5, 2024
b3fca91
gitignore jupyter notebooks
tristanmenard Oct 9, 2024
deb8ba1
enable progress measure calculations for indicators with units and se…
tristanmenard Oct 10, 2024
3db5edb
re-organize progress measure code as classes, add progress measure ca…
tristanmenard Nov 4, 2024
c1ac384
merge base and indicator progress measure classes, add threshold redu…
tristanmenard Nov 5, 2024
76d89ce
add progress column
tristanmenard Nov 5, 2024
b05418e
rename indicator and series progress classes
tristanmenard Nov 5, 2024
496852e
remove old progress measure functions
tristanmenard Nov 5, 2024
3d9a9c5
read disaggregations for progress measure organized as field/values p…
tristanmenard Nov 27, 2024
bc5f9ca
Add caching to avoid repeating calculation during translated builds
brockfanning Dec 3, 2024
1d0b885
fixes for issues with progress column
tristanmenard Dec 3, 2024
707211c
Merge pull request #2 from brockfanning/progress-measure-dev
tristanmenard Dec 11, 2024
1b674aa
cache_store=None fix
tristanmenard Dec 11, 2024
15c07b7
re-align scores for both methodologies
tristanmenard Dec 11, 2024
7e3267d
update comments
tristanmenard Dec 11, 2024
7fa7c76
use progress_thresholds in score calculation
tristanmenard Dec 12, 2024
998315f
generalisation allowing easy replacement of series score aggregation …
tristanmenard Dec 17, 2024
a18e7cc
bugfix for scores with reduced thresholds
tristanmenard Dec 18, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -112,3 +112,5 @@ venv.bak/
# IDEs
.vscode
.DS_Store
.idea
*.ipynb
9 changes: 9 additions & 0 deletions sdg/IndicatorOptions.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ def __init__(self):
]
self.series_column = 'Series'
self.unit_column = 'Units'
self.progress_column = 'Progress'
self.observation_attributes = []


Expand Down Expand Up @@ -51,3 +52,11 @@ def set_unit_column(self, column):

def get_unit_column(self):
return self.unit_column

def set_progress_column(self, column):
self.progress_column = column
return self


def get_progress_column(self):
return self.progress_column
457 changes: 457 additions & 0 deletions sdg/ProgressMeasure.py

Large diffs are not rendered by default.

5 changes: 5 additions & 0 deletions sdg/open_sdg.py
Original file line number Diff line number Diff line change
Expand Up @@ -203,6 +203,7 @@ def open_sdg_indicator_options_defaults():
'Series',
'Value',
'GeoCode',
'Progress',
'Observation status',
'Unit multiplier',
'Unit measure',
Expand All @@ -218,6 +219,7 @@ def open_sdg_indicator_options_defaults():
],
'series_column': 'Series',
'unit_column': 'Units',
'progress_column': 'Progress',
}


Expand All @@ -233,6 +235,8 @@ def open_sdg_indicator_options_from_dict(options):
options_obj.set_series_column(options['series_column'])
if 'unit_column' in options:
options_obj.set_unit_column(options['unit_column'])
if 'progress_column' in options:
options_obj.set_progress_column(options['progress_column'])
return options_obj


Expand Down Expand Up @@ -358,6 +362,7 @@ def open_sdg_prep(options):
logging=options['logging'],
indicator_export_filename=options['indicator_export_filename'],
ignore_out_of_scope_disaggregation_stats=options['ignore_out_of_scope_disaggregation_stats'],
cache_store = {},
)

if callable(options['alter_indicator']):
Expand Down
16 changes: 15 additions & 1 deletion sdg/outputs/OutputOpenSdg.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from sdg.outputs import OutputBase
from sdg.data import write_csv
from sdg.json import write_json, df_to_list_dict
from sdg.ProgressMeasure import IndicatorProgress

class OutputOpenSdg(OutputBase):
"""Output SDG data/metadata in the formats expected by Open SDG."""
Expand All @@ -11,7 +12,7 @@ class OutputOpenSdg(OutputBase):
def __init__(self, inputs, schema, output_folder='_site', translations=None,
reporting_status_extra_fields=None, indicator_options=None,
indicator_downloads=None, logging=None, indicator_export_filename='all_indicators',
ignore_out_of_scope_disaggregation_stats=False):
ignore_out_of_scope_disaggregation_stats=False, cache_store=None):
"""Constructor for OutputOpenSdg.

Parameters
Expand All @@ -28,6 +29,9 @@ def __init__(self, inputs, schema, output_folder='_site', translations=None,
A filename (without the extension) for the zipped indicator export.
ignore_out_of_scope_disaggregation_stats : boolean
Whether to ignore the "not applicable" disaggregation stats.
cache_store : dict
A store that is passed in to allow caching during the build, to
avoid useless duplication.
"""
if translations is None:
translations = []
Expand All @@ -38,6 +42,7 @@ def __init__(self, inputs, schema, output_folder='_site', translations=None,
self.indicator_downloads = indicator_downloads
self.indicator_export_filename = indicator_export_filename
self.ignore_na = ignore_out_of_scope_disaggregation_stats
self.cache_store = cache_store


def build(self, language=None):
Expand All @@ -63,6 +68,15 @@ def build(self, language=None):

for indicator_id in self.get_indicator_ids():
indicator = self.get_indicator_by_id(indicator_id).language(language)
# Use the methodology to calculate a progress status.
progress_status = IndicatorProgress(
indicator,
logging=self.logging,
cache_store=self.cache_store,
).get_indicator_status()
if progress_status:
# If the calculations returned something, set it in the indicator's 'meta' property.
indicator.meta['progress_status'] = progress_status
# Output all the csvs
status = status & write_csv(indicator_id, indicator.data, ftype='data', site_dir=site_dir)
status = status & write_csv(indicator_id, indicator.edges, ftype='edges', site_dir=site_dir)
Expand Down
7 changes: 7 additions & 0 deletions tests/assets/progress-calculation/data/indicator_1-1-1.csv
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Year,SEX,Value
2020,,100
2021,,120
2020,M,50
2021,M,60
2020,F,70
2021,F,80
3 changes: 3 additions & 0 deletions tests/assets/progress-calculation/indicator-config/1-1-1.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
foo: bar
page_content: Hello world
auto_progress_calculation: true