From c0b72536983761aac1fce64be9cf2516af7fcfd8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Thu, 10 Oct 2024 16:34:56 +0200 Subject: [PATCH 1/8] refactor: re-generate all categorizations --- climate_categories/data/BURDI.yaml | 6 ++++-- climate_categories/data/BURDI_class.yaml | 3 ++- climate_categories/data/CRFDI.yaml | 3 ++- climate_categories/data/CRFDI_class.yaml | 3 ++- climate_categories/data/IPCC2006_PRIMAP.py | 7 +++++-- climate_categories/data/IPCC2006_PRIMAP.yaml | 3 ++- data_generation/IPCC2006_PRIMAP.py | 2 +- 7 files changed, 18 insertions(+), 9 deletions(-) diff --git a/climate_categories/data/BURDI.yaml b/climate_categories/data/BURDI.yaml index c94423ff..26a8050f 100644 --- a/climate_categories/data/BURDI.yaml +++ b/climate_categories/data/BURDI.yaml @@ -2,9 +2,11 @@ name: BURDI title: BUR GHG emission categories (DI query interface) comment: Biannual Update Report categories of GHG emissions and removals and some other quantities as obtained from the di.unfccc.int flexible query interface -references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 +references: + https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 and decision 17/CP.8, 'Guidelines for the preparation of national communications - from Parties not included in Annex I to the Convention' available at https://unfccc.int/files/meetings/workshops/other_meetings/application/pdf/dec17-cp.pdf + from Parties not included in Annex I to the Convention' available at + https://unfccc.int/files/meetings/workshops/other_meetings/application/pdf/dec17-cp.pdf institution: UNFCCC hierarchical: true last_update: '2021-11-05' diff --git a/climate_categories/data/BURDI_class.yaml b/climate_categories/data/BURDI_class.yaml index c312723e..8911a04e 100644 --- a/climate_categories/data/BURDI_class.yaml +++ b/climate_categories/data/BURDI_class.yaml @@ -3,7 +3,8 @@ title: BUR GHG emission categories (DI query interface) + classifications comment: Biannual Update Report categories of GHG emissions and removals and some other quantities as obtained from the di.unfccc.int flexible query interface extended by sub-classifications also provided by the DI flexible query interface -references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 +references: + https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 institution: UNFCCC hierarchical: true last_update: '2021-11-05' diff --git a/climate_categories/data/CRFDI.yaml b/climate_categories/data/CRFDI.yaml index d09d25ff..cfd06969 100644 --- a/climate_categories/data/CRFDI.yaml +++ b/climate_categories/data/CRFDI.yaml @@ -2,7 +2,8 @@ name: CRFDI title: CRF GHG emission categories (DI query interface) comment: Common Reporting Format categories of GHG emissions and removals and some other quantities as obtained from the di.unfccc.int flexible query interface -references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 +references: + https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 institution: UNFCCC hierarchical: true last_update: '2021-11-05' diff --git a/climate_categories/data/CRFDI_class.yaml b/climate_categories/data/CRFDI_class.yaml index 201313d2..c7d3af38 100644 --- a/climate_categories/data/CRFDI_class.yaml +++ b/climate_categories/data/CRFDI_class.yaml @@ -3,7 +3,8 @@ title: CRF GHG emission categories (DI query interface) + classifications comment: Common Reporting Format categories of GHG emissions and removals and some other quantities as obtained from the di.unfccc.int flexible query interface extended by sub-classifications also provided by the DI flexible query interface -references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 +references: + https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7 institution: UNFCCC hierarchical: true last_update: '2021-11-05' diff --git a/climate_categories/data/IPCC2006_PRIMAP.py b/climate_categories/data/IPCC2006_PRIMAP.py index afee36a6..6ad2b866 100644 --- a/climate_categories/data/IPCC2006_PRIMAP.py +++ b/climate_categories/data/IPCC2006_PRIMAP.py @@ -7,7 +7,7 @@ "references": "", "institution": "PIK", "hierarchical": True, - "last_update": "2021-10-12", + "last_update": "2024-10-10", "version": "2006", "total_sum": True, "canonical_top_level_category": "0", @@ -1737,7 +1737,10 @@ "title": "Livestock", "comment": "Methane emissions from enteric fermentation, and methane and nitrous oxide emissions from manure management.", "alternative_codes": ["3A"], - "info": {"gases": ["CH4"], "corresponding_categories_IPCC1996": ["4"]}, + "info": { + "gases": ["N2O", "CH4"], + "corresponding_categories_IPCC1996": ["4"], + }, "children": [["3.A.1", "3.A.2"]], }, "3.A.1": { diff --git a/climate_categories/data/IPCC2006_PRIMAP.yaml b/climate_categories/data/IPCC2006_PRIMAP.yaml index a504886e..d0de1135 100644 --- a/climate_categories/data/IPCC2006_PRIMAP.yaml +++ b/climate_categories/data/IPCC2006_PRIMAP.yaml @@ -6,7 +6,7 @@ comment: IPCC classification of green-house gas emissions into categories, 2006 references: '' institution: PIK hierarchical: true -last_update: '2021-10-12' +last_update: '2024-10-10' version: '2006' total_sum: true canonical_top_level_category: '0' @@ -3204,6 +3204,7 @@ categories: - 3A info: gases: + - N2O - CH4 corresponding_categories_IPCC1996: - '4' diff --git a/data_generation/IPCC2006_PRIMAP.py b/data_generation/IPCC2006_PRIMAP.py index b28a1517..487dc38b 100644 --- a/data_generation/IPCC2006_PRIMAP.py +++ b/data_generation/IPCC2006_PRIMAP.py @@ -115,7 +115,7 @@ def main(): name=name, title=title, comment=comment, - last_update=datetime.date.fromisoformat("2021-10-12"), + last_update=datetime.date.fromisoformat("2024-10-10"), categories=categories, children=children, ) From 8048a6eec1eeeb20521e0a54f4da8ae2c1710f56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Thu, 10 Oct 2024 16:35:56 +0200 Subject: [PATCH 2/8] fix: use pdf-table-extraction instead of camelot, update makefile to fit --- Makefile | 45 ++++++++++++++++++++------------------------- setup.cfg | 4 ++-- 2 files changed, 22 insertions(+), 27 deletions(-) diff --git a/Makefile b/Makefile index 46c73d18..f946731b 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: clean docs help update-venv cache test test-full lint coverage release update-citation +.PHONY: docs help virtual-environment install-pre-commit update-venv cache test test-full lint coverage release update-citation .DEFAULT_GOAL := help define PRINT_HELP_PYSCRIPT @@ -21,36 +21,29 @@ lint: venv ## check style with pre-commit hooks test: venv ## run tests quickly with the default Python venv/bin/pytest --xdoc -rx -test-all: venv ## run tests with all Python versions; needs python versions already set up - tox -p +test-all: venv ## run tests on every Python version with tox + venv/bin/tox -p coverage: venv ## check code coverage quickly with the default Python venv/bin/coverage run --source climate_categories -m pytest --xdoc -rx venv/bin/coverage report -m venv/bin/coverage html + ls htmlcov/index.html -docs: venv ## generate Sphinx HTML documentation - $(MAKE) -C docs clean - $(MAKE) -C docs html +clean-docs: venv ## Remove generated parts of documentation, then build docs + . venv/bin/activate ; $(MAKE) -C docs clean + . venv/bin/activate ; $(MAKE) -C docs html -servedocs: docs ## compile the docs watching for changes - watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D . +docs: venv ## generate Sphinx HTML documentation, including API docs + . venv/bin/activate ; $(MAKE) -C docs html -clean: ## clean up after dist - rm -rf dist/ - rm -rf climate_categories.egg-info/ - rm -rf build/ - -update-citation: ## Update the citation information from zenodo - venv/bin/python update_citation_info.py - git commit -am 'Update citation information from zenodo.' - -release: dist ## package and upload a release +release: venv dist ## package and upload a release venv/bin/twine upload --repository climate-categories dist/* dist: clean venv ## builds source and wheel package - venv/bin/python -m build - ls -l dist + # because we update the citation info after releasing on github and zenodo but + # before building for pypi, we need to force the correct version. + SETUPTOOLS_SCM_PRETEND_VERSION=0.10.1 venv/bin/python -m build install: clean ## install the package to the active Python's site-packages python setup.py install @@ -58,14 +51,16 @@ install: clean ## install the package to the active Python's site-packages virtual-environment: venv ## setup a virtual environment for development venv: setup.py pyproject.toml setup.cfg - [ -d venv ] || python3 -m venv venv - venv/bin/python -m pip install -e .[dev] + [ -d venv ] || python3 -m venv --system-site-packages venv + venv/bin/python -m pip install --upgrade wheel uv + . venv/bin/activate ; venv/bin/uv pip install --upgrade -e .[dev] touch venv -update-venv: +update-venv: ## update all packages in the development environment [ -d venv ] || python3 -m venv venv - venv/bin/python -m pip install --upgrade pip wheel - venv/bin/python -m pip install --upgrade --upgrade-strategy eager -e .[dev] + venv/bin/python .check_python_version.py + venv/bin/python -m pip install --upgrade wheel uv + . venv/bin/activate ; venv/bin/uv pip install --upgrade --resolution highest -e .[dev] touch venv install-pre-commit: update-venv ## install the pre-commit hooks diff --git a/setup.cfg b/setup.cfg index 99a0831b..79f9bada 100644 --- a/setup.cfg +++ b/setup.cfg @@ -68,8 +68,8 @@ dev = pycountry lxml tqdm -ipcc = - camelot-py[base] >= 0.11.0 + pypdf-table-extraction + numpy < 2 [options.package_data] * = From 8793d1301047ea9452732337b920477dbe2875dc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Wed, 23 Oct 2024 13:36:27 +0200 Subject: [PATCH 3/8] feat: add target to Makefile to re-generate all data files --- Makefile | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/Makefile b/Makefile index f946731b..8ce056ae 100644 --- a/Makefile +++ b/Makefile @@ -84,6 +84,24 @@ cache: climate_categories/data/ISO3_GCAM.py cache: climate_categories/data/BURDI.py cache: climate_categories/data/BURDI_class.py ## Generate Python specs from YAML files +data: climate_categories/data/BURDI_class.yaml +data: climate_categories/data/BURDI.yaml +data: climate_categories/data/CRF1999.yaml +data: climate_categories/data/CRF2013_2021.yaml +data: climate_categories/data/CRF2013_2022.yaml +data: climate_categories/data/CRF2013_2023.yaml +data: climate_categories/data/CRF2013.yaml +data: climate_categories/data/CRFDI_class.yaml +data: climate_categories/data/CRFDI.yaml +data: climate_categories/data/gas.yaml +data: climate_categories/data/IPCC1996.yaml +data: climate_categories/data/IPCC2006_PRIMAP.yaml +data: climate_categories/data/IPCC2006.yaml +data: climate_categories/data/ISO3_GCAM.yaml +data: climate_categories/data/ISO3.yaml +data: climate_categories/data/RCMIP.yaml ## Generate data files + + climate_categories/data/%.yaml: data_generation/%.py data_generation/utils.py venv/bin/python $< From 91c8484c0fceca204381323b2a5a454af1e40a06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Wed, 23 Oct 2024 13:36:52 +0200 Subject: [PATCH 4/8] fix: re-generate all data files --- climate_categories/data/RCMIP.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/climate_categories/data/RCMIP.yaml b/climate_categories/data/RCMIP.yaml index 70fad7b2..e6881189 100644 --- a/climate_categories/data/RCMIP.yaml +++ b/climate_categories/data/RCMIP.yaml @@ -1,10 +1,10 @@ name: RCMIP title: Emissions categories from the Reduced Complexity Model Intercomparison Project (RCMIP) -comment: "AFOLU in the SSPDb is AFOLU minus any agriculture\nrelated fossil fuel based\ - \ emissions hence is not the same as the\nWG3 AFOLU definition. Rather AFOLU in\ - \ the SSPDb is AFOLU as expected by\nMAGICC (i.e. exluding agriculture related fossil\ - \ fuel use), hence\nwe call it MAGICC AFOLU." +comment: "AFOLU in the SSPDb is AFOLU minus any agriculture\nrelated fossil fuel based + emissions hence is not the same as the\nWG3 AFOLU definition. Rather AFOLU in the + SSPDb is AFOLU as expected by\nMAGICC (i.e. exluding agriculture related fossil + fuel use), hence\nwe call it MAGICC AFOLU." references: 'Nicholls, Z. R. J., Meinshausen, M., Lewis, J., Gieseke, R., Dommenget, D., Dorheim, K., Fan, C.-S., Fuglestvedt, J. S., Gasser, T., Golüke, U., Goodwin, P., Hartin, C., Hope, A. P., Kriegler, E., Leach, N. J., Marchegiani, D., McBride, From e5f99eb9a7bce4e6d028a83da38784a175ea9b0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Wed, 23 Oct 2024 13:38:58 +0200 Subject: [PATCH 5/8] feat: re-generate gas categorization to include newer mixtures --- climate_categories/data/gas.py | 21 ++++++++++++++++-- climate_categories/data/gas.yaml | 38 ++++++++++++++++++++++++++++++-- data_generation/gas.py | 4 ++-- 3 files changed, 57 insertions(+), 6 deletions(-) diff --git a/climate_categories/data/gas.py b/climate_categories/data/gas.py index 7aad256a..0553b130 100644 --- a/climate_categories/data/gas.py +++ b/climate_categories/data/gas.py @@ -7,8 +7,8 @@ "references": "Derived from openscm_units (https://github.com/openscm/openscm-units) 'standard gases' and mixtures.", "institution": "openscm", "hierarchical": True, - "last_update": "2021-05-27", - "version": "0.3.0", + "last_update": "2024-10-23", + "version": "0.3.1", "total_sum": False, "categories": { "C": {"title": "carbon", "alternative_codes": ["carbon"]}, @@ -112,6 +112,7 @@ "HFE227ea": {"title": "HFE227ea"}, "HFE236ea2": {"title": "HFE236ea2"}, "HFE236fa": {"title": "HFE236fa"}, + "HFE254cb2": {"title": "HFE254cb2"}, "HFE245fa1": {"title": "HFE245fa1"}, "HFE263fb2": {"title": "HFE263fb2"}, "HFE329mcc2": {"title": "HFE329mcc2"}, @@ -122,20 +123,36 @@ "HFE356pcf3": {"title": "HFE356pcf3"}, "HFE365mcf3": {"title": "HFE365mcf3"}, "HFE374pc2": {"title": "HFE374pc2"}, + "HFE7100": {"title": "HFE7100"}, + "HFE7200": {"title": "HFE7200"}, "PFPMIE": {"title": "PFPMIE"}, "HFO1234yf": {"title": "HFO1234yf"}, "HFO1234ze": {"title": "HFO1234ze"}, + "CHOH": {"title": "CHOH"}, + "CHOCHF2": {"title": "CHOCHF2"}, + "CHOCH3": {"title": "CHOCH3"}, + "CF3": {"title": "CF3"}, + "CF3I": {"title": "CF3I"}, + "CFOCH3": {"title": "CFOCH3"}, + "CF3CH2OH": {"title": "CF3CH2OH"}, + "CF3CF2CH2OH": {"title": "CF3CF2CH2OH"}, "CCl4": {"title": "CCl4"}, "CHCl3": {"title": "CHCl3"}, "CH2Cl2": {"title": "CH2Cl2"}, "CH3CCl3": {"title": "CH3CCl3"}, + "CH3OCH3": {"title": "CH3OCH3"}, "CH3Cl": {"title": "CH3Cl"}, + "CHBrF2": {"title": "CHBrF2"}, + "CH2Br2": {"title": "CH2Br2"}, "CH3Br": {"title": "CH3Br"}, "SF5CF3": {"title": "SF5CF3"}, "SF6": {"title": "SF6"}, "SO2F2": {"title": "SO2F2"}, "NF3": {"title": "NF3"}, "HCO1130": {"title": "HCO1130"}, + "HGalden1040x": {"title": "HGalden1040x"}, + "HG10": {"title": "HG10"}, + "HG01": {"title": "HG01"}, "CFC400": { "title": "CFC400", "comment": "The refrigerant CFC400, which is a mixture of 50% CFC12, 50% CFC114.", diff --git a/climate_categories/data/gas.yaml b/climate_categories/data/gas.yaml index 065fdea9..3f0ac986 100644 --- a/climate_categories/data/gas.yaml +++ b/climate_categories/data/gas.yaml @@ -5,8 +5,8 @@ references: Derived from openscm_units (https://github.com/openscm/openscm-units 'standard gases' and mixtures. institution: openscm hierarchical: true -last_update: '2021-05-27' -version: 0.3.0 +last_update: '2024-10-23' +version: 0.3.1 total_sum: false categories: C: @@ -248,6 +248,8 @@ categories: title: HFE236ea2 HFE236fa: title: HFE236fa + HFE254cb2: + title: HFE254cb2 HFE245fa1: title: HFE245fa1 HFE263fb2: @@ -268,12 +270,32 @@ categories: title: HFE365mcf3 HFE374pc2: title: HFE374pc2 + HFE7100: + title: HFE7100 + HFE7200: + title: HFE7200 PFPMIE: title: PFPMIE HFO1234yf: title: HFO1234yf HFO1234ze: title: HFO1234ze + CHOH: + title: CHOH + CHOCHF2: + title: CHOCHF2 + CHOCH3: + title: CHOCH3 + CF3: + title: CF3 + CF3I: + title: CF3I + CFOCH3: + title: CFOCH3 + CF3CH2OH: + title: CF3CH2OH + CF3CF2CH2OH: + title: CF3CF2CH2OH CCl4: title: CCl4 CHCl3: @@ -282,8 +304,14 @@ categories: title: CH2Cl2 CH3CCl3: title: CH3CCl3 + CH3OCH3: + title: CH3OCH3 CH3Cl: title: CH3Cl + CHBrF2: + title: CHBrF2 + CH2Br2: + title: CH2Br2 CH3Br: title: CH3Br SF5CF3: @@ -296,6 +324,12 @@ categories: title: NF3 HCO1130: title: HCO1130 + HGalden1040x: + title: HGalden1040x + HG10: + title: HG10 + HG01: + title: HG01 CFC400: title: CFC400 comment: The refrigerant CFC400, which is a mixture of 50% CFC12, 50% CFC114. diff --git a/data_generation/gas.py b/data_generation/gas.py index a74b26ce..ab10ba80 100644 --- a/data_generation/gas.py +++ b/data_generation/gas.py @@ -28,8 +28,8 @@ def main(): "comment": "Gases and other climate-forcing substances", "references": "Derived from openscm_units " "(https://github.com/openscm/openscm-units) 'standard gases' and mixtures.", - "last_update": "2021-05-27", - "version": "0.3.0", + "last_update": "2024-10-23", + "version": "0.3.1", "institution": "openscm", "categories": categories, "hierarchical": True, From 82aed1336a27dada9729e700390fc5673cd4e88b Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 23 Oct 2024 11:40:24 +0000 Subject: [PATCH 6/8] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- changelog_unreleased/163.improvement.rst | 2 +- data_generation/IPCC2006_PRIMAP.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/changelog_unreleased/163.improvement.rst b/changelog_unreleased/163.improvement.rst index 29205893..bfb5082e 100644 --- a/changelog_unreleased/163.improvement.rst +++ b/changelog_unreleased/163.improvement.rst @@ -1 +1 @@ -* add M.BIO category \ No newline at end of file +* add M.BIO category diff --git a/data_generation/IPCC2006_PRIMAP.py b/data_generation/IPCC2006_PRIMAP.py index ecbf10bc..bcb43f26 100644 --- a/data_generation/IPCC2006_PRIMAP.py +++ b/data_generation/IPCC2006_PRIMAP.py @@ -37,9 +37,9 @@ def main(): "comment": "Agricultural part of AFOLU excluding livestock", }, "3.A.1.i": {"title": "Poultry", "comment": "From CRF data"}, - "M.BIO" : { - "title" : "CO₂ emissions from biomass burning", - "comment" : "CO₂ emissions from biomass burning for energy use", + "M.BIO": { + "title": "CO₂ emissions from biomass burning", + "comment": "CO₂ emissions from biomass burning for energy use", }, "M.BK": { "title": "International Bunkers", From 182106cb91f468fb3e9cfa0f544c528c9012e6ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Wed, 23 Oct 2024 13:41:52 +0200 Subject: [PATCH 7/8] docs: changelog --- changelog_unreleased/162.fix.rst | 1 + changelog_unreleased/162.improvement.rst | 1 + 2 files changed, 2 insertions(+) create mode 100644 changelog_unreleased/162.fix.rst create mode 100644 changelog_unreleased/162.improvement.rst diff --git a/changelog_unreleased/162.fix.rst b/changelog_unreleased/162.fix.rst new file mode 100644 index 00000000..9440362f --- /dev/null +++ b/changelog_unreleased/162.fix.rst @@ -0,0 +1 @@ +* Use pypdf-table-extraction instead of kamelot for extracting categorizations from pdf files. diff --git a/changelog_unreleased/162.improvement.rst b/changelog_unreleased/162.improvement.rst new file mode 100644 index 00000000..76ad3a95 --- /dev/null +++ b/changelog_unreleased/162.improvement.rst @@ -0,0 +1 @@ +* Add new gas mixtures to the gas categorization. From 89c3027d2b981aca39b78d2a2254725f0bdcf4be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mika=20Pfl=C3=BCger?= Date: Wed, 23 Oct 2024 14:31:44 +0200 Subject: [PATCH 8/8] Update 162.fix.rst fix typo --- changelog_unreleased/162.fix.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/changelog_unreleased/162.fix.rst b/changelog_unreleased/162.fix.rst index 9440362f..689c19ef 100644 --- a/changelog_unreleased/162.fix.rst +++ b/changelog_unreleased/162.fix.rst @@ -1 +1 @@ -* Use pypdf-table-extraction instead of kamelot for extracting categorizations from pdf files. +* Use pypdf-table-extraction instead of camelot for extracting categorizations from pdf files.