Merge pull request #162 from primap-community/camelot-new-name

Makefile

@@ -1,4 +1,4 @@
-.PHONY: clean docs help update-venv cache test test-full lint coverage release update-citation
+.PHONY: docs help virtual-environment install-pre-commit update-venv cache test test-full lint coverage release update-citation
 .DEFAULT_GOAL := help
 
 define PRINT_HELP_PYSCRIPT
@@ -21,51 +21,46 @@ lint: venv ## check style with pre-commit hooks
 test: venv ## run tests quickly with the default Python
 	venv/bin/pytest --xdoc -rx
 
-test-all: venv ## run tests with all Python versions; needs python versions already set up
-	tox -p
+test-all: venv ## run tests on every Python version with tox
+	venv/bin/tox -p
 
 coverage: venv ## check code coverage quickly with the default Python
 	venv/bin/coverage run --source climate_categories -m pytest --xdoc -rx
 	venv/bin/coverage report -m
 	venv/bin/coverage html
+	ls htmlcov/index.html
 
-docs: venv ## generate Sphinx HTML documentation
-	$(MAKE) -C docs clean
-	$(MAKE) -C docs html
+clean-docs: venv ## Remove generated parts of documentation, then build docs
+	. venv/bin/activate ; $(MAKE) -C docs clean
+	. venv/bin/activate ; $(MAKE) -C docs html
 
-servedocs: docs ## compile the docs watching for changes
-	watchmedo shell-command -p '*.rst' -c '$(MAKE) -C docs html' -R -D .
+docs: venv ## generate Sphinx HTML documentation, including API docs
+	. venv/bin/activate ; $(MAKE) -C docs html
 
-clean: ## clean up after dist
-	rm -rf dist/
-	rm -rf climate_categories.egg-info/
-	rm -rf build/
-
-update-citation: ## Update the citation information from zenodo
-	venv/bin/python update_citation_info.py
-	git commit -am 'Update citation information from zenodo.'
-
-release: dist ## package and upload a release
+release: venv dist ## package and upload a release
 	venv/bin/twine upload --repository climate-categories dist/*
 
 dist: clean venv ## builds source and wheel package
-	venv/bin/python -m build
-	ls -l dist
+	# because we update the citation info after releasing on github and zenodo but
+	# before building for pypi, we need to force the correct version.
+	SETUPTOOLS_SCM_PRETEND_VERSION=0.10.1 venv/bin/python -m build
 
 install: clean ## install the package to the active Python's site-packages
 	python setup.py install
 
 virtual-environment: venv ## setup a virtual environment for development
 
 venv: setup.py pyproject.toml setup.cfg
-	[ -d venv ] || python3 -m venv venv
-	venv/bin/python -m pip install -e .[dev]
+	[ -d venv ] || python3 -m venv --system-site-packages venv
+	venv/bin/python -m pip install --upgrade wheel uv
+	. venv/bin/activate ; venv/bin/uv pip install --upgrade -e .[dev]
 	touch venv
 
-update-venv:
+update-venv: ## update all packages in the development environment
 	[ -d venv ] || python3 -m venv venv
-	venv/bin/python -m pip install --upgrade pip wheel
-	venv/bin/python -m pip install --upgrade --upgrade-strategy eager -e .[dev]
+	venv/bin/python .check_python_version.py
+	venv/bin/python -m pip install --upgrade wheel uv
+	. venv/bin/activate ; venv/bin/uv pip  install --upgrade --resolution highest -e .[dev]
 	touch venv
 
 install-pre-commit: update-venv ## install the pre-commit hooks
@@ -89,6 +84,24 @@ cache: climate_categories/data/ISO3_GCAM.py
 cache: climate_categories/data/BURDI.py
 cache: climate_categories/data/BURDI_class.py  ## Generate Python specs from YAML files
 
+data: climate_categories/data/BURDI_class.yaml
+data: climate_categories/data/BURDI.yaml
+data: climate_categories/data/CRF1999.yaml
+data: climate_categories/data/CRF2013_2021.yaml
+data: climate_categories/data/CRF2013_2022.yaml
+data: climate_categories/data/CRF2013_2023.yaml
+data: climate_categories/data/CRF2013.yaml
+data: climate_categories/data/CRFDI_class.yaml
+data: climate_categories/data/CRFDI.yaml
+data: climate_categories/data/gas.yaml
+data: climate_categories/data/IPCC1996.yaml
+data: climate_categories/data/IPCC2006_PRIMAP.yaml
+data: climate_categories/data/IPCC2006.yaml
+data: climate_categories/data/ISO3_GCAM.yaml
+data: climate_categories/data/ISO3.yaml
+data: climate_categories/data/RCMIP.yaml  ## Generate data files
+
+
 climate_categories/data/%.yaml: data_generation/%.py data_generation/utils.py
 	venv/bin/python $<
 

changelog_unreleased/162.fix.rst

@@ -0,0 +1 @@
+* Use pypdf-table-extraction instead of camelot for extracting categorizations from pdf files.

changelog_unreleased/162.improvement.rst

@@ -0,0 +1 @@
+* Add new gas mixtures to the gas categorization.

climate_categories/data/BURDI.yaml

@@ -2,9 +2,11 @@ name: BURDI
 title: BUR GHG emission categories (DI query interface)
 comment: Biannual Update Report categories of GHG emissions and removals and some
   other quantities as obtained from the di.unfccc.int flexible query interface
-references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
+references:
+  https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
   and decision 17/CP.8, 'Guidelines for the preparation of national communications
-  from Parties not included in Annex I to the Convention' available at https://unfccc.int/files/meetings/workshops/other_meetings/application/pdf/dec17-cp.pdf
+  from Parties not included in Annex I to the Convention' available at
+  https://unfccc.int/files/meetings/workshops/other_meetings/application/pdf/dec17-cp.pdf
 institution: UNFCCC
 hierarchical: true
 last_update: '2021-11-05'

climate_categories/data/BURDI_class.yaml

@@ -3,7 +3,8 @@ title: BUR GHG emission categories (DI query interface) + classifications
 comment: Biannual Update Report categories of GHG emissions and removals and some
   other quantities as obtained from the di.unfccc.int flexible query interface extended
   by sub-classifications also provided by the DI flexible query interface
-references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
+references:
+  https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
 institution: UNFCCC
 hierarchical: true
 last_update: '2021-11-05'

climate_categories/data/CRFDI.yaml

@@ -2,7 +2,8 @@ name: CRFDI
 title: CRF GHG emission categories (DI query interface)
 comment: Common Reporting Format categories of GHG emissions and removals and some
   other quantities as obtained from the di.unfccc.int flexible query interface
-references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
+references:
+  https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
 institution: UNFCCC
 hierarchical: true
 last_update: '2021-11-05'

climate_categories/data/CRFDI_class.yaml

@@ -3,7 +3,8 @@ title: CRF GHG emission categories (DI query interface) + classifications
 comment: Common Reporting Format categories of GHG emissions and removals and some
   other quantities as obtained from the di.unfccc.int flexible query interface extended
   by sub-classifications also provided by the DI flexible query interface
-references: https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
+references:
+  https://unfccc.int/process-and-meetings/transparency-and-reporting/greenhouse-gas-data/data-interface-help#eq-7
 institution: UNFCCC
 hierarchical: true
 last_update: '2021-11-05'

climate_categories/data/IPCC2006_PRIMAP.py

@@ -7,7 +7,7 @@
     "references": "",
     "institution": "PIK",
     "hierarchical": True,
-    "last_update": "2021-10-12",
+    "last_update": "2024-10-10",
     "version": "2006",
     "total_sum": True,
     "canonical_top_level_category": "0",
@@ -1737,7 +1737,10 @@
             "title": "Livestock",
             "comment": "Methane emissions from enteric fermentation, and methane and nitrous oxide emissions from manure management.",
             "alternative_codes": ["3A"],
-            "info": {"gases": ["CH4"], "corresponding_categories_IPCC1996": ["4"]},
+            "info": {
+                "gases": ["N2O", "CH4"],
+                "corresponding_categories_IPCC1996": ["4"],
+            },
             "children": [["3.A.1", "3.A.2"]],
         },
         "3.A.1": {

climate_categories/data/IPCC2006_PRIMAP.yaml

@@ -6,7 +6,7 @@ comment: IPCC classification of green-house gas emissions into categories, 2006
 references: ''
 institution: PIK
 hierarchical: true
-last_update: '2021-10-12'
+last_update: '2024-10-10'
 version: '2006'
 total_sum: true
 canonical_top_level_category: '0'

climate_categories/data/RCMIP.yaml

@@ -1,10 +1,10 @@
 name: RCMIP
 title: Emissions categories from the Reduced Complexity Model Intercomparison Project
   (RCMIP)
-comment: "AFOLU in the SSPDb is AFOLU minus any agriculture\nrelated fossil fuel based\
-  \ emissions hence is not the same as the\nWG3 AFOLU definition. Rather AFOLU in\
-  \ the SSPDb is AFOLU as expected by\nMAGICC (i.e. exluding agriculture related fossil\
-  \ fuel use), hence\nwe call it MAGICC AFOLU."
+comment: "AFOLU in the SSPDb is AFOLU minus any agriculture\nrelated fossil fuel based
+  emissions hence is not the same as the\nWG3 AFOLU definition. Rather AFOLU in the
+  SSPDb is AFOLU as expected by\nMAGICC (i.e. exluding agriculture related fossil
+  fuel use), hence\nwe call it MAGICC AFOLU."
 references: 'Nicholls, Z. R. J., Meinshausen, M., Lewis, J., Gieseke, R., Dommenget,
   D., Dorheim, K., Fan, C.-S., Fuglestvedt, J. S., Gasser, T., Golüke, U., Goodwin,
   P., Hartin, C., Hope, A. P., Kriegler, E., Leach, N. J., Marchegiani, D., McBride,

climate_categories/data/gas.py

@@ -7,8 +7,8 @@
     "references": "Derived from openscm_units (https://github.com/openscm/openscm-units) 'standard gases' and mixtures.",
     "institution": "openscm",
     "hierarchical": True,
-    "last_update": "2021-05-27",
-    "version": "0.3.0",
+    "last_update": "2024-10-23",
+    "version": "0.3.1",
     "total_sum": False,
     "categories": {
         "C": {"title": "carbon", "alternative_codes": ["carbon"]},
@@ -112,6 +112,7 @@
         "HFE227ea": {"title": "HFE227ea"},
         "HFE236ea2": {"title": "HFE236ea2"},
         "HFE236fa": {"title": "HFE236fa"},
+        "HFE254cb2": {"title": "HFE254cb2"},
         "HFE245fa1": {"title": "HFE245fa1"},
         "HFE263fb2": {"title": "HFE263fb2"},
         "HFE329mcc2": {"title": "HFE329mcc2"},
@@ -122,20 +123,36 @@
         "HFE356pcf3": {"title": "HFE356pcf3"},
         "HFE365mcf3": {"title": "HFE365mcf3"},
         "HFE374pc2": {"title": "HFE374pc2"},
+        "HFE7100": {"title": "HFE7100"},
+        "HFE7200": {"title": "HFE7200"},
         "PFPMIE": {"title": "PFPMIE"},
         "HFO1234yf": {"title": "HFO1234yf"},
         "HFO1234ze": {"title": "HFO1234ze"},
+        "CHOH": {"title": "CHOH"},
+        "CHOCHF2": {"title": "CHOCHF2"},
+        "CHOCH3": {"title": "CHOCH3"},
+        "CF3": {"title": "CF3"},
+        "CF3I": {"title": "CF3I"},
+        "CFOCH3": {"title": "CFOCH3"},
+        "CF3CH2OH": {"title": "CF3CH2OH"},
+        "CF3CF2CH2OH": {"title": "CF3CF2CH2OH"},
         "CCl4": {"title": "CCl4"},
         "CHCl3": {"title": "CHCl3"},
         "CH2Cl2": {"title": "CH2Cl2"},
         "CH3CCl3": {"title": "CH3CCl3"},
+        "CH3OCH3": {"title": "CH3OCH3"},
         "CH3Cl": {"title": "CH3Cl"},
+        "CHBrF2": {"title": "CHBrF2"},
+        "CH2Br2": {"title": "CH2Br2"},
         "CH3Br": {"title": "CH3Br"},
         "SF5CF3": {"title": "SF5CF3"},
         "SF6": {"title": "SF6"},
         "SO2F2": {"title": "SO2F2"},
         "NF3": {"title": "NF3"},
         "HCO1130": {"title": "HCO1130"},
+        "HGalden1040x": {"title": "HGalden1040x"},
+        "HG10": {"title": "HG10"},
+        "HG01": {"title": "HG01"},
         "CFC400": {
             "title": "CFC400",
             "comment": "The refrigerant CFC400, which is a mixture of 50% CFC12, 50% CFC114.",

climate_categories/data/gas.yaml

@@ -5,8 +5,8 @@ references: Derived from openscm_units (https://github.com/openscm/openscm-units
   'standard gases' and mixtures.
 institution: openscm
 hierarchical: true
-last_update: '2021-05-27'
-version: 0.3.0
+last_update: '2024-10-23'
+version: 0.3.1
 total_sum: false
 categories:
   C:
@@ -248,6 +248,8 @@ categories:
     title: HFE236ea2
   HFE236fa:
     title: HFE236fa
+  HFE254cb2:
+    title: HFE254cb2
   HFE245fa1:
     title: HFE245fa1
   HFE263fb2:
@@ -268,12 +270,32 @@ categories:
     title: HFE365mcf3
   HFE374pc2:
     title: HFE374pc2
+  HFE7100:
+    title: HFE7100
+  HFE7200:
+    title: HFE7200
   PFPMIE:
     title: PFPMIE
   HFO1234yf:
     title: HFO1234yf
   HFO1234ze:
     title: HFO1234ze
+  CHOH:
+    title: CHOH
+  CHOCHF2:
+    title: CHOCHF2
+  CHOCH3:
+    title: CHOCH3
+  CF3:
+    title: CF3
+  CF3I:
+    title: CF3I
+  CFOCH3:
+    title: CFOCH3
+  CF3CH2OH:
+    title: CF3CH2OH
+  CF3CF2CH2OH:
+    title: CF3CF2CH2OH
   CCl4:
     title: CCl4
   CHCl3:
@@ -282,8 +304,14 @@ categories:
     title: CH2Cl2
   CH3CCl3:
     title: CH3CCl3
+  CH3OCH3:
+    title: CH3OCH3
   CH3Cl:
     title: CH3Cl
+  CHBrF2:
+    title: CHBrF2
+  CH2Br2:
+    title: CH2Br2
   CH3Br:
     title: CH3Br
   SF5CF3:
@@ -296,6 +324,12 @@ categories:
     title: NF3
   HCO1130:
     title: HCO1130
+  HGalden1040x:
+    title: HGalden1040x
+  HG10:
+    title: HG10
+  HG01:
+    title: HG01
   CFC400:
     title: CFC400
     comment: The refrigerant CFC400, which is a mixture of 50% CFC12, 50% CFC114.

data_generation/IPCC2006_PRIMAP.py

@@ -119,7 +119,7 @@ def main():
         name=name,
         title=title,
         comment=comment,
-        last_update=datetime.date.fromisoformat("2021-10-12"),
+        last_update=datetime.date.fromisoformat("2024-10-10"),
         categories=categories,
         children=children,
     )

data_generation/gas.py

@@ -28,8 +28,8 @@ def main():
         "comment": "Gases and other climate-forcing substances",
         "references": "Derived from openscm_units "
         "(https://github.com/openscm/openscm-units) 'standard gases' and mixtures.",
-        "last_update": "2021-05-27",
-        "version": "0.3.0",
+        "last_update": "2024-10-23",
+        "version": "0.3.1",
         "institution": "openscm",
         "categories": categories,
         "hierarchical": True,

setup.cfg

@@ -68,8 +68,8 @@ dev =
     pycountry
     lxml
     tqdm
-ipcc =
-    camelot-py[base] >= 0.11.0
+    pypdf-table-extraction
+    numpy < 2
 
 [options.package_data]
 * =