Add ESMValTool example metric (#40)

.github/workflows/ci.yaml

@@ -94,6 +94,7 @@ jobs:
           uv run --package ref pytest packages/ref -r a -v --doctest-modules --cov=packages/ref/src --cov-report=term
           uv run --package ref-core pytest packages/ref-core -r a -v --doctest-modules --cov=packages/ref-core/src --cov-report=term --cov-append
           uv run --package ref-metrics-example pytest packages/ref-metrics-example -r a -v --doctest-modules --cov=packages/ref-metrics-example/src --cov-report=term --cov-append
+          uv run --package ref-metrics-esmvaltool pytest packages/ref-metrics-esmvaltool -r a -v --doctest-modules --cov=packages/ref-metrics-esmvaltool/src --cov-report=term --cov-append
           uv run coverage xml
           # Run integration tests (without adding to the coverage)
           uv run pytest tests -r a -v

Makefile

@@ -32,6 +32,7 @@ mypy:  ## run mypy on the codebase
 	MYPYPATH=stubs uv run --package ref-core mypy packages/ref-core
 	MYPYPATH=stubs uv run --package ref mypy packages/ref
 	MYPYPATH=stubs uv run --package ref-metrics-example mypy packages/ref-metrics-example
+	MYPYPATH=stubs uv run --package ref-metrics-esmvaltool mypy packages/ref-metrics-esmvaltool
 
 .PHONY: ruff-fixes
 ruff-fixes:  ## fix the code using ruff
@@ -56,14 +57,20 @@ test-metrics-example:  ## run the tests
 		pytest packages/ref-metrics-example \
 		-r a -v --doctest-modules --cov=packages/ref-metrics-example/src
 
+.PHONY: test-metrics-esmvaltool
+test-metrics-esmvaltool:  ## run the tests
+	uv run --package ref-metrics-esmvaltool \
+		pytest packages/ref-metrics-esmvaltool \
+		-r a -v --doctest-modules --cov=packages/ref-metrics-esmvaltool/src
+
 .PHONY: test-integration
 test-integration:  ## run the integration tests
 	uv run \
 		pytest tests \
 		-r a -v
 
 .PHONY: test
-test: test-core test-ref test-metrics-example test-integration ## run the tests
+test: test-core test-ref test-metrics-example test-metrics-esmvaltool test-integration ## run the tests
 
 # Note on code coverage and testing:
 # If you want to debug what is going on with coverage, we have found

changelog/40.feature.md

@@ -0,0 +1 @@
+Added an example ESMValTool metric.

docs/gen_doc_stubs.py

@@ -130,6 +130,7 @@ def write_module_page(
 write_module_page("ref")
 write_module_page("ref_core")
 write_module_page("ref_metrics_example")
+write_module_page("ref_metrics_esmvaltool")
 
 with mkdocs_gen_files.open(ROOT_DIR / "NAVIGATION.md", "w") as fh:
     fh.writelines(nav.build_literate_nav())

packages/ref-metrics-esmvaltool/README.md

@@ -0,0 +1,8 @@
+# ref-metrics-esmvaltool
+
+Use [ESMValTool](https://esmvaltool.org/) as a REF metrics provider.
+
+To use this, install ESMValTool and then install the REF into the same conda
+environment.
+
+See [running-metrics-locally](https://cmip-ref.readthedocs.io/en/latest/how-to-guides/running-metrics-locally/) for usage instructions.

packages/ref-metrics-esmvaltool/pyproject.toml

@@ -0,0 +1,40 @@
+[project]
+name = "ref-metrics-esmvaltool"
+version = "0.1.0"
+description = "ESMValTool metrics provider for the CMIP Rapid Evaluation Framework"
+readme = "README.md"
+authors = [
+    { name = "ESMValTool development team", email = "esmvaltool-dev@listserv.dfn.de " }
+]
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "pooch >= 1.8",
+    "ref-core",
+    "ruamel.yaml >= 0.18",
+    "xarray >= 2022",
+]
+
+[project.license]
+text = "Apache-2.0"
+
+[tool.uv]
+dev-dependencies = [
+    "pytest-mock >= 3.12",
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

packages/ref-metrics-esmvaltool/src/ref_metrics_esmvaltool/__init__.py

@@ -0,0 +1,16 @@
+"""
+Rapid evaluating CMIP data with ESMValTool.
+"""
+
+import importlib.metadata
+
+from ref_core.providers import MetricsProvider
+
+from ref_metrics_esmvaltool.example import GlobalMeanTimeseries
+
+__version__ = importlib.metadata.version("ref_metrics_esmvaltool")
+__core_version__ = importlib.metadata.version("ref_core")
+
+# Initialise the metrics manager and register the example metric
+provider = MetricsProvider("ESMValTool", __version__)
+provider.register(GlobalMeanTimeseries())

packages/ref-metrics-esmvaltool/src/ref_metrics_esmvaltool/example.py

@@ -0,0 +1,108 @@
+from typing import Any
+
+import xarray
+from ref_core.datasets import FacetFilter, SourceDatasetType
+from ref_core.metrics import DataRequirement, Metric, MetricExecutionDefinition, MetricResult
+from ruamel.yaml import YAML
+
+from ref_metrics_esmvaltool.recipe import dataframe_to_recipe, load_recipe, run_recipe
+
+yaml = YAML()
+
+
+def format_cmec_output_bundle(dataset: xarray.Dataset) -> dict[str, Any]:
+    """
+    Create a simple CMEC output bundle for the dataset.
+
+    Parameters
+    ----------
+    dataset
+        Processed dataset
+
+    Returns
+    -------
+        A CMEC output bundle ready to be written to disk
+    """
+    # TODO: Check how timeseries data are generally serialised
+    cmec_output = {
+        "DIMENSIONS": {
+            "dimensions": {
+                "source_id": {dataset.attrs["source_id"]: {}},
+                "region": {"global": {}},
+                "variable": {"tas": {}},
+            },
+            "json_structure": [
+                "model",
+                "region",
+                "statistic",
+            ],
+        },
+        # Is the schema tracked?
+        "SCHEMA": {
+            "name": "CMEC-REF",
+            "package": "example",
+            "version": "v1",
+        },
+        "RESULTS": {
+            dataset.attrs["source_id"]: {"global": {"tas": 0}},
+        },
+    }
+
+    return cmec_output
+
+
+class GlobalMeanTimeseries(Metric):
+    """
+    Calculate the annual mean global mean timeseries for a dataset
+    """
+
+    name = "Global Mean Timeseries"
+    slug = "esmvaltool-global-mean-timeseries"
+
+    data_requirements = (
+        DataRequirement(
+            source_type=SourceDatasetType.CMIP6,
+            filters=(FacetFilter(facets={"variable_id": ("tas",)}),),
+            # Add cell areas to the groups
+            # constraints=(AddCellAreas(),),
+            # Run the metric on each unique combination of model, variable, experiment, and variant
+            group_by=("source_id", "variable_id", "experiment_id", "variant_label"),
+        ),
+    )
+
+    def run(self, definition: MetricExecutionDefinition) -> MetricResult:
+        """
+        Run a metric
+
+        Parameters
+        ----------
+        definition
+            A description of the information needed for this execution of the metric
+
+        Returns
+        -------
+        :
+            The result of running the metric.
+        """
+        # Load recipe and clear unwanted elements
+        recipe = load_recipe("examples/recipe_python.yml")
+        recipe["datasets"].clear()
+        recipe["diagnostics"].pop("map")
+        variables = recipe["diagnostics"]["timeseries"]["variables"]
+        variables.clear()
+
+        # Prepare updated variables section in recipe.
+        recipe_variables = dataframe_to_recipe(definition.metric_dataset[SourceDatasetType.CMIP6].datasets)
+        for variable in recipe_variables.values():
+            variable["preprocessor"] = "annual_mean_global"
+            variable["caption"] = "Annual global mean {long_name} according to {dataset}."
+
+        # Populate recipe with new variables/datasets.
+        variables.update(recipe_variables)
+
+        # Run recipe
+        result_dir = run_recipe(recipe, definition)
+        result = next(result_dir.glob("work/timeseries/script1/*.nc"))
+        annual_mean_global_mean_timeseries = xarray.open_dataset(result)
+
+        return MetricResult.build(definition, format_cmec_output_bundle(annual_mean_global_mean_timeseries))