Merge pull request #8 from CMIP-REF/ref-package

.github/workflows/ci.yaml

@@ -74,6 +74,7 @@ jobs:
           make fetch-test-data
       - name: Run tests
         run: |
+          uv run --package ref pytest packages/ref -r a -v --doctest-modules --cov=packages/ref/src --cov-report=term
           uv run --package ref-core pytest packages/ref-core -r a -v --doctest-modules --cov=packages/ref-core/src --cov-report=term
           uv run --package ref-metrics-example pytest packages/ref-metrics-example -r a -v --doctest-modules --cov=packages/ref-metrics-example/src --cov-report=term --cov-append
           uv run coverage xml

.gitignore

@@ -148,3 +148,6 @@ dmypy.json
 
 # Esgpull
 .esgpull
+
+# Generated output
+out

Makefile

@@ -30,13 +30,20 @@ pre-commit:  ## run all the linting checks of the codebase
 .PHONY: mypy
 mypy:  ## run mypy on the codebase
 	MYPYPATH=stubs uv run --package ref-core mypy packages/ref-core
+	MYPYPATH=stubs uv run --package ref mypy packages/ref
 	MYPYPATH=stubs uv run --package ref-metrics-example mypy packages/ref-metrics-example
 
 .PHONY: ruff-fixes
 ruff-fixes:  ## fix the code using ruff
 	uv run ruff check --fix
 	uv run ruff format
 
+.PHONY: test-ref
+test-ref:  ## run the tests
+	uv run --package ref \
+		pytest packages/ref \
+		-r a -v --doctest-modules --cov=packages/ref/src
+
 .PHONY: test-core
 test-core:  ## run the tests
 	uv run --package ref-core \
@@ -56,7 +63,7 @@ test-integration:  ## run the integration tests
 		-r a -v
 
 .PHONY: test
-test: test-core test-metrics-example test-integration ## run the tests
+test: test-core test-ref test-metrics-example test-integration ## run the tests
 
 # Note on code coverage and testing:
 # If you want to debug what is going on with coverage, we have found

changelog/8.feature.md

@@ -0,0 +1 @@
+Adds the `ref` package with a basic CLI interface that will allow for users to interact with the database of jobs.

packages/ref-core/src/ref_core/exceptions.py

@@ -0,0 +1,4 @@
+class RefException(Exception):
+    """Base class for exceptions related to REF operations"""
+
+    pass

packages/ref/README.md

@@ -0,0 +1,10 @@
+# ref
+
+The `ref` package orchestrates the tracking and execution of model benchmarking metrics
+against CMIP data.
+
+
+## Usage
+
+The `ref` package exposes a command line interface (CLI) that can be used to
+interact with the

packages/ref/pyproject.toml

@@ -0,0 +1,41 @@
+[project]
+name = "ref"
+version = "0.1.0"
+description = "Application which runs the CMIP Rapid Evaluation Framework"
+readme = "README.md"
+authors = [
+    { name = "Jared Lewis", email = "jared.lewis@climate-resource.com" }
+]
+requires-python = ">=3.10"
+classifiers = [
+    "Development Status :: 4 - Beta",
+    "Intended Audience :: Developers",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Science/Research",
+    "Programming Language :: Python",
+    "Programming Language :: Python :: 3",
+    "Programming Language :: Python :: 3.10",
+    "Programming Language :: Python :: 3.11",
+    "Programming Language :: Python :: 3.12",
+    "Programming Language :: Python :: 3.13",
+    "Topic :: Scientific/Engineering",
+]
+dependencies = [
+    "ref-core",
+    "attrs>=24.2.0",
+    "cattrs>=24.1.2",
+    "environs>=11.0.0",
+    "tomlkit>=0.13.2",
+    "typer>=0.12.5",
+]
+
+[project.scripts]
+ref = "ref.cli:app"
+
+[tool.uv]
+dev-dependencies = [
+]
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"

packages/ref/src/ref/__init__.py

@@ -0,0 +1,8 @@
+"""
+Rapid evaluating CMIP data
+"""
+
+import importlib.metadata
+
+__version__ = importlib.metadata.version("ref")
+__core_version__ = importlib.metadata.version("ref_core")

packages/ref/src/ref/cli/__init__.py

@@ -0,0 +1,10 @@
+"""Entrypoint for the CLI"""
+
+import typer
+
+from ref.cli import config, sync
+
+app = typer.Typer(name="ref")
+
+app.command(name="sync")(sync.sync)
+app.add_typer(config.app, name="config")

packages/ref/src/ref/cli/config.py

@@ -0,0 +1,40 @@
+"""
+View and update the REF configuration
+"""
+
+from pathlib import Path
+
+import typer
+
+from ref.config import Config
+from ref.constants import config_filename
+
+app = typer.Typer(help=__doc__)
+
+
+@app.command()
+def list(configuration_directory: Path | None = typer.Option(None, help="Configuration directory")) -> None:
+    """
+    Print the current ref configuration
+
+    If a configuration directory is provided,
+    the configuration will attempt to load from the specified directory.
+    """
+    try:
+        if configuration_directory:
+            config = Config.load(configuration_directory / config_filename, allow_missing=False)
+        else:
+            config = Config.default()
+    except FileNotFoundError:
+        typer.secho("Configuration file not found", fg=typer.colors.RED)
+        raise typer.Exit(1)
+
+    print(config.dumps(defaults=True))
+
+
+@app.command()
+def update() -> None:
+    """
+    Update a configuration value
+    """
+    print("config")

packages/ref/src/ref/cli/sync.py

@@ -0,0 +1,11 @@
+import typer
+
+app = typer.Typer()
+
+
+@app.command()
+def sync() -> None:
+    """
+    Placeholder command for syncing data
+    """  # noqa: D401
+    print("syncing data")

packages/ref/src/ref/config.py

@@ -0,0 +1,218 @@
+"""Configuration management"""
+
+# The basics of the configuration management takes a lot of inspiration from the
+# `esgpull` configuration management system with some of the extra complexity removed.
+# https://github.com/ESGF/esgf-download/blob/main/esgpull/config.py
+
+from pathlib import Path
+from typing import Any
+
+import tomlkit
+from attrs import Factory, define, field
+from cattrs import Converter
+from cattrs.gen import make_dict_unstructure_fn, override
+from tomlkit import TOMLDocument
+
+from ref.constants import config_filename
+from ref.env import env
+
+
+def _pop_empty(d: dict[str, Any]) -> None:
+    keys = list(d.keys())
+    for key in keys:
+        value = d[key]
+        if isinstance(value, dict):
+            _pop_empty(value)
+            if not value:
+                d.pop(key)
+
+
+@define
+class Paths:
+    """
+    Common paths used by the REF application
+    """
+
+    data: Path = field(converter=Path)
+    db: Path = field(converter=Path)
+    log: Path = field(converter=Path)
+    tmp: Path = field(converter=Path)
+
+    @data.default
+    def _data_factory(self) -> Path:
+        return env.path("REF_CONFIGURATION") / "data"
+
+    @db.default
+    def _db_factory(self) -> Path:
+        return env.path("REF_CONFIGURATION") / "db"
+
+    @log.default
+    def _log_factory(self) -> Path:
+        return env.path("REF_CONFIGURATION") / "log"
+
+    @tmp.default
+    def _tmp_factory(self) -> Path:
+        return env.path("REF_CONFIGURATION") / "tmp"
+
+
+@define
+class Db:
+    """
+    Database configuration
+    """
+
+    filename: str = "sqlite://ref.db"
+
+
+@define
+class Config:
+    """
+    REF configuration
+
+    This class is used to store the configuration of the REF application.
+    """
+
+    paths: Paths = Factory(Paths)
+    db: Db = Factory(Db)
+    _raw: TOMLDocument | None = field(init=False, default=None)
+    _config_file: Path | None = field(init=False, default=None)
+
+    @classmethod
+    def load(cls, config_file: Path, allow_missing: bool = True) -> "Config":
+        """
+        Load the configuration from a file
+
+        Parameters
+        ----------
+        config_file
+            Path to the configuration file.
+            This should be a TOML file.
+
+        Returns
+        -------
+        :
+            The configuration loaded from the file
+        """
+        if config_file.is_file():
+            with config_file.open() as fh:
+                doc = tomlkit.load(fh)
+                raw = doc
+        else:
+            if not allow_missing:
+                raise FileNotFoundError(f"Configuration file not found: {config_file}")
+
+            doc = TOMLDocument()
+            raw = None
+        config = _converter_defaults.structure(doc, cls)
+        config._raw = raw
+        config._config_file = config_file
+        return config
+
+    def save(self, config_file: Path | None = None) -> None:
+        """
+        Save the configuration as a TOML file
+
+        The configuration will be saved to the specified file.
+        If no file is specified, the configuration will be saved to the file
+        that was used to load the configuration.
+
+        Parameters
+        ----------
+        config_file
+            The file to save the configuration to
+
+        Raises
+        ------
+        ValueError
+            If no configuration file is specified and the configuration was not loaded from a file
+        """
+        if config_file is None:
+            if self._config_file is None:  # pragma: no cover
+                # I'm not sure if this is possible
+                raise ValueError("No configuration file specified")
+            config_file = self._config_file
+
+        config_file.parent.mkdir(parents=True, exist_ok=True)
+
+        with open(config_file, "w") as fh:
+            fh.write(self.dumps())
+
+    @classmethod
+    def default(cls) -> "Config":
+        """
+        Load the default configuration
+
+        This will load the configuration from the default configuration location,
+        which is typically the user's configuration directory.
+        This location can be overridden by setting the `REF_CONFIGURATION` environment variable.
+
+        Returns
+        -------
+        :
+            The default configuration
+        """
+        root = env.path("REF_CONFIGURATION")
+        return cls.load(root / config_filename)
+
+    def dumps(self, defaults: bool = True) -> str:
+        """
+        Dump the configuration to a TOML string
+
+        Parameters
+        ----------
+        defaults
+            If True, include default values in the output
+
+        Returns
+        -------
+        :
+            The configuration as a TOML string
+        """
+        return self.dump(defaults).as_string()
+
+    def dump(
+        self,
+        defaults: bool = True,
+    ) -> TOMLDocument:
+        """
+        Dump the configuration to a TOML document
+
+        Parameters
+        ----------
+        defaults
+            If True, include default values in the output
+
+        Returns
+        -------
+        :
+            The configuration as a TOML document
+        """
+        if defaults:
+            converter = _converter_defaults
+        else:
+            converter = _converter_no_defaults
+        dump = converter.unstructure(self)
+        if not defaults:
+            _pop_empty(dump)
+        doc = TOMLDocument()
+        doc.update(dump)
+        return doc
+
+
+def _make_converter(omit_default: bool) -> Converter:
+    conv = Converter(omit_if_default=omit_default, forbid_extra_keys=True)
+    conv.register_unstructure_hook(Path, str)
+    conv.register_unstructure_hook(
+        Config,
+        make_dict_unstructure_fn(
+            Config,
+            conv,
+            _raw=override(omit=True),
+            _config_file=override(omit=True),
+        ),
+    )
+    return conv
+
+
+_converter_defaults = _make_converter(omit_default=False)
+_converter_no_defaults = _make_converter(omit_default=True)