Merge pull request #54 from CMIP-REF/fix-alembic

.github/workflows/ci.yaml

@@ -78,8 +78,7 @@ jobs:
           uv pip install dist/*.whl
           uv pip freeze
           uv run --no-sync python -c "import cmip_ref; print(cmip_ref.__version__)"
-  # TODO: Alembic needs to be packaged correctly
-  #          uv run --no-sync ref config list
+          uv run --no-sync ref config list
 
   # Check if a changelog message was added to the PR
   # Only runs on pull requests

changelog/54.fix.md

@@ -0,0 +1 @@
+Move alembic configuration and migrations to `cmip_ref` package so that they can be included in the distribution.

docs/development.md

@@ -144,6 +144,32 @@ This mono-repo approach might change once the packages become more mature,
 but since we are in the early stages of development,
 there will be a lot of refactoring of the interfaces to find the best approach.
 
+## Database management
+
+The REF uses a local Sqlite database to store state information.
+We use [alembic](https://alembic.sqlalchemy.org/en/latest/) to manage our database migrations
+as the schema of this database changes.
+
+When making changes to the database models (`cmip_ref.models`),
+a migration must also be added (see below).
+
+The migration definitions (and the alembic configuration file)
+are included in the `cmip_ref` package (`packages/ref/src/cmip_ref/migrations`)
+to enable users to apply these migrations transparently.
+Any new migrations are performed automatically when using the `ref` command line tool.
+
+### Adding a database migration
+
+If you have made changes to the database models,
+you will need to create a new migration to apply these changes.
+Alembic can autogenerate these migrations for you,
+but they will need to be reviewed to ensure they are correct.
+
+```
+uv run alembic -c packages/ref/src/cmip_ref/alembic.ini \
+   revision --autogenerate --message "your_migration_message"
+```
+
 [](){releasing-reference}
 ## Releasing
 

packages/ref/conftest.py

@@ -7,8 +7,7 @@
 from cmip_ref.datasets.cmip6 import CMIP6DatasetAdapter
 
 # Ignore the alembic folder
-collect_ignore = ["alembic"]
-
+collect_ignore = ["src/cmip_ref/migrations"]
 
 # Add a representer for pandas Timestamps/NaT in the regression tests
 RegressionYamlDumper.add_representer(

alembic.ini → packages/ref/src/cmip_ref/alembic.ini

@@ -3,13 +3,13 @@
 [alembic]
 # path to migration scripts
 # Use forward slashes (/) also on windows to provide an os agnostic path
-script_location = %(here)s/packages/ref/alembic
+script_location = %(here)s/migrations
 
 # template used to generate migration file names; The default value is %%(rev)s_%%(slug)s
 # Uncomment the line below if you want the files to be prepended with date and time
 # see https://alembic.sqlalchemy.org/en/latest/tutorial.html#editing-the-ini-file
 # for all available tokens
-# file_template = %%(year)d_%%(month).2d_%%(day).2d_%%(hour).2d%%(minute).2d-%%(rev)s_%%(slug)s
+file_template = %%(year)d-%%(month).2d-%%(day).2dT%%(hour).2d%%(minute).2d_%%(rev)s_%%(slug)s
 
 # sys.path path, will be prepended to sys.path if present.
 # defaults to the current working directory.
@@ -36,10 +36,10 @@ prepend_sys_path = .
 # sourceless = false
 
 # version location specification; This defaults
-# to packages/ref/alembic/versions.  When using multiple version
+# to alembic/versions.  When using multiple version
 # directories, initial revisions must be specified with --version-path.
 # The path separator used here should be the separator specified by "version_path_separator" below.
-# version_locations = %(here)s/bar:%(here)s/bat:packages/ref/alembic/versions
+# version_locations = %(here)s/bar:%(here)s/bat:alembic/versions
 
 # version path separator; As mentioned above, this is the character used to split
 # version_locations. The default within new alembic.ini files is "os", which uses os.pathsep.
@@ -71,11 +71,11 @@ version_path_separator = os  # Use os.pathsep. Default configuration used for ne
 hooks = ruff-fix, ruff-format
 
 ruff-fix.type = exec
-ruff-fix.executable = %(here)s/.venv/bin/ruff
+ruff-fix.executable = ruff
 ruff-fix.options = check -q --fix REVISION_SCRIPT_FILENAME
 
 ruff-format.type = exec
-ruff-format.executable = %(here)s/.venv/bin/ruff
+ruff-format.executable = ruff
 ruff-format.options = format -q REVISION_SCRIPT_FILENAME
 
 # Logging configuration

packages/ref/src/cmip_ref/database.py

@@ -8,6 +8,7 @@
 It provides a session object that can be used to interact with the database and run queries.
 """
 
+import importlib.resources
 from pathlib import Path
 from typing import Any
 from urllib import parse as urlparse
@@ -82,10 +83,9 @@ def __init__(self, url: str, run_migrations: bool = True) -> None:
             self._migrate()
 
     def _migrate(self) -> None:
-        root_dir = Path(__file__).parents[4]
-
-        alembic_config = AlembicConfig(root_dir / "alembic.ini")
-        alembic_config.attributes["connection"] = self._engine
+        with importlib.resources.path("cmip_ref", "alembic.ini") as fspath:
+            alembic_config = AlembicConfig(fspath)
+            alembic_config.attributes["connection"] = self._engine
 
         script = ScriptDirectory.from_config(alembic_config)
         head = script.get_current_head()

pyproject.toml

@@ -78,6 +78,10 @@ exclude_also = [
     # Exclude ... literals
     "\\.\\.\\."
 ]
+omit = [
+    # Skip alembic migration files
+    "*/migrations/*",
+]
 
 [tool.mypy]
 strict = true
@@ -91,7 +95,7 @@ warn_unreachable = true
 # importing following uses default settings
 follow_imports = "normal"
 exclude = [
-    "alembic",
+    "migrations",
     "build",
     "dist",
     "notebooks",

ruff.toml

@@ -42,7 +42,7 @@ ignore = [
     "S101",  # Use of `assert` detected
     "PLR2004",  # Magic value used in comparison.
 ]
-"*/alembic/versions/*" = [
+"*/migrations/versions/*" = [
     "D103",  # Missing docstring in public function
 ]
 "scripts/*" = [