Merge remote-tracking branch 'origin/main' into 188-the-frequency-in-…

…the-intake-catalog-does-not-account-for-snapshot-data
ACCESS-NRI · Jan 9, 2025 · 763aa2a · 763aa2a
2 parents 178ac23 + 1e08b06
commit 763aa2a
Show file tree

Hide file tree

Showing 243 changed files with 66,035 additions and 1,437 deletions.
diff --git a/.github/workflows/cd.yml b/.github/workflows/cd.yml
@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.2.0
+        uses: actions/checkout@v4.2.2
 
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.2.0
+        uses: actions/checkout@v4.2.2
 
       - name: Setup conda environment
         uses: conda-incubator/setup-miniconda@v3

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -6,7 +6,7 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.2.0
+      - uses: actions/checkout@v4.2.2
       - uses: actions/setup-python@v5
       - uses: pre-commit/action@v3.0.1
 
@@ -15,11 +15,11 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        python-version: ["3.9", "3.10", "3.11"]
+        python-version: ["3.10", "3.11", "3.12", "3.13"]
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.2.0
+        uses: actions/checkout@v4.2.2
 
       - name: Setup conda environment
         uses: conda-incubator/setup-miniconda@v3
@@ -32,18 +32,22 @@ jobs:
 
       - name: Install source
         shell: bash -l {0}
-        run: python -m pip install -e .
+        run: python -m pip install .
 
       - name: List installed packages
         shell: bash -l {0}
         run: conda list
 
       - name: Run tests
         shell: bash -l {0}
-        run: python -m pytest -s .
+        run: coverage run -m --source=access_nri_intake pytest tests
+
+      - name: Generate coverage report
+        shell: bash -l {0}
+        run: coverage xml
 
       - name: Upload code coverage
-        uses: codecov/codecov-action@v4
+        uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.codecov_token }}
           files: ./coverage.xml
diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml
@@ -0,0 +1,19 @@
+name: Run end-to-end tests on Gadi
+on:
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Run end-to-end tests
+        uses: appleboy/ssh-action@v1.1.0
+        with:
+          host: gadi.nci.org.au
+          username: ${{secrets.GADI_USER}}
+          key: ${{secrets.DEPLOY_KEY}}
+          script: |
+            cd ${{secrets.GADI_REPO_PATH}}
+            qsub bin/test_end_to_end.sh
+
+            
diff --git a/.github/workflows/gadi_deploy.yml b/.github/workflows/gadi_deploy.yml
@@ -10,21 +10,8 @@ jobs:
   build:
     runs-on: ubuntu-latest
     steps:
-      - name: Checkout repository
-        ### Latest at time of writing
-        uses: actions/checkout@v4.2.0
-      - name: Sync repository to Gadi
-        ### Latest at time of writing
-        uses: up9cloud/action-rsync@v1.4
-        env:
-          HOST: gadi.nci.org.au
-          TARGET: ${{secrets.GADI_REPO_PATH}}
-          KEY: ${{secrets.DEPLOY_KEY}}
-          USER: ${{secrets.GADI_USER}}
-          PRE_SCRIPT: |
-            export PROJECT=xp65_w
       - name: Update catalog
-        uses: appleboy/ssh-action@v1.1.0
+        uses: appleboy/ssh-action@v1.2.0
         with:
           host: gadi.nci.org.au
           username: ${{secrets.GADI_USER}}

diff --git a/.github/workflows/project.yml b/.github/workflows/project.yml
@@ -0,0 +1,18 @@
+name: Add Issue to MED project
+
+on:
+  issues:
+    types:
+      - opened
+
+jobs:
+  add-to-project:
+    name: Add issue to project
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/add-to-project@v1.0.2
+        with:
+          # You can target a project in a different organization
+          # to the issue
+          project-url: https://github.com/orgs/ACCESS-NRI/projects/17
+          github-token: ${{ secrets.ADD_TO_PROJECT_PAT }}
diff --git a/.gitignore b/.gitignore
@@ -131,3 +131,9 @@ dmypy.json
 sandpit.ipynb
 *.DS_Store
 bin/build_all.sh.o*
+
+# Vs Code 
+.vscode/
+
+# User-made files
+bin/build_all_test.sh
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,12 +1,19 @@
 repos:
-  - repo: https://github.com/charliermarsh/ruff-pre-commit
-    rev: 'v0.0.238'
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    rev: 'v0.8.1'
     hooks:
       - id: ruff
         args: ['--fix']
 
   - repo: https://github.com/psf/black
-    rev: 22.12.0
+    rev: 24.10.0
     hooks:
     - id: black
       language_version: python3
+# Mypy
+  - repo: https://github.com/pre-commit/mirrors-mypy
+    rev: 'v1.13.0'  
+    hooks:
+    - id: mypy 
+      name: mypy 
+      additional_dependencies: [types-PyYAML==6.0.12.20240808]
diff --git a/.zenodo.json b/.zenodo.json
@@ -10,6 +10,16 @@
             "affiliation": "ACCESS-NRI",
             "name": "Steketee, Anton"
         },
+        {
+            "orcid": "0000-0003-3882-418X",
+            "affiliation": "ACCESS-NRI",
+            "name": "White, Marc"
+        },
+        {
+            "orcid": "0000-0002-3262-4972",
+            "affiliation": "ACCESS-NRI",
+            "name": "Turner, Charles"
+        },       
         {
             "orcid": "0000-0002-2320-4239",
             "affiliation": "ACCESS-NRI",
@@ -35,7 +45,5 @@
 
     "communities": [
         {"identifier": "access-nri"}
-    ],
-
-    "grants": [{"id":""}]
+    ]
 }
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -4,6 +4,39 @@ Changelog
 This CHANGELOG documents only key changes between versions. For a full description 
 of all changes see https://github.com/ACCESS-NRI/access-nri-intake-catalog/releases
 
+v1.0.0
+------
+
+02/12/2024
+
+- Switch to stable release
+- Added MOM6 data support
+- Unit test fixes & upgrades
+- Documentation upgrades, obsolete data references removed
+
+v0.1.5
+------
+
+21/11/2024
+
+- Switch to a date-based catalog versioning schema
+- Add support for BARPA data
+- Allow for coordinate searching
+- Translators rationalisation
+- metadata-validate provides complete report
+- EraiTranslator and catalogs removed
+- Add support for ccam data
+- Support for Python 3.12
+
+v0.1.4
+------
+
+03/10/2024
+
+- Added archive/reference copies of existing catalog source metadata
+- Store experiment and metadata schemas internally
+- Restucture Builders
+
 v0.1.3
 ------
 

diff --git a/README.rst b/README.rst
@@ -2,6 +2,9 @@
 ACCESS-NRI Intake Catalog
 =========================
 
+.. image:: https://zenodo.org/badge/620085541.svg
+  :target: https://doi.org/10.5281/zenodo.8388384
+
 **Tools and configuration info used to manage ACCESS-NRI's intake catalog**
 
 Read the `documentation here <https://access-nri-intake-catalog.readthedocs.io/en/latest/index.html>`_. 

diff --git a/bin/build_all.sh b/bin/build_all.sh
@@ -1,7 +1,7 @@
 #!/bin/bash -l
 
 #PBS -P iq82
-#PBS -l storage=gdata/xp65+gdata/ik11+gdata/cj50+gdata/hh5+gdata/p73+gdata/dk92+gdata/al33+gdata/rr3+gdata/fs38+gdata/oi10
+#PBS -l storage=gdata/xp65+gdata/ik11+gdata/cj50+gdata/p73+gdata/dk92+gdata/al33+gdata/rr3+gdata/fs38+gdata/oi10+gdata/hq89+gdata/py18+gdata/ig45+gdata/zz63+gdata/rt52
 #PBS -q normal
 #PBS -W block=true
 #PBS -l walltime=03:00:00
@@ -26,16 +26,17 @@ if [ ! $# -eq 0 ]; then
 fi
 
 module use /g/data/xp65/public/modules
-module load conda/access-med-0.6
+module load conda/analysis3-24.12
+module load openmpi
 
 OUTPUT_BASE_PATH=/g/data/xp65/public/apps/access-nri-intake-catalog
 CONFIG_DIR=/g/data/xp65/admin/access-nri-intake-catalog/config
-CONFIGS=( cmip5.yaml cmip6.yaml access-om2.yaml access-cm2.yaml access-esm1-5.yaml ) # erai.yaml
+CONFIGS=( cmip5.yaml cmip6.yaml access-om2.yaml access-cm2.yaml access-esm1-5.yaml ccam.yaml barpa.yaml cordex.yaml mom6.yaml narclim2.yaml era5.yaml )
 
 config_paths=( "${CONFIGS[@]/#/${CONFIG_DIR}/}" )
 
 if [ -z "$version" ]; then
-    catalog-build --build_base_path=${OUTPUT_BASE_PATH} ${config_paths[@]}
+    catalog-build --build_base_path=${OUTPUT_BASE_PATH} --catalog_base_path=${OUTPUT_BASE_PATH} ${config_paths[@]}
 else
-    catalog-build --build_base_path=${OUTPUT_BASE_PATH} --version=${version} ${config_paths[@]}
+    catalog-build --build_base_path=${OUTPUT_BASE_PATH} --catalog_base_path=${OUTPUT_BASE_PATH} --version=${version} ${config_paths[@]}
 fi