Merge remote-tracking branch 'origin/main' into 188-the-frequency-in-…

…the-intake-catalog-does-not-account-for-snapshot-data

.github/workflows/cd.yml

@@ -12,7 +12,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.2.0
 
       - name: Set up Python 3.11
         uses: actions/setup-python@v5
@@ -39,7 +39,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.2.0
 
       - name: Setup conda environment
         uses: conda-incubator/setup-miniconda@v3

.github/workflows/ci.yml

@@ -6,7 +6,7 @@ jobs:
   pre-commit:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v4.1.7
+      - uses: actions/checkout@v4.2.0
       - uses: actions/setup-python@v5
       - uses: pre-commit/action@v3.0.1
 
@@ -19,7 +19,7 @@ jobs:
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.2.0
 
       - name: Setup conda environment
         uses: conda-incubator/setup-miniconda@v3

.github/workflows/gadi_deploy.yml

@@ -12,17 +12,19 @@ jobs:
     steps:
       - name: Checkout repository
         ### Latest at time of writing
-        uses: actions/checkout@v4.1.7
+        uses: actions/checkout@v4.2.0
       - name: Sync repository to Gadi
         ### Latest at time of writing
-        uses: up9cloud/action-rsync@v1.3
+        uses: up9cloud/action-rsync@v1.4
         env:
           HOST: gadi.nci.org.au
           TARGET: ${{secrets.GADI_REPO_PATH}}
           KEY: ${{secrets.DEPLOY_KEY}}
           USER: ${{secrets.GADI_USER}}
+          PRE_SCRIPT: |
+            export PROJECT=xp65_w
       - name: Update catalog
-        uses: appleboy/ssh-action@v1.0.0
+        uses: appleboy/ssh-action@v1.1.0
         with:
           host: gadi.nci.org.au
           username: ${{secrets.GADI_USER}}
@@ -32,8 +34,5 @@ jobs:
             export RELEASE=${{ github.event.inputs.release_version }}
             cd bin
             qsub -v version=${RELEASE} build_all.sh
-            cd ..
-            git add src/access_nri_intake/cat
-            git commit "Update catalog to $RELEASE"
 
             

bin/build_all.sh

@@ -1,8 +1,9 @@
 #!/bin/bash -l
 
-#PBS -P tm70
-#PBS -l storage=gdata/tm70+gdata/xp65+gdata/ik11+gdata/cj50+gdata/hh5+gdata/p73+gdata/dk92
+#PBS -P iq82
+#PBS -l storage=gdata/xp65+gdata/ik11+gdata/cj50+gdata/hh5+gdata/p73+gdata/dk92+gdata/al33+gdata/rr3+gdata/fs38+gdata/oi10
 #PBS -q normal
+#PBS -W block=true
 #PBS -l walltime=03:00:00
 #PBS -l mem=192gb
 #PBS -l ncpus=48
@@ -24,10 +25,11 @@ if [ ! $# -eq 0 ]; then
     version=$1
 fi
 
-conda activate access-nri-intake-test
+module use /g/data/xp65/public/modules
+module load conda/access-med-0.6
 
 OUTPUT_BASE_PATH=/g/data/xp65/public/apps/access-nri-intake-catalog
-CONFIG_DIR=/g/data/tm70/ds0092/projects/access-nri-intake-catalog/config
+CONFIG_DIR=/g/data/xp65/admin/access-nri-intake-catalog/config
 CONFIGS=( cmip5.yaml cmip6.yaml access-om2.yaml access-cm2.yaml access-esm1-5.yaml ) # erai.yaml
 
 config_paths=( "${CONFIGS[@]/#/${CONFIG_DIR}/}" )

config/barpa.yaml

@@ -0,0 +1,9 @@
+builder: null
+
+translator: BarpaTranslator
+
+sources:
+
+  - metadata_yaml: /g/data/xp65/admin/access-nri-intake-catalog/config/metadata_sources/barpa-py18/metadata.yaml
+    path:
+      - /g/data/py18/catalog/v2/esm/catalog.json

config/cmip5.yaml

@@ -4,10 +4,10 @@ translator: Cmip5Translator
 
 sources:
 
-  - metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip5_al33/metadata.yaml
+  - metadata_yaml: /g/data/xp65/admin/access-nri-intake-catalog/config/metadata_sources/cmip5-al33/metadata.yaml
     path:
       - /g/data/al33/catalog/v2/esm/catalog.json
 
-  - metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip5_rr3/metadata.yaml
+  - metadata_yaml: /g/data/xp65/admin/access-nri-intake-catalog/config/metadata_sources/cmip5-rr3/metadata.yaml
     path:
       - /g/data/rr3/catalog/v2/esm/catalog.json

config/cmip6.yaml

@@ -4,10 +4,10 @@ translator: Cmip6Translator
 
 sources:
 
-  - metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip6_fs38/metadata.yaml
+  - metadata_yaml: /g/data/xp65/admin/access-nri-intake-catalog/config/metadata_sources/cmip6-fs38/metadata.yaml
     path:
       - /g/data/fs38/catalog/v2/esm/catalog.json
 
-  - metadata_yaml: /g/data/xp65/admin/intake/metadata/cmip6_oi10/metadata.yaml
+  - metadata_yaml: /g/data/xp65/admin/access-nri-intake-catalog/config/metadata_sources/cmip6-oi10/metadata.yaml
     path:
       - /g/data/oi10/catalog/v2/esm/catalog.json