Merge branch 'master' into longterm-monitoring

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 2.4.0
+current_version = 2.5.3
 commit = True
 tag = False
 tag_name = {new_version}
@@ -30,11 +30,11 @@ search = {current_version}
 replace = {new_version}
 
 [bumpversion:file:RELEASE.txt]
-search = {current_version} 2024-06-04T17:41:34Z
+search = {current_version} 2024-09-11T22:57:09Z
 replace = {new_version} {utcnow:%Y-%m-%dT%H:%M:%SZ}
 
 [bumpversion:part:releaseTime]
-values = 2024-06-04T17:41:34Z
+values = 2024-09-11T22:57:09Z
 
 [bumpversion:file(version):birdhouse/components/canarie-api/docker_configuration.py.template]
 search = 'version': '{current_version}'

.github/PULL_REQUEST_TEMPLATE.md

@@ -31,7 +31,11 @@ Links to other issues or sources.
 <!--
   The test suite can be run using a different DACCS config with ``birdhouse_daccs_configs_branch: branch_name`` in the PR description.
   To globally skip the test suite regardless of the commit message use ``birdhouse_skip_ci`` set to ``true`` in the PR description.
-  Note that using ``[skip ci]``, ``[ci skip]`` or ``[no ci]`` in the commit message will override ``birdhouse_skip_ci`` from the PR description.
+
+  Using ``[<cmd>]`` (with the brackets) where ``<cmd> = skip ci`` in the commit message will override ``birdhouse_skip_ci`` from the PR description.
+  Such commit command can be used to override the PR description behavior for a specific commit update.
+  However, a commit message cannot 'force run' a PR which the description turns off the CI.
+  To run the CI, the PR should instead be updated with a ``true`` value, and a running message can be posted in following PR comments to trigger tests once again.
 -->
 
 birdhouse_daccs_configs_branch: master

CHANGES.md

@@ -52,6 +52,117 @@
   This also includes an update to the prometheus version from `v2.19.0` to the current latest `v2.52.0`. This is to
   required to support the interaction between prometheus and thanos.
 
+[2.5.3](https://github.com/bird-house/birdhouse-deploy/tree/2.5.3) (2024-09-11)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Magpie/Twitcher: update Python packages and base Docker image to address security vulnerabilities 
+
+  - [Magpie 4.1.1](https://github.com/Ouranosinc/Magpie/blob/master/CHANGES.rst#411-2024-07-23)
+    (relates to [Ouranosinc/Magpie#622](https://github.com/Ouranosinc/Magpie/pull/622)).
+  - [Twitcher 0.10.0](https://github.com/bird-house/twitcher/blob/master/CHANGES.rst#0100-2024-07-22)
+    (relates to [bird-house/twitcher#136](https://github.com/bird-house/twitcher/pull/136)).
+
+- xclim-testdata: adapt repository cloning script to the new data structure
+
+  The `xclim-testdata` repo has been restructured to include the data in a `data` subdirectory.
+  This change updates the cloning script to account for this new structure and to ensure that the
+  user experience is consistent with the previous version.
+
+  See:
+  * [xclim-testdata PR/29](https://github.com/Ouranosinc/xclim-testdata/pull/29)
+  * [xclim PR/1889](https://github.com/Ouranosinc/xclim/pull/1889)
+
+[2.5.2](https://github.com/bird-house/birdhouse-deploy/tree/2.5.2) (2024-07-19)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- GeoServer: upgrade to 2.25.2 to fix vulnerabilities
+
+  See:
+  * https://nsfocusglobal.com/remote-code-execution-vulnerability-between-geoserver-and-geotools-cve-2024-36401-cve-2024-36404-notification/
+  * https://github.com/geoserver/geoserver/security/advisories/GHSA-6jj6-gm7p-fcvv
+  * https://github.com/geotools/geotools/security/advisories/GHSA-w3pj-wh35-fq8w
+
+  This change will upgrade to GeoServer 2.25.2 and GeoTools 31.2 (the version of `gt-complex.jar`).
+
+  ```shell
+  $ docker exec -u 0 geoserver find / -iname '**gt-complex**'
+  /usr/local/tomcat/webapps/geoserver/WEB-INF/lib/gt-complex-31.2.jar
+  ```
+
+  The previous version was GeoServer 2.22.2 and GeoTools 28.2.
+
+  ```shell
+  $ docker exec -u 0 geoserver find / -iname '**gt-complex**'
+  /usr/local/tomcat/webapps/geoserver/WEB-INF/lib/gt-complex-28.2.jar
+  ```
+
+  Also enable
+  * OGC-API plugins https://docs.geoserver.org/stable/en/user/community/ogc-api/features/index.html
+    so we can slowly transition from the WPS plugin.
+  * STAC Datastore plugin https://docs.geoserver.org/latest/en/user/community/stac-datastore/index.html
+    so we can test integration with our STAC component.
+
+
+[2.5.1](https://github.com/bird-house/birdhouse-deploy/tree/2.5.1) (2024-07-10)
+------------------------------------------------------------------------------------------------------------------
+
+- Cowbird: bump version to [2.4.0](https://github.com/Ouranosinc/cowbird/blob/master/CHANGES.rst#240-2024-07-09).
+
+  - Includes multiple dependency updates for latest security and performance improvements.
+
+[2.5.0](https://github.com/bird-house/birdhouse-deploy/tree/2.5.0) (2024-06-20)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+
+- Weaver: bump version to [5.6.1](https://github.com/crim-ca/weaver/tree/5.6.1).
+
+  - See full changes details in 
+    [Weaver changes](https://pavics-weaver.readthedocs.io/en/latest/changes.html#changes-5-6-1)
+  - In summary:
+    - multiple control setting options to customize some behaviors
+    - improved *OGC API - Processes* standard conformance
+    - improved support of *Common Workflow Language (CWL)* features (secrets, sub-workflow, auth-propagation, etc.)
+
+- Weaver: WPS retry logic on post-compose step.
+  - Apply `--network birdhouse_default` to the Docker `curl` image to allow HTTP requests to properly resolve
+    against the running services (WPS bird providers, Weave and Magpie). In some cases, this network would not
+    be automatically resolved.
+  - Fix the index used during HTTP request retry to avoid going one step over the intended retry attempts.
+
+[2.4.2](https://github.com/bird-house/birdhouse-deploy/tree/2.4.2) (2024-06-12)
+------------------------------------------------------------------------------------------------------------------
+
+## Changes
+- deploy-data: allow more flexibility to deploy files from other checkout in same config file
+
+  Given the config file can specify multiple checkouts, this flexibility to
+  have `SRC_DIR` be an absolute path will allow one checkout to take files from
+  other checkouts, using absolute path to the other checkouts.  `SRC_DIR` can
+  still be a relative path of the current checkout, as before, to preserve
+  backward-compatibility.
+
+  Possible use-case: re-organize the layout of various files from the various
+  checkouts in an intermediate location before rsyncing this intermediate
+  location to the final destination.
+
+
+[2.4.1](https://github.com/bird-house/birdhouse-deploy/tree/2.4.1) (2024-06-05)
+------------------------------------------------------------------------------------------------------------------
+
+## Fixes
+- Weaver: Adjust invalid `data_sources.yml` definitions.
+
+  - Add the missing `data_sources.yml` volume mount for  `weaver-worker`.
+  - When `weaver-worker` runs a `Workflow`, the nested `step` process locations need to be resolved according to the
+    current `"localhost"` instance. However, the Web API running in `weaver` service is not visible from the worker.
+    Since the configuration is shared between `weaver` and `weaver-worker`, use the public endpoint of `weaver` to
+    make process URL resolution consistent, and also provide more useful references in job logs when resolution fails.
+
 [2.4.0](https://github.com/bird-house/birdhouse-deploy/tree/2.4.0) (2024-06-04)
 ------------------------------------------------------------------------------------------------------------------
 

CONTRIBUTING.rst

@@ -37,9 +37,9 @@ Policy rules
 
 3. Contributions will trigger a test suite that must successfully pass before being merged (or integrated).
 
-    * The test suite can be run using a different DACCS config with ``birdhouse_daccs_configs_branch: branch_name`` in the PR description.
-    * It is possible to skip the test suite if the latest commit contains either ``[skip ci]``, ``[ci skip]`` or ``[no ci]``.
-      To globally skip the test suite regardless of the commit message use ``birdhouse_skip_ci: true`` in the PR description.
+   * The test suite can be run using a different DACCS config with ``birdhouse_daccs_configs_branch: {branch_name}`` in the PR description.
+   * To globally skip the test suite regardless of the commit message use ``birdhouse_skip_ci: true`` in the PR description.
+   * It is possible to skip the test suite if the latest commit message contains ``[skip ci]``.
 
 4. Contributions must be reviewed by every willing organizations
    (Default reviewers are `@mishaschwartz`_ for `UofT` , `@tlvu`_ for `Ouranos`_ and `@fmigneault`_ for `CRIM`_).

Makefile

@@ -1,7 +1,7 @@
 # Generic variables
 override SHELL       := bash
 override APP_NAME    := birdhouse-deploy
-override APP_VERSION := 2.4.0
+override APP_VERSION := 2.5.3
 
 # utility to remove comments after value of an option variable
 override clean_opt = $(shell echo "$(1)" | $(_SED) -r -e "s/[ '$'\t'']+$$//g")

README.rst

@@ -18,13 +18,13 @@ for a full-fledged production platform.
     * - citation
       - | |citation|
 
-.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.4.0.svg
+.. |commits-since| image:: https://img.shields.io/github/commits-since/bird-house/birdhouse-deploy/2.5.3.svg
     :alt: Commits since latest release
-    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.4.0...master
+    :target: https://github.com/bird-house/birdhouse-deploy/compare/2.5.3...master
 
-.. |latest-version| image:: https://img.shields.io/badge/tag-2.4.0-blue.svg?style=flat
+.. |latest-version| image:: https://img.shields.io/badge/tag-2.5.3-blue.svg?style=flat
     :alt: Latest Tag
-    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.4.0
+    :target: https://github.com/bird-house/birdhouse-deploy/tree/2.5.3
 
 .. |readthedocs| image:: https://readthedocs.org/projects/birdhouse-deploy/badge/?version=latest
     :alt: ReadTheDocs Build Status (latest version)
@@ -42,7 +42,7 @@ for a full-fledged production platform.
 
 
 Birdhouse
-------
+---------
 
 For GitHub navigation, see the following README pages:
 

RELEASE.txt

@@ -1 +1 @@
-2.4.0 2024-06-04T17:41:34Z
+2.5.3 2024-09-11T22:57:09Z

birdhouse/components/canarie-api/docker_configuration.py.template

@@ -108,8 +108,8 @@ SERVICES = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.4.0',
-            'releaseTime': '2024-06-04T17:41:34Z',
+            'version': '2.5.3',
+            'releaseTime': '2024-09-11T22:57:09Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',
@@ -141,8 +141,8 @@ PLATFORMS = {
             # NOTE:
             #   Below version and release time auto-managed by 'make VERSION=x.y.z bump'.
             #   Do NOT modify it manually. See 'Tagging policy' in 'birdhouse/README.rst'.
-            'version': '2.4.0',
-            'releaseTime': '2024-06-04T17:41:34Z',
+            'version': '2.5.3',
+            'releaseTime': '2024-09-11T22:57:09Z',
             'institution': '${BIRDHOUSE_INSTITUTION}',
             'researchSubject': '${BIRDHOUSE_SUBJECT}',
             'supportEmail': '${BIRDHOUSE_SUPPORT_EMAIL}',

birdhouse/components/cowbird/default.env

@@ -29,7 +29,7 @@ VARS="$VARS $EXTRA_VARS"
 # Cowbird Configuration
 # =====================
 
-export COWBIRD_VERSION="2.3.0"
+export COWBIRD_VERSION="2.4.0"
 
 # default release of the MongoDB version employed by Cowbird
 # note that MongoDB>=5.0 is REQUIRED for Cowbird>=2.0.0

birdhouse/components/geoserver/default.env

@@ -8,21 +8,34 @@
 # "moving" tags, meaning not reproducible behavior !
 # See https://github.com/kartoza/docker-geoserver/issues/232#issuecomment-808754831
 # The version is used for representation in CanarieAPI, while the full tag is used to reference the image.
-export GEOSERVER_DOCKER=pavics/geoserver
-export GEOSERVER_VERSION=2.22.2
-export GEOSERVER_TAGGED=2.22.2-kartoza-build20230226-r7-allow-change-context-root-and-fix-missing-stable-plugins-and-avoid-chown-datadir
+export GEOSERVER_DOCKER="pavics/geoserver"
+export GEOSERVER_VERSION="2.25.2"
+export GEOSERVER_TAGGED="2.25.2--v2024.06.25-kartoza"
 export GEOSERVER_IMAGE='${GEOSERVER_DOCKER}:${GEOSERVER_TAGGED}'
 export GEOSERVER_IMAGE_URI='registry.hub.docker.com/${GEOSERVER_IMAGE}'
 
 export GEOSERVER_ADMIN_USER="admin"
 
 # # Install the stable plugin specified in
 # https://github.com/kartoza/docker-geoserver/blob/master/build_data/stable_plugins.txt
-export GEOSERVER_STABLE_EXTENSIONS="grib-plugin,netcdf-plugin,netcdf-out-plugin,csw-iso-plugin,metadata-plugin"
+export GEOSERVER_STABLE_EXTENSIONS="grib-plugin,\
+netcdf-plugin,\
+netcdf-out-plugin,\
+csw-iso-plugin,\
+metadata-plugin"
 
 # Install the community edition plugins specified in
 # https://github.com/kartoza/docker-geoserver/blob/master/build_data/community_plugins.txt
-export GEOSERVER_COMMUNITY_EXTENSIONS="geopkg-plugin"
+export GEOSERVER_COMMUNITY_EXTENSIONS="geopkg-plugin,\
+ogcapi-coverages-plugin,\
+ogcapi-dggs-plugin,\
+ogcapi-features-plugin,\
+ogcapi-images-plugin,\
+ogcapi-maps-plugin,\
+ogcapi-styles-plugin,\
+ogcapi-tiled-features-plugin,\
+ogcapi-tiles-plugin,\
+stac-datastore-plugin"
 
 # Must use single-quote for delayed eval.
 export GEOSERVER_DATA_DIR='${BIRDHOUSE_DATA_PERSIST_ROOT}/geoserver'

birdhouse/components/magpie/default.env

@@ -5,7 +5,7 @@
 # are applied and must be added to the list of DELAYED_EVAL.
 
 # Tag version that will be used to update Magpie API, Magpie CLI, and matching Twitcher with Magpie Adapter
-export MAGPIE_VERSION=3.38.0
+export MAGPIE_VERSION=4.1.1
 
 export MAGPIE_DB_NAME="magpiedb"
 

birdhouse/components/weaver/config/weaver/data_sources.yml.template

@@ -2,7 +2,7 @@
 # Employed by default for looking at files available directly (e.g.: pre-fetched)
 localhost:
     netloc: "localhost"
-    ades: "http://localhost:4001"
+    ades: "https://${BIRDHOUSE_FQDN_PUBLIC}/${WEAVER_MANAGER_NAME}"
     default: true
 
 # Weaver self-reference, but using the exposed endpoint on the birdhouse instance

birdhouse/components/weaver/default.env

@@ -53,7 +53,7 @@ OPTIONAL_VARS="
 export WEAVER_CONFIG=HYBRID
 
 # default release version that will be used to fetch docker images (API mananger & celery workers services)
-export WEAVER_VERSION=4.32.0
+export WEAVER_VERSION=5.6.1
 export WEAVER_DOCKER=pavics/weaver
 export WEAVER_IMAGE='${WEAVER_DOCKER}:${WEAVER_VERSION}'
 export WEAVER_MANAGER_IMAGE='${WEAVER_IMAGE}-manager'

birdhouse/components/weaver/docker-compose-extra.yml

@@ -64,6 +64,7 @@ services:
     volumes:
       #- wps_private:/data/wps_private
       - ./components/weaver/config/weaver/weaver.ini:/opt/local/src/weaver/config/weaver.ini:ro
+      - ./components/weaver/config/weaver/data_sources.yml:/opt/local/src/weaver/config/data_sources.yml:ro
       - ./components/weaver/config/weaver/request_options.yml:/opt/local/src/weaver/config/request_options.yml:ro
       - ./components/weaver/celery-healthcheck:/opt/local/bin/weaver/celery-healthcheck:ro
       # WARNING:

birdhouse/components/weaver/post-docker-compose-up

@@ -120,7 +120,7 @@ RANDOM_NUMBER=${RANDOM:-$(tr -dc 0-9 < /dev/urandom 2>/dev/null | head -c 5)}
 # To know when a docker run was started in case it hangs.
 DOCKER_RUN_TAG="weaver_post_curl_$(date -Isecond | sed 's/:/_/g' | sed 's/+/p/g')_${RANDOM_NUMBER}"
 curl_cmd() {
-    docker run --rm --name "${DOCKER_RUN_TAG}" "${WEAVER_CURL_IMAGE}" "$@"
+    docker run --network birdhouse_default --rm --name "${DOCKER_RUN_TAG}" "${WEAVER_CURL_IMAGE}" "$@"
 }
 
 # pull image if missing to avoid mangling output messages on first call
@@ -317,7 +317,7 @@ for prov in ${WEAVER_WPS_PROVIDERS}; do
         if [ ${ret} -ne 0 ] || [ "${code}" -ne 201 ]; then
             printf "\n%s\n" "${WARN}Failed registration of remote WPS provider [${prov}] on [${prov_url}]${retry_msg}."
             printf "Error:\n%s\n" "${body}"
-            if [ ${retry} -gt ${total} ]; then
+            if [ ${retry} -ge ${total} ]; then
               echo "${ERROR}Maximum retry attempts ${total} reached for WPS provider [${prov}]. Aborting."
               reset_state
               exit 23

birdhouse/deployment/deploy-data

@@ -151,6 +151,14 @@ for GIT_REPO_URL in $GIT_REPO_URLS; do
         echo "sync '$SRC_DIR' to '$DEST_DIR'"
         DEST_DIR_PARENT="`dirname "$DEST_DIR"`"
         SRC_DIR_ABS_PATH="`pwd`/$SRC_DIR"
+        if [ ! -e "$SRC_DIR_ABS_PATH" ]; then
+            # SRC_DIR could already be absolute.
+            # Support rsync from source outside of current checkout, ex: from
+            # other checkouts in same config file because config file can have
+            # multiple checkouts or from intermediate location used to
+            # re-organize the layout before the final rsync to destination.
+            SRC_DIR_ABS_PATH="$SRC_DIR"
+        fi
         USER_ID="`id -u`"
         GROUP_ID="`id -g`"
         RSYNC_USER_GRP="$USER_ID:$GROUP_ID"

birdhouse/deployment/deploy-data-xclim-testdata-to-thredds.yml

@@ -5,7 +5,7 @@ deploy:
   checkout_name: xclim-testdata
   dir_maps:
   # rsync content below source_dir into dest_dir
-  - source_dir: .
+  - source_dir: ./data
     dest_dir: /data/datasets/testdata/xclim
     # only sync .nc files
     rsync_extra_opts: --include=*/ --include=*.nc --exclude=*

docs/source/conf.py

@@ -69,9 +69,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '2.4.0'
+version = '2.5.3'
 # The full version, including alpha/beta/rc tags.
-release = '2.4.0'
+release = '2.5.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -357,4 +357,20 @@
 
 
 # Example configuration for intersphinx: refer to the Python standard library.
-intersphinx_mapping = {'https://docs.python.org/': None}
+intersphinx_mapping = {
+    'python': ('http://docs.python.org/', None),
+    'phoenix': ('https://pyramid-phoenix.readthedocs.io/en/latest/', None),
+    'malleefowl': ('https://malleefowl.readthedocs.io/en/latest/', None),
+    'weaver': ('https://pavics-weaver.readthedocs.io/en/latest/', None),
+    'magpie': ('https://pavics-magpie.readthedocs.io/en/latest/', None),
+    'twitcher': ('https://twitcher.readthedocs.io/en/latest/', None),
+    'flyingpigeon': ('https://flyingpigeon.readthedocs.io/en/latest/', None),
+    'hummingbird': ('https://birdhouse-hummingbird.readthedocs.io/en/latest/', None),
+    'finch': ('https://finch.readthedocs.io/en/latest/', None),
+    'raven': ('https://pavics-raven.readthedocs.io/en/latest/', None),
+    'emu': ('https://emu.readthedocs.io/en/latest/', None),
+    'pelican': ('https://github.com/bird-house/pelican', None),
+    'birdy': ('https://birdy.readthedocs.io/en/latest/', None),
+    'bootstrap': ('https://birdhousebuilderbootstrap.readthedocs.io/en/latest/', None),
+    'birdhouse-deploy': ('https://birdhouse-deploy.readthedocs.io/en/latest/', None),
+}