Release TerriaMap using create-docker-context

.github/workflows/release.yml

@@ -8,6 +8,7 @@ on:
 env:
   REGISTRY: ghcr.io
   IMAGE_NAME: ${{ github.repository }}
+  NODE_VERSION: 18
 
 permissions:
   packages: write
@@ -18,32 +19,45 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: ${{ env.NODE_VERSION }}
+
+      - name: Install dependencies
+        run: yarn install --frozen-lockfile
+        env:
+          NODE_OPTIONS: "--max_old_space_size=4096"
+
+      - name: Build TerriaMap
+        run: yarn gulp lint release
+        env:
+          NODE_OPTIONS: "--max_old_space_size=4096"
 
       - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v2
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
+        uses: docker/setup-buildx-action@v2
 
       - name: Login to GHCR
-        uses: docker/login-action@v1
+        uses: docker/login-action@v3
         with:
           registry: ghcr.io
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
 
       - name: Tag image
         id: meta
-        uses: docker/metadata-action@v3
+        uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
           tags: |
             type=semver,pattern={{version}}
+        env:
+          DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index
 
       - name: Build and push
-        uses: docker/build-push-action@v2
-        with:
-          platforms: linux/amd64,linux/arm64
-          tags: ${{ steps.meta.outputs.tags }}
-          push: true
+        run: yarn docker-build-prod --metadata

.prettierignore

@@ -43,4 +43,4 @@ wwwroot/*.html
 
 
 ckanext-cesiumpreview
-deploy/helm/terria/charts/terriamap/templates/
+deploy/helm/terriamap/templates/

architecture/0001-npm-lockfiles.md

@@ -4,7 +4,7 @@ Date: 2020-03-02
 
 ## Status
 
-Accepted
+Superseded 2021-10-09 by decision to use yarn everywhere.
 
 ## Context
 

architecture/0002-docker-multi-arch-build.md

@@ -0,0 +1,56 @@
+# 2. Docker multi-architecture build using create-docker-context
+
+Date: 2024-07-18
+
+## Status
+
+Proposed
+
+## Context
+
+I (crispy) consider a multi-stage dockerfile to be the gold standard of
+reproducible, mutli-architecture builds. Something like the following:
+
+- Build container copies workspace, installs development dependencies and builds
+  the app.
+- Production container copies build artifacts and installs only production
+  dependencies.
+
+is ideal. This ensures only production dependencies are present, and you can run
+this process on every architecture to create a multi-arch docker image. Binaries
+downloaded during dependency installation will fetch the correct architecture
+binary as depencies are installed separately on each architecture. But
+installing dependencies and building JS is extremely slow on emulated
+architectures, such as docker buildx on GitHub Actions (this can take 2.5 hours
+to build the image).
+
+If instead we can (on the build machine/VM):
+
+1. install all dependencies
+2. build the app
+3. copy build artifacts and only production dependencies to the multi-arch
+   docker image
+
+then **as long as production dependencies are portable**, we have a working
+multi-arch docker image with very little computation being run on emulated
+architectures. The `create-docker-context.js` script allows us to do this,
+copying build artifacts and only production dependencies to an intermediate
+"context" folder which is then used to create the image.
+
+Currently none of our production dependencies install non-portable binaries.
+
+## Decision
+
+While all production dependencies remain portable, we will build
+multi-architecture docker images by building TerriaMap on the VM and copying
+only production-necessary files and dependencies to the final docker image.
+
+## Consequences
+
+- We will replace current GitHub Actions release process with one using
+  `create-docker-context.js`.
+- Our GitHub Actions TerriaMap release time will reduce from 2.5 hours to less
+  than 10 minutes.
+- If in future TerriaMap uses a binary installed by side effect during JS
+  dependency installation and this binary cannot be run on an architecture for
+  which an image is created, that image will fail when run on that architecture.

deploy/docker/Dockerfile

@@ -1,9 +1,12 @@
 # Docker image for the primary terria map application server
-FROM node:16
+# Intended for use only with a "context" directory created by create-docker-context.js
+FROM node:16-slim
 
-RUN mkdir -p /usr/src/app && mkdir -p /etc/config/client
-WORKDIR /usr/src/app/component
-COPY . /usr/src/app
+RUN mkdir -p /etc/config/client
+
+USER node
+WORKDIR /usr/src/app
+COPY --chown=node:node component /usr/src/app
 
 EXPOSE 3001
 ENV NODE_ENV=production

deploy/docker/create-docker-context.js

@@ -1,8 +1,16 @@
 #!/usr/bin/env node
 
+// MAJOR ASSSUMPTION: build artifacts and node_modules content for all production
+// dependencies is cross-platform, or care is taken to only install dependencies,
+// run create-docker-context.js and create docker images on a compatible platform
+// See architecture/0002-docker-multi-arch-build.md
+
 // Based off @magda/docker-utils@2.1.0 create-docker-context-for-node-component
 // Changes made:
-// - The Dockerfile path is configurable in package.json
+// - The Dockerfile path is configurable in package.json (I don't want a dockerfile
+//    intended to be used only through a script to be in the top level directory)
+// - Can parse metadata from GitHub Action docker/metadata-action@v5 and add this
+//    to the created image
 
 const childProcess = require("child_process");
 const fse = require("fs-extra");
@@ -42,7 +50,7 @@ const argv = yargs
     },
     name: {
       description:
-        "The package name to use in auto tag generation. Will default to ''. Used to override the docker nanme config in package.json during the auto tagging. Requires --tag=auto",
+        "The package name to use in auto tag generation. Will default to ''. Used to override the docker name config in package.json during the auto tagging. Requires --tag=auto",
       type: "string",
       default: process.env.MAGDA_DOCKER_NAME
     },
@@ -86,6 +94,12 @@ const argv = yargs
       description:
         "Version to cache from when building, using the --cache-from field in docker. Will use the same repository and name. Using this options causes the image to be pulled before build.",
       type: "string"
+    },
+    metadata: {
+      description:
+        "Use tags and annotations from https://github.com/docker/metadata-action v5. Utilises env.DOCKER_METADATA_OUTPUT_JSON. Overrides --tag",
+      type: "boolean",
+      default: false
     }
   })
   .help().argv;
@@ -166,16 +180,22 @@ if (argv.build) {
     }
   );
 
-  const tags = getTags(
-    argv.tag,
-    argv.local,
-    argv.repository,
-    argv.version,
-    argv.name
-  );
-  const tagArgs = tags
-    .map((tag) => ["-t", tag])
-    .reduce((soFar, tagArgs) => soFar.concat(tagArgs), []);
+  // metadata json from GitHub Action docker/metadata-action@v5
+  const metadata =
+    argv.metadata && env.DOCKER_METADATA_OUTPUT_JSON
+      ? JSON.parse(env.DOCKER_METADATA_OUTPUT_JSON)
+      : undefined;
+
+  const tags = metadata
+    ? metadata.tags
+    : getTags(argv.tag, argv.local, argv.repository, argv.version, argv.name);
+  const tagArgs = tags.flatMap((tag) => ["-t", tag]);
+
+  const annotationArgs =
+    metadata?.annotations?.flatMap((annotation) => [
+      "--annotation",
+      annotation
+    ]) ?? [];
 
   const cacheFromArgs = cacheFromImage ? ["--cache-from", cacheFromImage] : [];
 
@@ -186,6 +206,7 @@ if (argv.build) {
       ...(argv.platform ? ["buildx"] : []),
       "build",
       ...tagArgs,
+      ...annotationArgs,
       ...cacheFromArgs,
       ...(argv.noCache ? ["--no-cache"] : []),
       ...(argv.platform ? ["--platform", argv.platform, "--push"] : []),

deploy/helm/terria/.helmignore → deploy/helm/terriamap/.helmignore

@@ -1,6 +1,7 @@
 # Patterns to ignore when building packages.
 # This supports shell glob matching, relative path matching, and
 # negation (prefixed with !). Only one pattern per line.
+.helmignore
 .DS_Store
 # Common VCS dirs
 .git/

deploy/helm/terriamap/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+description: A Helm chart for Kubernetes
+name: terriamap
+version: 0.1.2
+home: https://github.com/TerriaJS/terriamap
+sources: ["https://github.com/TerriaJS/terriamap"]

deploy/helm/terria/charts/terriamap/templates/deployment.yaml → deploy/helm/terriamap/templates/deployment.yaml

@@ -31,7 +31,7 @@ spec:
           - name: terriamap-config-server
             mountPath: /etc/config/server
           - name: terriamap-config-client
-            mountPath: /usr/src/app/component/wwwroot/config.json
+            mountPath: /usr/src/app/wwwroot/config.json
             subPath: config.json
       volumes:
         - name: terriamap-config-client

deploy/helm/terria/charts/terriamap/values.yaml → deploy/helm/terriamap/values.yaml

@@ -1,12 +1,20 @@
+global:
+  rollingUpdate:
+    maxUnavailable: 0
+  exposeNodePorts: false
+  image:
+    repository:
+    tags:
+    pullPolicy:
 nodePort:
 image:
   # By default this pulls ghcr.io/terriajs/terrimap:latest
   # Set "full" to specify a custom terriamap image to be used
   # Or you can set "repository" or "tag" if required
   # full: "ghcr.io/terriajs/terriamap:0.1.1"
-  # repository: ghcr.io/terriajs
+  repository: ghcr.io/terriajs
   # tag: latest
-  pullPolicy: Always
+  pullPolicy: IfNotPresent
 clientConfig:
   initializationUrls:
     - helm

package.json

@@ -13,7 +13,7 @@
     ]
   },
   "name": "terriajs-map",
-  "version": "0.1.2",
+  "version": "0.2.0-alpha.4",
   "description": "Geospatial catalog explorer based on TerriaJS.",
   "license": "Apache-2.0",
   "engines": {
@@ -96,7 +96,7 @@
   "scripts": {
     "prepare": "husky install",
     "docker-build-local": "node deploy/docker/create-docker-context.js --build --push --tag auto --local",
-    "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --tag auto --repository=ghcr.io/terriajs",
+    "docker-build-prod": "node deploy/docker/create-docker-context.js --build --push --platform=linux/amd64,linux/arm64",
     "docker-build-ci": "node deploy/docker/create-docker-context.js --build",
     "start": "terriajs-server --config-file serverconfig.json",
     "gulp": "gulp",