build: add docker images

.dockerignore

@@ -5,5 +5,6 @@
 !wort/
 !pixi.lock
 !config/
+!migrations/
 !wortapp.py
 !pyproject.toml

.github/workflows/docker-images.yml

@@ -0,0 +1,166 @@
+# Based on 
+#  - https://docs.github.com/en/actions/use-cases-and-examples/publishing-packages/publishing-docker-images
+#  - https://docs.docker.com/build/ci/github-actions/multi-platform/#distribute-build-across-multiple-runners
+# and the branchwater config
+#  - https://github.com/sourmash-bio/branchwater/blob/main/.github/workflows/docker-images.yml
+
+name: Create and publish Docker images
+
+on:
+  push:
+    branches:
+      - 'main'
+    tags:
+      - 'v*'
+  pull_request:
+    paths:
+      - .github/workflows/docker-images.yml
+      - Dockerfile
+## disabling for most PRs, too many builds
+#    branches:
+#      - 'main'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        target: [web, worker]
+        platform:
+          - linux/amd64
+          - linux/arm64
+        include:
+          - platform: linux/amd64
+            os: ubuntu-latest
+          - platform: linux/arm64
+            os: ubuntu-24.04-arm
+
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+
+    steps:
+      - name: Delete huge unnecessary tools folder
+        run: |
+          rm -rf /opt/hostedtoolcache
+          cd /opt
+          find . -maxdepth 1 -mindepth 1 '!' -path ./containerd '!' -path ./actionarchivecache '!' -path ./runner '!' -path ./runner-cache -exec rm -rf '{}' ';'
+
+      - name: Checkout repository
+        uses: actions/checkout@85e6279cec87321a52edac9c87bce653a07cf6c2
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5
+
+      - name: Prepare
+        run: |
+          platform=${{ matrix.platform }}
+          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and push by digest
+        id: build
+        uses: docker/build-push-action@ca877d9245402d1537745e0e356eab47c3520991
+        with:
+          context: .
+          platforms: ${{ matrix.platform }}
+          labels: ${{ steps.meta.outputs.labels }}
+          target: ${{ matrix.target }}
+          outputs: type=image,"name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}",push-by-digest=true,name-canonical=true,push=true
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+          # don't push on PRs
+          push: ${{ ! startsWith(github.ref, 'refs/pull/') }}
+
+      - name: Export digest
+        run: |
+          mkdir -p ${{ runner.temp }}/digests
+          digest="${{ steps.build.outputs.digest }}"
+          touch "${{ runner.temp }}/digests/${digest#sha256:}"
+
+      - name: Upload digest
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08
+        with:
+          name: digests-${{ matrix.target }}-${{ env.PLATFORM_PAIR }}
+          path: ${{ runner.temp }}/digests/*
+          if-no-files-found: error
+          retention-days: 1
+
+  merge:
+    runs-on: ubuntu-latest
+    # avoid tagging on PRs
+    if: ${{ ! startsWith(github.ref, 'refs/pull/') }}
+    needs:
+      - build
+
+    strategy:
+      matrix:
+        target: [web, worker]
+
+    permissions:
+      contents: read
+      packages: write
+      attestations: write
+      id-token: write
+
+    steps:
+      - name: Download digests
+        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16
+        with:
+          path: ${{ runner.temp }}/digests
+          pattern: digests-${{ matrix.target }}-*
+          merge-multiple: true
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@327cd5a69de6c009b9ce71bce8395f28e651bf99
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@6524bf65af31da8d45b59e8c27de4bd072b392f5
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@8e1d5461f02b7886d3c1a774bfbd873650445aa2
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=ref,event=branch,prefix=${{ matrix.target }}-
+            type=ref,event=pr,prefix=${{ matrix.target }}-pr-
+            type=semver,prefix=${{ matrix.target }}-,pattern={{version}}
+
+      - name: Create manifest list and push
+        working-directory: ${{ runner.temp }}/digests
+        run: |
+          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
+            $(printf '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}@sha256:%s ' *)
+
+      - name: Inspect image
+        run: |
+          docker buildx imagetools inspect ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.meta.outputs.version }}
+
+#      - name: Generate artifact attestation
+#        uses: actions/attest-build-provenance@v2
+#        with:
+#          subject-name: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+#          subject-digest: ${{ steps.build.outputs.digest }}
+#          push-to-registry: true

Dockerfile

@@ -1,18 +1,24 @@
 FROM ghcr.io/prefix-dev/pixi:0.40.3 AS build
 
-COPY . /app
 WORKDIR /app
+
+COPY . .
+
+RUN --mount=type=cache,target=/root/.cache/rattler/cache,sharing=private pixi install
+
 RUN pixi run build-wheel
+
 RUN pixi run -e web postinstall-prod
 RUN pixi shell-hook -e web > /shell-hook-web
 RUN echo 'exec "$@"' >> /shell-hook-web
+
 RUN pixi run -e worker postinstall-prod
 RUN pixi shell-hook -e worker > /shell-hook-worker
 RUN echo 'exec "$@"' >> /shell-hook-worker
 
 #--------------------
 
-FROM ubuntu:24.04 AS web 
+FROM ubuntu:24.04 AS web
 
 # only copy the production environment into prod container
 COPY --from=build /app/.pixi/envs/web /app/.pixi/envs/web
@@ -21,13 +27,14 @@ COPY --from=build /shell-hook-web /shell-hook
 RUN groupadd user && \
     useradd --create-home --home-dir /home/user -g user -s /bin/bash user
 
+USER user
+
 COPY wortapp.py /app
 COPY config/ /app/config
+COPY migrations/ /app/migrations
 
 WORKDIR /app
 
-#USER user 
-
 ENTRYPOINT ["/bin/bash", "/shell-hook"]
 CMD ["gunicorn", "-b", "0.0.0.0:5000", "--access-logfile", "-", "'wortapp:create_app()'"]
 
@@ -47,7 +54,7 @@ COPY config/ /app/config
 
 WORKDIR /app
 
-USER user 
+USER user
 
 # Configure sra-toolkit to disable cache
 RUN mkdir ~/.ncbi
@@ -56,16 +63,19 @@ RUN echo '## auto-generated configuration file - DO NOT EDIT ##''' >> ~/.ncbi/us
 RUN echo '' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/LIBS/GUID = "7737545d-30d4-4d05-875a-2c562df521d5"' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/config/default = "false"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/libs/vdb/quality = "ZR"' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/libs/cloud/accept_aws_charges = "false"' >> ~/.ncbi/user-settings.mkfg
-RUN echo '/libs/cloud/report_instance_identity = "true"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/libs/cloud/report_instance_identity = "false"' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/apps/file/volumes/flatAd = "."' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/apps/refseq/volumes/refseqAd = "."' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/apps/sra/volumes/sraAd = "."' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/apps/sraPileup/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/apps/sraRealign/volumes/ad = "."' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/ad/public/apps/wgs/volumes/wgsAd = "."' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/ad/public/root = "."' >> ~/.ncbi/user-settings.mkfg
-RUN echo '/repository/user/default-path = "/root/ncbi"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/repository/user/default-path = "/home/user/ncbi"' >> ~/.ncbi/user-settings.mkfg
 RUN echo '/repository/user/main/public/cache-disabled = "true"' >> ~/.ncbi/user-settings.mkfg
+RUN echo '/tools/prefetch/download_to_cache = "false"' >> ~/.ncbi/user-settings.mkfg
 
 ENV RAYON_NUM_THREADS 3
 ENTRYPOINT ["/bin/bash", "/shell-hook"]

docker-compose.yml

@@ -1,21 +1,19 @@
-version: '3.4'
-
 services:
 
   db:
-    restart: always
     image: postgres:9.6-alpine
+    restart: always
     volumes:
       - ./data/postgres-data:/var/lib/postgresql/data
     env_file:
       - env.production
 
   worker:
+    image: ghcr.io/sourmash-bio/wort:worker-main
     build:
       context: .
       dockerfile: Dockerfile
       target: worker
-    image: wort-worker
     env_file:
       - iam/wort_s3.env
     command: >
@@ -26,11 +24,12 @@ services:
              -l INFO -c 1
 
   web:
-    restart: always
+    image: ghcr.io/sourmash-bio/wort:web-main
     build:
       context: .
       dockerfile: Dockerfile
       target: web
+    restart: always
     command: >
       gunicorn -b 0.0.0.0:5000
         --access-logfile -
@@ -40,8 +39,6 @@ services:
       FLASK_APP: 'wortapp.py'
     ports:
       - "8082:5000"
-    links:
-      - db
     depends_on:
       - db
       - redis