test-build.yml

name: Test & Build
on: [push, pull_request]

concurrency:
  group: test-build-${{ github.ref_name }}-${{ github.event_name }}
  cancel-in-progress: true

jobs:
  test:
    name: ${{ matrix.module }} Tests (Py ${{ matrix.python-version }})
    runs-on: ubuntu-latest
    timeout-minutes: 60
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11"]
        module: [Api, Core]

    # We want to run on external PRs, but not on our own internal PRs as they'll be run
    # by the push to the branch. This prevents duplicated runs on internal PRs.
    # Some discussion of this here:
    # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository

    steps:
      - uses: actions/checkout@v4

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Apt Packages
        run: |
          sudo apt-get update
          sudo apt-get install --yes libxmlsec1-dev libxml2-dev

      - name: Install Poetry
        uses: ./.github/actions/poetry

      - name: Install Tox
        run: |
          poetry install --only ci
        env:
          POETRY_VIRTUALENVS_CREATE: false

      - name: Run Tests
        run: tox
        env:
          MODULE: ${{ matrix.module }}

      - name: Upload coverage to Codecov
        if: false # Disable temporarily
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: ${{ matrix.module }}

  test-migrations:
    name: Migration Tests
    runs-on: ubuntu-latest
    permissions:
      contents: read

    # We want to run on external PRs, but not on our own internal PRs as they'll be run
    # by the push to the branch. This prevents duplicated runs on internal PRs.
    # Some discussion of this here:
    # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository

    steps:
      - uses: actions/checkout@v4

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Apt Packages
        run: |
          sudo apt-get update
          sudo apt-get install --yes libxmlsec1-dev libxml2-dev

      - name: Install Poetry
        uses: ./.github/actions/poetry

      - name: Install Tox
        run: |
          poetry install --only ci
        env:
          POETRY_VIRTUALENVS_CREATE: false

      - name: Run Migration Tests
        run: tox -e "migration-docker"

      - name: Upload coverage to Codecov
        if: false # Disable temporarily
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: migration

  docker-test-migrations:
    name: Docker migration test
    runs-on: ubuntu-latest
    permissions:
      contents: read

    # We want to run on external PRs, but not on our own internal PRs as they'll be run
    # by the push to the branch. This prevents duplicated runs on internal PRs.
    # Some discussion of this here:
    # https://github.community/t/duplicate-checks-on-push-and-pull-request-simultaneous-event/18012
    if: github.event_name == 'push' || github.event.pull_request.head.repo.full_name != github.repository

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Test migrations
        run: ./docker/ci/test_migrations.sh

  docker-image-build:
    name: Docker build
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    # Only build docker containers on a push event. Otherwise, we won't have
    # permissions to push the built containers into registry.
    if: github.event_name == 'push'

    outputs:
      baseimage-changed: ${{ steps.changes.outputs.baseimage }}
      baseimage: ${{ steps.baseimage.outputs.tag }}

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      # If the base image build was changed, we build it first, so we can test
      # using these changes throughout the rest of the build. If the base image
      # build wasn't changed, we don't use it and just rely on scheduled build.
      - name: Check if base image was changed by this branch
        uses: dorny/paths-filter@v3
        id: changes
        with:
          filters: |
            baseimage:
              - 'docker/Dockerfile.baseimage'

      # We use docker/metadata-action to generate tags, instead of using string
      # interpolation, because it properly handles making sure special
      # characters are escaped, and the repo owner string is lowercase.
      - name: Generate tags for base image
        id: baseimage-meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-baseimage
          tags: |
            type=ref,event=branch
            type=sha
            type=raw,value=latest,enable=${{ github.ref_name == 'main' }}

      # We are using docker/metadata-action here for the same reason as above.
      - name: Generate tag for latest
        id: baseimage-latest
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-baseimage
          tags: |
            type=raw,value=latest

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}
        if: steps.changes.outputs.baseimage == 'true'

      # Build the base image, only if needed.
      - name: Build base image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./docker/Dockerfile.baseimage
          target: baseimage
          cache-from: |
            type=registry,ref=${{ fromJSON(steps.baseimage-latest.outputs.json).tags[0] }}
            type=registry,ref=${{ fromJSON(steps.baseimage-meta.outputs.json).tags[0] }}
          cache-to: |
            type=inline
          platforms: linux/amd64, linux/arm64
          tags: ${{ steps.baseimage-meta.outputs.tags }}
          labels: ${{ steps.baseimage-meta.outputs.labels }}
          push: true
        if: steps.changes.outputs.baseimage == 'true'

      # If the base image was changed, we need to use the tag we just pushed
      # to build the common image. Otherwise, if the base image wasn't changed,
      # we use the latest tag. If the local repo has a built base image, we use
      # that, otherwise we just fall back to the main projects tag.
      - name: Set correct base-image for common image build
        id: baseimage
        run: |
          docker buildx imagetools inspect ${{ fromJSON(steps.baseimage-latest.outputs.json).tags[0] }} > /dev/null
          tag_exists=$?
          if [[ "${{ steps.changes.outputs.baseimage }}" == "true" ]]; then
            tag="${{ fromJSON(steps.baseimage-meta.outputs.json).tags[0] }}"
          elif [[ $tag_exists -eq 0 ]]; then
            tag="${{ fromJSON(steps.baseimage-latest.outputs.json).tags[0] }}"
          else
            tag="ghcr.io/natlibfi/ekirjasto-circ-baseimage:latest"
          fi
          echo "Base image tag: $tag"
          echo tag="$tag" >> "$GITHUB_OUTPUT"

      - name: Build common image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./docker/Dockerfile
          target: common
          cache-to: |
            type=gha,scope=buildkit-${{ github.run_id }},mode=min
          platforms: linux/amd64, linux/arm64
          build-args: |
            BASE_IMAGE=${{ steps.baseimage.outputs.tag }}

  docker-image-test:
    name: Docker test ekirjasto-circ-${{ matrix.image }} (${{ matrix.platform }})
    runs-on: ubuntu-latest
    needs: [docker-image-build]
    permissions:
      contents: read
    strategy:
      fail-fast: false
      matrix:
        platform: ["linux/amd64", "linux/arm64"]
        image: ["scripts", "webapp"]

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Build & Start container
        run: docker compose up -d --build ${{ matrix.image }}
        env:
          BUILD_PLATFORM: ${{ matrix.platform }}
          BUILD_CACHE_FROM: type=gha,scope=buildkit-${{ github.run_id }}
          BUILD_BASE_IMAGE: ${{ needs.docker-image-build.outputs.baseimage }}

      - name: Run tests
        run: ./docker/ci/test_${{ matrix.image }}.sh ${{ matrix.image }}

      - name: Output logs
        if: failure()
        run: docker logs circulation-${{ matrix.image }}-1

      - name: Stop container
        if: always()
        run: docker compose down

  docker-image-push:
    name: Push ekirjasto-circ-${{ matrix.image }}
    runs-on: ubuntu-latest
    needs: [test, test-migrations, docker-test-migrations, docker-image-build, docker-image-test]
    permissions:
      contents: read
      packages: write
    strategy:
      fail-fast: false
      matrix:
        image: ["scripts", "webapp", "exec"]

    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      # See comment here: https://github.com/actions/runner-images/issues/1187#issuecomment-686735760
      - name: Disable network offload
        run: sudo ethtool -K eth0 tx off rx off

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install Poetry
        uses: ./.github/actions/poetry

      - name: Setup Dunamai
        run: poetry install --only ci
        env:
          POETRY_VIRTUALENVS_CREATE: false

      - name: Create version file
        run: |
          echo "__version__ = '$(dunamai from git --style semver)'" >> core/_version.py
          echo "__commit__ = '$(dunamai from git --format {commit} --full-commit)'" >> core/_version.py
          echo "__branch__ = '$(dunamai from git --format {branch})'" >> core/_version.py

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Generate tags for image
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ghcr.io/${{ github.repository_owner }}/ekirjasto-circ-${{ matrix.image }}
          tags: |
            type=semver,pattern={{major}}.{{minor}},priority=10
            type=semver,pattern={{version}},priority=20
            type=ref,event=branch,priority=30
            type=sha,priority=40

      - name: Push image
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ./docker/Dockerfile
          push: true
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          target: ${{ matrix.image }}
          cache-from: type=gha,scope=buildkit-${{ github.run_id }}
          platforms: linux/amd64, linux/arm64
          build-args: |
            BASE_IMAGE=${{ needs.docker-image-build.outputs.baseimage }}