From 3bb0623bdc6d2da6eae54a5bf4edb2dc772779b7 Mon Sep 17 00:00:00 2001
From: Jason Dreyzehner <jason@dreyzehner.com>
Date: Thu, 18 Jan 2024 16:42:50 -0500
Subject: [PATCH] Shard playwright tests in CI

---
 .github/workflows/ci.yml | 131 ++++++++++++++++++++++++++++++---------
 .gitignore               |   2 +
 package.json             |   4 +-
 3 files changed, 105 insertions(+), 32 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9fec7af..75c525c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,61 +6,132 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      - run: yarn install --immutable --immutable-cache
+      - name: Restore .yarn if cached
+        id: restore-yarn
+        uses: actions/cache@v4
+        with:
+          path: .yarn
+          key: yarn-${{ hashFiles('yarn.lock') }}
+      - name: Get Libauth version hash
+        id: libauth-version
+        run: echo "HASH=$(git submodule status libauth | cut -c -40)" >> $GITHUB_OUTPUT
+      - name: Restore libauth if cached
+        id: restore-libauth
+        uses: actions/cache@v4
+        with:
+          path: libauth
+          key: libauth-${{ steps.libauth-version.outputs.HASH }}
+      - name: Pull submodules if uncached
+        if: steps.restore-yarn.outputs.cache-hit != 'true' || steps.restore-libauth.outputs.cache-hit != 'true'
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+      - name: Install dependencies
+        run: yarn install --immutable --immutable-cache
       - run: yarn test:lint
 
-  prod-e2e:
+  e2e-tests:
+    # Only run e2e-tests if lint passes; this also saves running time by reusing cached submodules/dependencies
+    needs: [lint]
     runs-on: ubuntu-latest
     container:
       image: mcr.microsoft.com/playwright:v1.41.0-jammy
+    strategy:
+      fail-fast: false
+      matrix:
+        environment: ['dev', 'prod']
+        shardIndex: [1, 2, 3, 4]
+        shardTotal: [4]
+    name: e2e-tests (${{ matrix.environment }}, ${{ matrix.shardIndex }}/${{ matrix.shardTotal }})
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      # The "preinstall" script runs "git submodule update --init --recursive"
-      # to avoid requiring users to understand this project's submodules
-      # configuration. Since the files are owned by a different user than the
-      # GitHub Action runner (a quirk of GitHub Actions), updating the
-      # submodules again would error: `fatal: detected dubious ownership in repository at '/__w/bitauth-ide/bitauth-ide'`
-      # We can safely disable this check for CI.
-      - run: git config --global --add safe.directory '*'
-      - run: yarn install --immutable --immutable-cache
-      - run: yarn build
-      - run: yarn test:e2e:prod
+      - name: Restore Yarn cache
+        id: restore-yarn
+        uses: actions/cache/restore@v4
+        with:
+          path: .yarn
+          key: yarn-${{ hashFiles('yarn.lock') }}
+      - name: Get Libauth version hash
+        id: libauth-version
+        # The "preinstall" script runs "git submodule update --init --recursive"
+        # to avoid requiring users to understand this project's submodules
+        # configuration. Since the files are owned by a different user than the
+        # GitHub Action runner (a quirk of GitHub Actions), updating the
+        # submodules again would error: `fatal: detected dubious ownership in repository at '/__w/bitauth-ide/bitauth-ide'`
+        # We can safely disable this check for CI.
+        run: git config --global --add safe.directory '*' && echo "HASH=$(git submodule status libauth | cut -c -40)" >> $GITHUB_OUTPUT
+      - name: Restore Libauth build
+        id: restore-libauth
+        uses: actions/cache/restore@v4
+        with:
+          path: libauth
+          key: libauth-${{ steps.libauth-version.outputs.HASH }}
+      - name: Check cache hits
+        if: steps.restore-yarn.outputs.cache-hit != 'true' || steps.restore-libauth.outputs.cache-hit != 'true'
+        run: exit 1
+      - name: Test development build
+        run: yarn test:e2e --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter blob,github
+        if: ${{ matrix.environment == 'dev' }}
+        env:
+          HOME: /root # https://github.com/microsoft/playwright/issues/6500
+      - name: Test production build
+        run: yarn build && yarn test:e2e:prod --shard=${{ matrix.shardIndex }}/${{ matrix.shardTotal }} --reporter blob,github
+        if: ${{ matrix.environment == 'prod' }}
+        env:
+          HOME: /root # https://github.com/microsoft/playwright/issues/6500
       - uses: actions/upload-artifact@v4
-        if: always()
+        if: ${{ matrix.environment == 'prod' && matrix.shardIndex == 1 }}
         with:
           name: dist
           path: dist/
           retention-days: 90
+      - uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: blob-report-${{ matrix.environment }}-${{ matrix.shardIndex }}
+          path: blob-report
+          retention-days: 1
       - uses: codecov/codecov-action@v3
 
-  dev-e2e-and-coverage:
+  merge-reports:
+    # Merge reports after playwright-tests, even if some shards have failed
+    if: always()
+    needs: [e2e-tests]
     runs-on: ubuntu-latest
-    container:
-      image: mcr.microsoft.com/playwright:v1.41.0-jammy
     steps:
       - uses: actions/checkout@v4
-        with:
-          submodules: 'recursive'
       - uses: actions/setup-node@v4
         with:
           node-version: 20
-      - run: git config --global --add safe.directory '*'
-      - run: yarn install --immutable --immutable-cache
-      - run: yarn test:e2e
-      - uses: actions/upload-artifact@v4
-        if: always()
+      - name: Restore Yarn cache
+        id: restore-yarn
+        uses: actions/cache/restore@v4
+        with:
+          path: .yarn
+          key: yarn-${{ hashFiles('yarn.lock') }}
+      - name: Check cache hit
+        if: steps.restore-yarn.outputs.cache-hit != 'true'
+        run: exit 1
+      - run: ls -lah
+      - name: Download blob reports from GitHub Actions Artifacts
+        uses: actions/download-artifact@v4
         with:
-          name: playwright-report
-          path: playwright-report/
+          path: all-blob-reports
+          pattern: blob-report-*
+          merge-multiple: true
+      - run: ls -lah
+      - name: Merge into HTML Report
+        run: yarn playwright merge-reports --reporter html,github ./all-blob-reports
+      - run: ls -lah
+      - name: Upload HTML report
+        uses: actions/upload-artifact@v4
+        with:
+          name: html-report--run-${{ github.run_attempt }}
+          path: playwright-report
           retention-days: 90
-      - uses: codecov/codecov-action@v3
diff --git a/.gitignore b/.gitignore
index 5f176a5..5b38ea2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,6 +9,8 @@ package-lock.json
 /coverage
 /test-results/
 /playwright-report/
+/all-blob-reports/
+/blob-report/
 /playwright/.cache/
 /tests/**/*darwin.png
 /tests/**/*win32.png
diff --git a/package.json b/package.json
index f7bdd24..7e89e84 100644
--- a/package.json
+++ b/package.json
@@ -67,8 +67,8 @@
     "e2e": "playwright test --ui",
     "e2e:report": "playwright show-report",
     "e2e:docker:bash": "docker run --rm -p 9323:9323 -v $(pwd):/work/ -w /work/ -it mcr.microsoft.com/playwright:v1.41.0-jammy /bin/bash",
-    "e2e:docker": "node -e \"fetch('http://localhost:3000').then(res => res.status === 200 ? console.log('Running e2e tests in Docker using base URL: http://localhost:3000') : (console.error(`No server is listening at: http://localhost:3000. Run 'yarn start --host' in another tab and try again.`) && process.exit(1)));\" && yarn e2e:docker:bash -c 'DOCKER=1 yarn test:e2e'",
-    "e2e:docker:prod": "node -e \"fetch('http://localhost:31313').then(res => res.status === 200 ? console.log('Running e2e tests in Docker using base URL: http://localhost:31313') : (console.error(`No server is listening at: http://localhost:31313. Run 'yarn preview --host' in another tab and try again.`) && process.exit(1)));\" && yarn e2e:docker:bash -c 'DOCKER=1 yarn test:e2e:prod'",
+    "e2e:docker": "node -e \"fetch('http://localhost:3000').then(res => res.status === 200 ? console.log('Running e2e tests in Docker using base URL: http://localhost:3000') : Promise.reject()).catch(() => { console.error(`No server is listening at: http://localhost:3000. Run 'yarn start' in another tab and try again.`); process.exit(1); } );\" && yarn e2e:docker:bash -c 'DOCKER=1 yarn test:e2e'",
+    "e2e:docker:prod": "node -e \"fetch('http://localhost:31313').then(res => res.status === 200 ? console.log('Running e2e tests in Docker using base URL: http://localhost:31313') : Promise.reject()).catch(() => {console.error(`No server is listening at: http://localhost:31313. Run 'yarn preview --host' in another tab and try again.`); process.exit(1); });\" && yarn e2e:docker:bash -c 'DOCKER=1 yarn test:e2e:prod'",
     "e2e:update-har": "UPDATE_HAR=1 yarn test:e2e --project=chromium tests/routing.spec.ts",
     "cov": "rm -rf .nyc_output && yarn test:e2e; yarn cov:html && open coverage/index.html",
     "cov:both": "rm -rf .nyc_output && yarn test:e2e && yarn test:e2e:prod; yarn cov:html && open coverage/index.html",