From 5f476eb25e5b09b31640f6a40c96b3a57a32dadb Mon Sep 17 00:00:00 2001 From: Leorize Date: Tue, 10 Dec 2024 15:11:20 -0600 Subject: [PATCH] ci: split into smaller workflows with some improvements Previously, the main workflow runs in stages for each of the tested platforms. However, this poses a problem where a stage is blocked by the slowest runner and none of the other runners could move on. With the release of reusable workflows for Actions, we can now separate the stages into a different workflow, then parallelize that instead, allowing faster runners to proceed through test stages ahead of the rest. The response time for test results should now be greatly improved, with the M1 runner finishing its run in just 7 minutes. This separation also opened the doors to customizing how tests are run for each platforms, and we now only use 1 test batch for *nix while allocating 3 to Windows. This normalizes test time between Windows and *nix, while allowing freed up runners to process other jobs. Lastly, some tiny QoL improvements are added to the pipeline: - Test failures will now be displayed at the end of "Run tester" step, reducing the number of clicks required to see all failures to 1. - The diffoscope container is now used instead of the Ubuntu package, reducing installation time and let us receive latest features/bug fixes. - If there is a diffoscope failure, a test summary is now generated with a link to the test report. This should make it a lot easier to get to the report should that be necessary. --- .github/workflows/build-and-test.yml | 224 ++++++++++++ .github/workflows/ci.yml | 514 ++++----------------------- .github/workflows/package.yml | 150 ++++++++ doc/ci.rst | 64 ++-- 4 files changed, 490 insertions(+), 462 deletions(-) create mode 100644 .github/workflows/build-and-test.yml create mode 100644 .github/workflows/package.yml diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml new file mode 100644 index 00000000000..ee19fc77038 --- /dev/null +++ b/.github/workflows/build-and-test.yml @@ -0,0 +1,224 @@ +name: Build and test +on: + workflow_call: + inputs: + runner: + description: The runner to run the workflow with + required: true + type: string + + batch-count: + description: The number of test batches to parallelize + required: false + default: 1 + type: number + +defaults: + run: + shell: bash + +jobs: + context: + name: Compute context for the workflow + runs-on: ubuntu-latest + outputs: + result: ${{ steps.context.outputs.result }} + + steps: + - uses: actions/github-script@v7 + id: context + with: + script: | + let num_batches = Number.parseInt(process.env.BATCH_COUNT, 10); + num_batches = Number.isNaN(num_batches) ? 2 : num_batches; + num_batches = Math.max(num_batches, 1); + + const batch_array = Array(num_batches).fill(0).map((_, idx) => ({ + 'idx': idx, + 'display': idx + 1, + })); + + return { + 'test-batch': batch_array, + 'num-test-batch': batch_array.length, + }; + env: + BATCH_COUNT: ${{ inputs.batch-count }} + + binaries: + name: Build release binaries + runs-on: ${{ inputs.runner }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + + - name: Enable annotations + run: echo "::add-matcher::.github/nim-problem-matcher.json" + + - name: Install MinGW (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/setup-mingw + + - name: Setup vcpkg (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/setup-vcpkg + with: + triplet: x64-mingw-dynamic-release + host-triplet: x64-mingw-dynamic-release + revision: 2024.01.12 + overlay-triplets: ${{ github.workspace }}/tools/vcpkg/triplets + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + run: vcpkg install pcre sqlite3 + + - name: Set Xcode version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "15.0.1" + + - name: Set macOS SDK version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + run: | + sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) + echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" + + - name: Build release binaries + run: ./koch.py all-strict + + - name: Upload workspace to artifacts + uses: ./.github/actions/upload-compiler + + test: + needs: [context, binaries] + strategy: + fail-fast: false + + matrix: + batch: ${{ fromJSON(needs.context.outputs.result).test-batch }} + num-batch: + - ${{ fromJSON(needs.context.outputs.result).num-test-batch }} + + runs-on: ${{ inputs.runner }} + name: Test compiler and stdlib (batch ${{ matrix.batch.display }}/${{ matrix.num-batch }}) + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + + - uses: ./.github/actions/download-compiler + + - name: Install NodeJS + uses: actions/setup-node@v4 + with: + node-version: "16" + + - name: Install dependencies (Linux) + if: runner.os == 'Linux' + run: | + deps=( + # Required by ARC/ORC memory leak tests (only enabled on linux x64) + libc6-dbg + valgrind + ) + + sudo apt-get update + sudo apt-get install "${deps[@]}" + + - name: Set Xcode version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "15.0.1" + + - name: Set macOS SDK version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + run: | + sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) + echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" + + - name: Install dependencies (macOS) + if: runner.os == 'macOS' + run: | + brew update + + # Downgrade to OpenSSL 3.0 + brew unlink openssl + brew install openssl@3.0 + brew link --overwrite openssl@3.0 + + - name: Add Homebrew libraries to search path + if: runner.os == 'macOS' + run: echo "DYLD_FALLBACK_LIBRARY_PATH=$(brew --prefix)/lib" >> "$GITHUB_ENV" + + - name: Install MinGW (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/setup-mingw + + - name: Add DLLs to PATH (Windows) + if: runner.os == 'Windows' + run: | + $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" + $binPath | Out-File -Append $env:GITHUB_PATH + shell: pwsh + + - name: Run tester + run: | + if ! ./koch.py test --batch:"$TEST_BATCH" --tryFailing all; then + oldExitCode=$? + # XXX: Move this to either koch or testament since it's useful. + echo === + echo Errors occured during tests: + bin/nim r tools/ci_testresults + exit $oldExitCode + fi + env: + TEST_BATCH: ${{ matrix.batch.idx }}_${{ matrix.num-batch }} + + tooling: + needs: [binaries] + name: Build and test tooling + runs-on: ${{ inputs.runner }} + + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + + - name: Install MinGW (Windows) + if: runner.os == 'Windows' + uses: ./.github/actions/setup-mingw + + - name: Add DLLs to PATH (Windows) + if: runner.os == 'Windows' + run: | + $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" + $binPath | Out-File -Append $env:GITHUB_PATH + shell: pwsh + + - name: Set Xcode version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "15.0.1" + + - name: Set macOS SDK version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + run: | + sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) + echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" + + - uses: ./.github/actions/download-compiler + + - name: Enable annotations + run: echo "::add-matcher::.github/nim-problem-matcher.json" + + - name: Test tooling + run: ./koch.py testTools diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4f9aebd8e94..8867da45df9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,4 +1,4 @@ -name: Build and test +name: CI on: push: # Empty configuration means use default (ie. test all branches) @@ -66,20 +66,16 @@ jobs: # The schema is: # [ # { - # name: String, ## The name of the target being tested - # runner: String ## The runner to use of this target - # shared_builder?: Bool ## Whether this target should be used to - # ## build artifacts shared between - # ## platforms. Only one target may have - # ## this attribute. + # name: String, ## The name of the target being tested + # runner: String, ## The runner to use of this target + # batch-count?: Int, ## Number of parallel test batches # } # ] cat << "EOF" > matrix.json [ { "name": "Linux", - "runner": "ubuntu-20.04", - "shared_builder": true + "runner": "ubuntu-20.04" }, { "name": "macOS", @@ -91,17 +87,16 @@ jobs: }, { "name": "Windows", - "runner": "windows-2022" + "runner": "windows-2022", + "batch-count": 3 } ] EOF # Use jq to compact the matrix into one line to be used as the result echo "result=$(jq -c . matrix.json)" >> $GITHUB_OUTPUT - # Isolate the shared builder into its own thing as well - echo "shared=$(jq -c '.[] | select(.shared_builder)' matrix.json)" >> $GITHUB_OUTPUT - binaries: + test: needs: [pre_run] if: needs.pre_run.outputs.skip != 'true' @@ -111,199 +106,18 @@ jobs: matrix: target: ${{ fromJson(needs.pre_run.outputs.target_matrix) }} - name: Build release binaries (${{ matrix.target.name }}) - runs-on: ${{ matrix.target.runner }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - filter: tree:0 - - - name: Enable annotations - run: echo "::add-matcher::.github/nim-problem-matcher.json" - - - name: Install MinGW (Windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-mingw - - - name: Setup vcpkg (Windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-vcpkg - with: - triplet: x64-mingw-dynamic-release - host-triplet: x64-mingw-dynamic-release - revision: 2024.01.12 - overlay-triplets: ${{ github.workspace }}/tools/vcpkg/triplets - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: vcpkg install pcre sqlite3 - - - name: Set Xcode version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "15.0.1" - - - name: Set macOS SDK version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - run: | - sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) - echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" - - - name: Build release binaries - run: ./koch.py all-strict - - - name: Upload workspace to artifacts - uses: ./.github/actions/upload-compiler - - test: - needs: [pre_run, binaries] - - strategy: - fail-fast: false - - matrix: - target: ${{ fromJson(needs.pre_run.outputs.target_matrix) }} - # This controls the testament "batch" feature. - # - # If any additional batches are added, increment `total_batch` as well. - # - # This feature allow us to parallelize testing. - batch: [0, 1] - # This tells testament how many batches are used. Have to be placed in - # an array due to how Github Actions process matrices. - total_batch: [2] - - name: "Test the compiler and stdlib (${{ matrix.target.name }}, batch #${{ matrix.batch }})" - runs-on: ${{ matrix.target.runner }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - filter: tree:0 - - - uses: ./.github/actions/download-compiler - - - name: Install NodeJS - uses: actions/setup-node@v4 - with: - node-version: "16" - - - name: Install dependencies (Linux) - if: runner.os == 'Linux' - run: | - deps=( - # Required by ARC/ORC memory leak tests (only enabled on linux x64) - libc6-dbg - valgrind - ) - - sudo apt-get update - sudo apt-get install "${deps[@]}" - - - name: Set Xcode version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "15.0.1" - - - name: Set macOS SDK version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - run: | - sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) - echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" - - - name: Install dependencies (macOS) - if: runner.os == 'macOS' - run: | - brew update - - # Downgrade to OpenSSL 3.0 - brew unlink openssl - brew install openssl@3.0 - brew link --overwrite openssl@3.0 - - - name: Add Homebrew libraries to search path - if: runner.os == 'macOS' - run: echo "DYLD_FALLBACK_LIBRARY_PATH=$(brew --prefix)/lib" >> "$GITHUB_ENV" - - - name: Install MinGW (Windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-mingw - - - name: Add DLLs to PATH (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - shell: pwsh - - - name: Run tester - run: ./koch.py test --batch:"$TEST_BATCH" --tryFailing all - env: - TEST_BATCH: ${{ matrix.batch }}_${{ matrix.total_batch }} - - - name: Print all test errors - if: failure() - run: bin/nim r tools/ci_testresults - - tooling: - needs: [pre_run, binaries] - - strategy: - fail-fast: false - - matrix: - target: ${{ fromJson(needs.pre_run.outputs.target_matrix) }} - - name: Build and test tooling (${{ matrix.target.name }}) - runs-on: ${{ matrix.target.runner }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - filter: tree:0 - - - name: Install MinGW (Windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-mingw - - - name: Add DLLs to PATH (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - shell: pwsh - - - name: Set Xcode version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "15.0.1" - - - name: Set macOS SDK version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - run: | - sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) - echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" - - - uses: ./.github/actions/download-compiler - - - name: Enable annotations - run: echo "::add-matcher::.github/nim-problem-matcher.json" - - - name: Test tooling - run: ./koch.py testTools + name: Build and test (${{ matrix.target.name }}) + uses: ./.github/workflows/build-and-test.yml + with: + runner: ${{ matrix.target.runner }} + batch-count: ${{ matrix.target.batch-count || 1 }} source: - needs: [pre_run, binaries] + needs: [pre_run] + if: needs.pre_run.outputs.skip != 'true' name: Build source archive - runs-on: ${{ fromJson(needs.pre_run.outputs.shared_builder).runner }} + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -311,11 +125,12 @@ jobs: fetch-depth: 0 filter: tree:0 - - uses: ./.github/actions/download-compiler - - name: Enable annotations run: echo "::add-matcher::.github/nim-problem-matcher.json" + - name: Build compiler + run: ./koch.py boot -d:danger + - name: Generate csources run: ./koch.py csource -d:danger @@ -342,8 +157,8 @@ jobs: ${{ steps.archive.outputs.metadata }} if-no-files-found: error - source_binaries: - needs: [pre_run, source] + package-git: + needs: [pre_run] strategy: fail-fast: false @@ -351,117 +166,14 @@ jobs: matrix: target: ${{ fromJson(needs.pre_run.outputs.target_matrix) }} - name: Build release artifacts from source archive (${{ matrix.target.name }}) - runs-on: ${{ matrix.target.runner }} - - steps: - - name: Get actions - uses: actions/checkout@v4 - with: - path: git-src - sparse-checkout: .github/actions - - - name: Install MinGW (Windows) - if: runner.os == 'Windows' - uses: ./git-src/.github/actions/setup-mingw - - - name: Set Xcode version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "15.0.1" - - - name: Set macOS SDK version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - run: | - sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) - echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" - - - name: Download source archive - uses: actions/download-artifact@v4 - with: - name: source archive - path: source-archive - - - name: Unpack source archive - run: | - archive=source-archive/$(jq -r .name source-archive/source.json) - # Pipe from zstd to tar because macOS' tar does not support unpacking zstd - zstd -c -d "$archive" | tar -xf - --strip-components 1 - - - name: Setup vcpkg (Windows) - if: runner.os == 'Windows' - uses: ./git-src/.github/actions/setup-vcpkg - with: - triplet: x64-mingw-dynamic-release - host-triplet: x64-mingw-dynamic-release - revision: 2024.01.12 - overlay-triplets: ${{ github.workspace }}/tools/vcpkg/triplets - - - name: Install dependencies (Windows) - if: runner.os == 'Windows' - run: vcpkg install pcre sqlite3 - - - name: Add DLLs to PATH (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - shell: pwsh - - - name: Install DLLs to package (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - Copy-Item (Join-Path $binPath "libpcre.dll") -Destination bin - shell: pwsh - - - name: Build release binaries - run: ./koch.py all - - # Note: keep synchronized with package job - - name: Build docs - run: | - ./koch.py doc \ - --git.url:"https://github.com/$GITHUB_REPOSITORY" \ - --git.commit:"$GITHUB_SHA" \ - --git.devel:devel + name: Package from Git (${{ matrix.target.name }}) + uses: ./.github/workflows/package.yml + with: + runner: ${{ matrix.target.runner }} + output-name-format: release binaries {0} {1} - # Remove leftover nimskullcache - rm -rf doc/html/nimskullcache - - - id: package - name: Create release package - run: | - releasecmd=unixrelease - if [[ "$RUNNER_OS" == Windows ]]; then - releasecmd=winrelease - fi - ./koch.py "$releasecmd" - - archive=build/archive/$(jq -r .name build/archive/archive.json) - # Rename the archive manifest to avoid collision with other artifacts - os=$(jq -r .os build/archive/archive.json) - cpu=$(jq -r .cpu build/archive/archive.json) - metadata=build/archive/${os}_${cpu}.json - mv build/archive/archive.json "$metadata" - - # Let the uploader know what to upload - echo "archive=$archive" >> $GITHUB_OUTPUT - echo "metadata=$metadata" >> $GITHUB_OUTPUT - - - name: Upload release package to artifacts - uses: actions/upload-artifact@v4 - with: - name: binaries from source archive ${{ matrix.target.name }} - path: | - ${{ steps.package.outputs.archive }} - ${{ steps.package.outputs.metadata }} - if-no-files-found: error - - package: - needs: [pre_run, binaries] + package-source: + needs: [pre_run, source] strategy: fail-fast: false @@ -469,96 +181,18 @@ jobs: matrix: target: ${{ fromJson(needs.pre_run.outputs.target_matrix) }} - name: Build docs and release artifacts (${{ matrix.target.name }}) - runs-on: ${{ matrix.target.runner }} - - steps: - - uses: actions/checkout@v4 - with: - fetch-depth: 0 - filter: tree:0 - - - uses: ./.github/actions/download-compiler + name: Package from archive (${{ matrix.target.name }}) + uses: ./.github/workflows/package.yml + with: + runner: ${{ matrix.target.runner }} + source-archive: source archive + output-name-format: binaries from source archive {0} {1} - - name: Enable annotations - run: echo "::add-matcher::.github/nim-problem-matcher.json" - - - name: Install MinGW (Windows) - if: runner.os == 'Windows' - uses: ./.github/actions/setup-mingw - - - name: Add DLLs to PATH (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - shell: pwsh - - - name: Install DLLs to package (Windows) - if: runner.os == 'Windows' - run: | - $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" - $binPath | Out-File -Append $env:GITHUB_PATH - Copy-Item (Join-Path $binPath "libpcre.dll") -Destination bin - shell: pwsh - - - name: Set Xcode version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: "15.0.1" - - - name: Set macOS SDK version (macOS M1) - if: runner.os == 'macOS' && runner.arch == 'ARM64' - run: | - sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) - echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" - - # Note: keep synchronized with source_binaries job - - name: Build docs - run: | - ./koch.py doc \ - --git.url:"https://github.com/$GITHUB_REPOSITORY" \ - --git.commit:"$GITHUB_SHA" \ - --git.devel:devel - - # Remove leftover nimskullcache - rm -rf doc/html/nimskullcache - - - id: package - name: Create release package - run: | - releasecmd=unixrelease - if [[ "$RUNNER_OS" == Windows ]]; then - releasecmd=winrelease - fi - ./koch.py "$releasecmd" - - archive=build/archive/$(jq -r .name build/archive/archive.json) - # Rename the archive manifest to avoid collision with other artifacts - os=$(jq -r .os build/archive/archive.json) - cpu=$(jq -r .cpu build/archive/archive.json) - metadata=build/archive/${os}_${cpu}.json - mv build/archive/archive.json "$metadata" - - # Let the uploader know what to upload - echo "archive=$archive" >> $GITHUB_OUTPUT - echo "metadata=$metadata" >> $GITHUB_OUTPUT - - - name: Upload release package to artifacts - uses: actions/upload-artifact@v4 - with: - name: release binaries ${{ matrix.target.name }} - path: | - ${{ steps.package.outputs.archive }} - ${{ steps.package.outputs.metadata }} - if-no-files-found: error - - test_package: - needs: [pre_run, package, source_binaries] + test-package: + needs: [pre_run, package-git, package-source] name: Test release artifacts - runs-on: ${{ fromJSON(needs.pre_run.outputs.shared_builder).runner }} + runs-on: ubuntu-20.04 steps: - uses: actions/checkout@v4 @@ -566,11 +200,6 @@ jobs: fetch-depth: 0 filter: tree:0 - - uses: ./.github/actions/download-compiler - - - name: Enable annotations - run: echo "::add-matcher::.github/nim-problem-matcher.json" - - name: Download binaries built from source archive uses: actions/download-artifact@v4 with: @@ -592,36 +221,44 @@ jobs: name: source archive path: source-archive - - name: Install tools + - name: Binaries from git and source archive should be the same run: | - sudo apt-get update - sudo apt-get install -yqq --no-install-recommends diffoscope + podman run \ + --rm \ + --workdir /src \ + -v "$(pwd):/src" \ + --user 0:0 \ + registry.salsa.debian.org/reproducible-builds/diffoscope:latest \ + --progress \ + --html=build/diffoscope.html \ + --exclude-directory-metadata=yes \ + release-binary/ \ + binary-from-source/ + + - name: Upload diffoscope output on failure + id: diffoscope-output + if: failure() + uses: actions/upload-artifact@v4 + with: + name: differences between git and source binaries + path: build/diffoscope.html - - id: diff - name: Binaries from git and source archive should be the same + - name: Write summary on failure + if: failure() && steps.diffoscope-output.outputs.artifact-url != '' run: | - output_html=$RUNNER_TEMP/diffoscope.html - run_diff() { - # Exclude directory metadata as we only care about the files themselves - diffoscope \ - --html="$output_html" \ - --exclude-directory-metadata=yes \ - release-binary/ binary-from-source/ - } - if ! run_diff; then - echo "::error::There are differences when building from source archive compared to building from git, check the output uploaded to artifacts for more details" - echo "result=$output_html" >> $GITHUB_OUTPUT - exit 1 - else - echo "Success! Binaries built from git and source archive are the same" - fi + cat <> "$GITHUB_STEP_SUMMARY" + ### Binaries from git and source archive are not the same! - - name: Upload difference test result on failure - if: failure() && steps.diff.outputs.result != '' - uses: actions/upload-artifact@v4 - with: - name: differences between binary from source archive and git - path: ${{ steps.diff.outputs.result }} + The report can be downloaded [here]($DIFFOSCOPE_OUTPUT_URL) + EOF + env: + DIFFOSCOPE_OUTPUT_URL: ${{ steps.diffoscope-output.outputs.artifact-url }} + + - name: Enable annotations + run: echo "::add-matcher::.github/nim-problem-matcher.json" + + - name: Build compiler + run: ./koch.py boot -d:danger - name: Verify archive manifests run: | @@ -646,14 +283,9 @@ jobs: passed: name: All check passed needs: - - binaries - test - - tooling - - source - - source_binaries - - package - - test_package - if: always() + - test-package + if: cancelled() || failure() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/package.yml b/.github/workflows/package.yml new file mode 100644 index 00000000000..8d97469cb88 --- /dev/null +++ b/.github/workflows/package.yml @@ -0,0 +1,150 @@ +name: Build release package +on: + workflow_call: + inputs: + runner: + description: The runner to run the workflow with + required: true + type: string + + source-archive: + description: The artifact containing the source archive. If not set, build from Git instead. + required: false + type: string + + output-name-format: + description: The format string for the output artifact name. {0} and {1} are mapped to OS and architecture respectively. + required: false + default: release binaries {0} {1} + type: string + +defaults: + run: + shell: bash + +jobs: + package: + name: Build release artifacts + runs-on: ${{ inputs.runner }} + + steps: + - name: Checkout (for Git build) + if: inputs.source-archive == '' + uses: actions/checkout@v4 + with: + fetch-depth: 0 + filter: tree:0 + + - name: Extract actions to separate folder (for Git build) + if: inputs.source-archive == '' + run: | + mkdir -p git-src/.github + cp -r .github/actions git-src/.github/actions + + - name: Get actions (for source-archive builds) + if: inputs.source-archive != '' + uses: actions/checkout@v4 + with: + path: git-src + sparse-checkout: .github/actions + + - name: Install MinGW (Windows) + if: runner.os == 'Windows' + uses: ./git-src/.github/actions/setup-mingw + + - name: Set Xcode version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + uses: maxim-lobanov/setup-xcode@v1 + with: + xcode-version: "15.0.1" + + - name: Set macOS SDK version (macOS M1) + if: runner.os == 'macOS' && runner.arch == 'ARM64' + run: | + sdkpath=$(xcrun --sdk macosx14.0 --show-sdk-path) + echo "SDKROOT=$sdkpath" >> "$GITHUB_ENV" + + - name: Download source archive + if: inputs.source-archive != '' + uses: actions/download-artifact@v4 + with: + name: ${{ inputs.source-archive }} + path: ${{ runner.temp }}/source-archive + + - name: Unpack source archive + if: inputs.source-archive != '' + run: | + archive_base=$RUNNER_TEMP/source-archive + archive=$archive_base/$(jq -r .name $archive_base/source.json) + # Pipe from zstd to tar because macOS' tar does not support unpacking zstd + zstd -c -d "$archive" | tar -xf - --strip-components 1 + + - name: Setup vcpkg (Windows) + if: runner.os == 'Windows' + uses: ./git-src/.github/actions/setup-vcpkg + with: + triplet: x64-mingw-dynamic-release + host-triplet: x64-mingw-dynamic-release + revision: 2024.01.12 + overlay-triplets: ${{ github.workspace }}/tools/vcpkg/triplets + + - name: Install dependencies (Windows) + if: runner.os == 'Windows' + run: vcpkg install pcre sqlite3 + + - name: Add DLLs to PATH (Windows) + if: runner.os == 'Windows' + run: | + $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" + $binPath | Out-File -Append $env:GITHUB_PATH + shell: pwsh + + - name: Install DLLs to package (Windows) + if: runner.os == 'Windows' + run: | + $binPath = Join-Path $PWD "vcpkg" "installed" "x64-mingw-dynamic-release" "bin" + $binPath | Out-File -Append $env:GITHUB_PATH + Copy-Item (Join-Path $binPath "libpcre.dll") -Destination bin + shell: pwsh + + - name: Build release binaries + run: ./koch.py all + + - name: Build docs + run: | + ./koch.py doc \ + --git.url:"https://github.com/$GITHUB_REPOSITORY" \ + --git.commit:"$GITHUB_SHA" \ + --git.devel:devel + + # Remove leftover nimskullcache + rm -rf doc/html/nimskullcache + + - id: package + name: Create release package + run: | + releasecmd=unixrelease + if [[ "$RUNNER_OS" == Windows ]]; then + releasecmd=winrelease + fi + ./koch.py "$releasecmd" + + archive=build/archive/$(jq -r .name build/archive/archive.json) + # Rename the archive manifest to avoid collision with other artifacts + os=$(jq -r .os build/archive/archive.json) + cpu=$(jq -r .cpu build/archive/archive.json) + metadata=build/archive/${os}_${cpu}.json + mv build/archive/archive.json "$metadata" + + # Let the uploader know what to upload + echo "archive=$archive" >> $GITHUB_OUTPUT + echo "metadata=$metadata" >> $GITHUB_OUTPUT + + - name: Upload release package to artifacts + uses: actions/upload-artifact@v4 + with: + name: ${{ format(inputs.output-name-format, runner.os, runner.arch) }} + path: | + ${{ steps.package.outputs.archive }} + ${{ steps.package.outputs.metadata }} + if-no-files-found: error diff --git a/doc/ci.rst b/doc/ci.rst index b27c7273a69..5c4dbd00db4 100644 --- a/doc/ci.rst +++ b/doc/ci.rst @@ -124,17 +124,14 @@ Pipelines This section describes the design of each pipeline and the dependencies between them. -"Build and test" ----------------- +"CI" +---- Goals ..... * Verify that the compiler, tools, and standard library test suites pass -* Verify that the compiler can be bootstrapped with ORC. Whether this compiler - is fully functional is not tested at the moment. - * Verify that a source archive can be built. * Compiler and tools built from the source archive work. @@ -183,12 +180,42 @@ jobs. This allows for: * Quick turn-around time for test runs without waiting for packaging to finish. Github Actions UI provides an overall look into the dependency graph which can -be seen from "Summary" page of any "Build and test" `workflow run +be seen from "Summary" page of any "CI" `workflow run `_. In addition to the public API, there are several internal APIs used for sharing data between jobs in this pipeline: +* "binaries from source archive": Contains binaries built from the source + archive generated from the git clone. This is generated by + ``package-source`` job and will be compared to a similar artifact generated + by the ``package-git`` job. + + This comparison is done to make sure that we can generate the same binaries + regardless of whether it is compiled via cloning a Git repo or via our source + archive. + +* ``pre_run`` outputs: The ``pre_run`` job provides data that dictates how CI + will be run: + + * ``skipped``: Whether a run should be skipped. + + * ``matrix``: The target matrix shared between jobs. + +"Build and test" +---------------- + +This workflow builds and test the compiler against the full test suite on a +given runner. This workflow is reusable, which meant it can be called by other +workflows with varying customizations. Please refer to the workflow file at +``.github/workflows/build-and-test.yml`` for more information. + +This workflow is used by "CI" workflow to perform testing across all supported +platforms. + +Internally, this pipeline provides the following API used to share data between +jobs: + * "Download compiler" and "Upload compiler" actions: These can be found in ``.github/actions/download-compiler`` and ``.github/actions/upload-compiler`` respectively. These actions allow a job to upload its workspace, all files @@ -198,20 +225,16 @@ data between jobs in this pipeline: This is used to replicate the "workspace with binaries" across parallel jobs so they can run independently. -* "binaries from source archive": Contains binaries built from the source - archive generated from the git clone. This is generated by - ``source_binaries`` job and will be compared to a similar artifact generated - by the ``package`` job. - - For the comparison to succeed, it's crucial that the packaging code is - the same between ``source_binaries`` and ``package``. +"Build release package" +----------------------- -* ``pre_run`` outputs: The ``pre_run`` job provides data that dictates how CI - will be run: +This workflow builds the compiler release package (including docs) on a given +runner. It can build either from Git or from a source archive. Refer to the +workflow file at ``.github/workflows/package.yml`` for information on how +to employ these. - * ``skipped``: Whether a run should be skipped. - - * ``matrix``: The target matrix shared between jobs. +This workflow is used by "CI" workflow to build binaries for all supported +platforms, and to generate binaries from source and Git for comparison. "Test compiler build reproducibility" ------------------------------------- @@ -234,9 +257,8 @@ the `reprotest `_ tool. "Publish" --------- -Publishes build results from prior run of the "Build and test" pipeline on -the staging branch. Changes to the public API of "Build and test" must be -tested against "Publish". +Publishes build results from prior run of the "CI" pipeline on the staging +branch. Changes to the public API of "CI" must be tested against "Publish". This pipeline will perform pushes to the repository, so a lot of care must be put into any changes to it.