From 711d91aaabb9687cd26904443c23706cd4dbd1ee Mon Sep 17 00:00:00 2001 From: Maksym H <1177472+mordamax@users.noreply.github.com> Date: Wed, 7 Aug 2024 18:03:26 +0100 Subject: [PATCH] frame-omni-bencher short checks (#5268) - Part of https://github.com/paritytech/ci_cd/issues/1006 - Closes: https://github.com/paritytech/ci_cd/issues/1010 - Related: https://github.com/paritytech/polkadot-sdk/pull/4405 - Possibly affecting how frame-omni-bencher works on different runtimes: https://github.com/paritytech/polkadot-sdk/pull/5083 Currently works in parallel with gitlab short benchmarks. Triggered only by adding `GHA-migration` label to assure smooth transition (kind of feature-flag). Later when tested on random PRs we'll remove the gitlab and turn on by default these tests --------- Co-authored-by: Oliver Tale-Yazdi --- .github/workflows/check-changed-files.yml | 57 ------------- .../workflows/check-frame-omni-bencher.yml | 85 +++++++++++++++++++ .../reusable-check-changed-files.yml | 59 +++++++++++++ .github/workflows/tests-linux-stable.yml | 8 +- .github/workflows/tests.yml | 10 +-- 5 files changed, 153 insertions(+), 66 deletions(-) delete mode 100644 .github/workflows/check-changed-files.yml create mode 100644 .github/workflows/check-frame-omni-bencher.yml create mode 100644 .github/workflows/reusable-check-changed-files.yml diff --git a/.github/workflows/check-changed-files.yml b/.github/workflows/check-changed-files.yml deleted file mode 100644 index 657c05cd047d..000000000000 --- a/.github/workflows/check-changed-files.yml +++ /dev/null @@ -1,57 +0,0 @@ -# Reusable workflow to perform checks and generate conditions for other workflows. -# Currently it checks if any Rust (build-related) file is changed -# and if the current (caller) workflow file is changed. -# Example: -# -# jobs: -# changes: -# permissions: -# pull-requests: read -# uses: ./.github/workflows/check-changed-files.yml -# some-job: -# needs: changes -# if: ${{ needs.changes.outputs.rust }} -# ....... - -name: Check changes files - -on: - workflow_call: - # Map the workflow outputs to job outputs - outputs: - rust: - value: ${{ jobs.changes.outputs.rust }} - description: 'true if any of the build-related OR current (caller) workflow files have changed' - current-workflow: - value: ${{ jobs.changes.outputs.current-workflow }} - description: 'true if current (caller) workflow file has changed' - -jobs: - changes: - runs-on: ubuntu-latest - permissions: - pull-requests: read - outputs: - # true if current workflow (caller) file is changed - rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }} - current-workflow: ${{ steps.filter.outputs.current-workflow }} - steps: - - id: current-file - run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT - - run: echo "${{ steps.current-file.outputs.current-workflow-file }}" - # For pull requests it's not necessary to checkout the code - - id: filter - uses: dorny/paths-filter@v3 - with: - predicate-quantifier: 'every' - # current-workflow - check if the current (caller) workflow file is changed - # rust - check if any Rust (build-related) file is changed - filters: | - current-workflow: - - '${{ steps.current-file.outputs.current-workflow-file }}' - rust: - - '**/*' - - '!.github/**/*' - - '!prdoc/**/*' - - '!docs/**/*' - # \ No newline at end of file diff --git a/.github/workflows/check-frame-omni-bencher.yml b/.github/workflows/check-frame-omni-bencher.yml new file mode 100644 index 000000000000..e9db2d912979 --- /dev/null +++ b/.github/workflows/check-frame-omni-bencher.yml @@ -0,0 +1,85 @@ +name: Short benchmarks (frame-omni-bencher) + +on: + push: + branches: + - master + pull_request: + types: [ opened, synchronize, reopened, ready_for_review, labeled ] + merge_group: + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +env: + ARTIFACTS_NAME: frame-omni-bencher-artifacts + +jobs: + changes: + # TODO: remove once migration is complete or this workflow is fully stable + if: contains(github.event.label.name, 'GHA-migration') + permissions: + pull-requests: read + uses: ./.github/workflows/reusable-check-changed-files.yml + + set-image: + # GitHub Actions allows using 'env' in a container context. + # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322 + # This workaround sets the container image for each job using 'set-image' job output. + runs-on: ubuntu-latest + needs: changes + if: ${{ needs.changes.outputs.rust }} + outputs: + IMAGE: ${{ steps.set_image.outputs.IMAGE }} + steps: + - name: Checkout + uses: actions/checkout@v4 + - id: set_image + run: cat .github/env >> $GITHUB_OUTPUT + + run-frame-omni-bencher: + runs-on: arc-runners-polkadot-sdk-beefy + needs: [ set-image, changes ] # , build-frame-omni-bencher ] + if: ${{ needs.changes.outputs.rust }} + timeout-minutes: 30 + strategy: + fail-fast: false # keep running other workflows even if one fails, to see the logs of all possible failures + matrix: + runtime: + [ + westend-runtime, + rococo-runtime, + asset-hub-rococo-runtime, + asset-hub-westend-runtime, + bridge-hub-rococo-runtime, + bridge-hub-westend-runtime, + collectives-westend-runtime, + coretime-rococo-runtime, + coretime-westend-runtime, + people-rococo-runtime, + people-westend-runtime, + glutton-westend-runtime, + ] + container: + image: ${{ needs.set-image.outputs.IMAGE }} + env: + PACKAGE_NAME: ${{ matrix.runtime }} + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: script + run: | + RUNTIME_BLOB_NAME=$(echo $PACKAGE_NAME | sed 's/-/_/g').compact.compressed.wasm + RUNTIME_BLOB_PATH=./target/release/wbuild/$PACKAGE_NAME/$RUNTIME_BLOB_NAME + forklift cargo build --release --locked -p $PACKAGE_NAME -p frame-omni-bencher --features runtime-benchmarks + echo "Running short benchmarking for PACKAGE_NAME=$PACKAGE_NAME and RUNTIME_BLOB_PATH=$RUNTIME_BLOB_PATH" + ls -lrt $RUNTIME_BLOB_PATH + ./target/release/frame-omni-bencher v1 benchmark pallet --runtime $RUNTIME_BLOB_PATH --all --steps 2 --repeat 1 + confirm-frame-omni-benchers-passed: + runs-on: ubuntu-latest + name: All benchmarks passed + needs: run-frame-omni-bencher + steps: + - run: echo '### Good job! All the benchmarks passed 🚀' >> $GITHUB_STEP_SUMMARY diff --git a/.github/workflows/reusable-check-changed-files.yml b/.github/workflows/reusable-check-changed-files.yml new file mode 100644 index 000000000000..47f0620439c7 --- /dev/null +++ b/.github/workflows/reusable-check-changed-files.yml @@ -0,0 +1,59 @@ +# Reusable workflow to perform checks and generate conditions for other workflows. +# Currently it checks if any Rust (build-related) file is changed +# and if the current (caller) workflow file is changed. +# Example: +# +# jobs: +# changes: +# permissions: +# pull-requests: read +# uses: ./.github/workflows/reusable-check-changed-files.yml +# some-job: +# needs: changes +# if: ${{ needs.changes.outputs.rust }} +# ....... + +name: Check changes files + +on: + workflow_call: + # Map the workflow outputs to job outputs + outputs: + rust: + value: ${{ jobs.changes.outputs.rust }} + description: "true if any of the build-related OR current (caller) workflow files have changed" + current-workflow: + value: ${{ jobs.changes.outputs.current-workflow }} + description: "true if current (caller) workflow file has changed" + +jobs: + changes: + runs-on: ubuntu-latest + permissions: + pull-requests: read + outputs: + # true if current workflow (caller) file is changed + rust: ${{ steps.filter.outputs.rust == 'true' || steps.filter.outputs.current-workflow == 'true' }} + current-workflow: ${{ steps.filter.outputs.current-workflow }} + steps: + - id: current-file + run: echo "current-workflow-file=$(echo ${{ github.workflow_ref }} | sed -nE "s/.*(\.github\/workflows\/[a-zA-Z0-9_-]*\.y[a]?ml)@refs.*/\1/p")" >> $GITHUB_OUTPUT + - run: echo "${{ steps.current-file.outputs.current-workflow-file }}" + # For pull requests it's not necessary to checkout the code + - name: Checkout + if: github.event_name != 'pull_request' + uses: actions/checkout@v4 + - id: filter + uses: dorny/paths-filter@v3 + with: + predicate-quantifier: "every" + # current-workflow - check if the current (caller) workflow file is changed + # rust - check if any Rust (build-related) file is changed + filters: | + current-workflow: + - '${{ steps.current-file.outputs.current-workflow-file }}' + rust: + - '**/*' + - '!.github/**/*' + - '!prdoc/**/*' + - '!docs/**/*' diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml index 55addf11de06..da4bb40a2e78 100644 --- a/.github/workflows/tests-linux-stable.yml +++ b/.github/workflows/tests-linux-stable.yml @@ -6,7 +6,7 @@ on: branches: - master pull_request: - types: [opened, synchronize, reopened, ready_for_review] + types: [ opened, synchronize, reopened, ready_for_review ] merge_group: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -16,7 +16,7 @@ jobs: changes: permissions: pull-requests: read - uses: ./.github/workflows/check-changed-files.yml + uses: ./.github/workflows/reusable-check-changed-files.yml set-image: # GitHub Actions allows using 'env' in a container context. @@ -34,7 +34,7 @@ jobs: run: cat .github/env >> $GITHUB_OUTPUT test-linux-stable-int: - needs: [set-image, changes] + needs: [ set-image, changes ] if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 60 @@ -55,7 +55,7 @@ jobs: # https://github.com/paritytech/ci_cd/issues/864 test-linux-stable-runtime-benchmarks: - needs: [set-image, changes] + needs: [ set-image, changes ] if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 60 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index a413d3306159..1be2dd7921e0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,7 +5,7 @@ on: branches: - master pull_request: - types: [opened, synchronize, reopened, ready_for_review] + types: [ opened, synchronize, reopened, ready_for_review ] merge_group: concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -15,7 +15,7 @@ jobs: changes: permissions: pull-requests: read - uses: ./.github/workflows/check-changed-files.yml + uses: ./.github/workflows/reusable-check-changed-files.yml set-image: # GitHub Actions allows using 'env' in a container context. @@ -31,7 +31,7 @@ jobs: run: cat .github/env >> $GITHUB_OUTPUT quick-benchmarks: - needs: [set-image, changes] + needs: [ set-image, changes ] if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 60 @@ -50,7 +50,7 @@ jobs: # cf https://github.com/paritytech/polkadot-sdk/issues/1652 test-syscalls: - needs: [set-image, changes] + needs: [ set-image, changes ] if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 60 @@ -75,7 +75,7 @@ jobs: # fi cargo-check-all-benches: - needs: [set-image, changes] + needs: [ set-image, changes ] if: ${{ needs.changes.outputs.rust }} runs-on: arc-runners-polkadot-sdk-beefy timeout-minutes: 60