tests-misc.yml

name: tests misc

on:
  push:
    branches:
      - master
  pull_request:
    types: [ opened, synchronize, reopened, ready_for_review ]
  merge_group:
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    permissions:
      pull-requests: read
    uses: ./.github/workflows/reusable-check-changed-files.yml

  set-image:
    needs: [ changes ]
    if: needs.changes.outputs.rust || needs.changes.outputs.current-workflow
    # GitHub Actions allows using 'env' in a container context.
    # However, env variables don't work for forks: https://github.com/orgs/community/discussions/44322
    # This workaround sets the container image for each job using 'set-image' job output.
    runs-on: ubuntu-latest
    outputs:
      IMAGE: ${{ steps.set_image.outputs.IMAGE }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - id: set_image
        run: cat .github/env >> $GITHUB_OUTPUT
  #
  #  # more information about this job can be found here:
  #  # https://github.com/paritytech/substrate/pull/3778
  #  test-full-crypto-feature:
  #    needs: [ set-image ]
  #    runs-on: arc-runners-polkadot-sdk-beefy
  #    timeout-minutes: 60
  #    container:
  #      image: ${{ needs.set-image.outputs.IMAGE }}
  #    env:
  #      # Enable debug assertions since we are running optimized builds for testing
  #      # but still want to have debug assertions.
  #      RUSTFLAGS: "-C debug-assertions"
  #      RUST_BACKTRACE: 1
  #    steps:
  #      - name: Checkout
  #        uses: actions/checkout@v4.1.7
  #
  #      - name: script
  #        run: |
  #          cd substrate/primitives/core/
  #          time forklift cargo build --locked --no-default-features --features full_crypto
  #          cd ../application-crypto
  #          time forklift cargo build --locked --no-default-features --features full_crypto
  #
  #  test-frame-examples-compile-to-wasm:
  #    timeout-minutes: 20
  #    # into one job
  #    needs: [ set-image, test-full-crypto-feature ]
  #    runs-on: arc-runners-polkadot-sdk-beefy
  #    container:
  #      image: ${{ needs.set-image.outputs.IMAGE }}
  #    env:
  #      # Enable debug assertions since we are running optimized builds for testing
  #      # but still want to have debug assertions.
  #      RUSTFLAGS: "-C debug-assertions"
  #      RUST_BACKTRACE: 1
  #    steps:
  #      - name: Checkout
  #        uses: actions/checkout@v4
  #      - name: script
  #        run: |
  #          cd substrate/frame/examples/offchain-worker/
  #          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
  #          cd ../basic
  #          time forklift cargo build --locked --target=wasm32-unknown-unknown --no-default-features
  #
  #  test-frame-ui:
  #    timeout-minutes: 60
  #    needs: [ set-image, test-frame-examples-compile-to-wasm ]
  #    runs-on: arc-runners-polkadot-sdk-beefy
  #    container:
  #      image: ${{ needs.set-image.outputs.IMAGE }}
  #    env:
  #      # Enable debug assertions since we are running optimized builds for testing
  #      # but still want to have debug assertions.
  #      RUSTFLAGS: "-C debug-assertions -D warnings"
  #      RUST_BACKTRACE: 1
  #      WASM_BUILD_NO_COLOR: 1
  #      WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
  #      # Ensure we run the UI tests.
  #      RUN_UI_TESTS: 1
  #    steps:
  #      - name: Checkout
  #        uses: actions/checkout@v4
  #      - name: script
  #        run: |
  #          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
  #          time forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
  #          time forklift cargo test --locked -q --profile testnet -p xcm-procedural
  #          time forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
  #          time forklift cargo test --locked -q --profile testnet -p sp-api-test
  #          # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
  #          time forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
  #
  #  test-deterministic-wasm:
  #    timeout-minutes: 20
  #    needs: [ set-image, test-frame-ui ]
  #    runs-on: arc-runners-polkadot-sdk-beefy
  #    container:
  #      image: ${{ needs.set-image.outputs.IMAGE }}
  #    env:
  #      WASM_BUILD_NO_COLOR: 1
  #    steps:
  #      - name: Checkout
  #        uses: actions/checkout@v4
  #      - name: script
  #        run: |
  #          # build runtime
  #          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
  #          # make checksum
  #          sha256sum target/release/wbuild/*-runtime/target/wasm32-unknown-unknown/release/*.wasm > checksum.sha256
  #          cargo clean
  #          # build again
  #          time forklift cargo build -q --locked --release -p westend-runtime -p rococo-runtime
  #          # confirm checksum
  #          sha256sum -c checksum.sha256
  #
  #
  #  cargo-check-benches:
  #    needs: [ set-image ]
  #    timeout-minutes: 60
  #    runs-on: arc-runners-polkadot-sdk-beefy
  #    container:
  #      image: ${{ needs.set-image.outputs.IMAGE }}
  #    steps:
  #      - name: Checkout
  #        uses: actions/checkout@v4
  #
  #      - name: Extract branch name
  #        shell: bash
  #        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
  #        id: extract_branch
  #
  #      - name: script
  #        run: |
  #          ARTIFACTS_DIR=./artifacts
  #          BENCH_TRIE_READ=::trie::read::small
  #          BENCH_NODE_IMPORT=::node::import::sr25519::transfer_keep_alive::paritydb::small
  #          mkdir -p $ARTIFACTS_DIR
  #
  #          SKIP_WASM_BUILD=1 forklift time cargo check --locked --benches --all;
  #          forklift cargo run --locked --release -p node-bench -- $BENCH_TRIE_READ --json | tee $ARTIFACTS_DIR/bench_trie_read_small.json;
  #          forklift cargo run --locked --release -p node-bench -- $BENCH_NODE_IMPORT --json | tee $ARTIFACTS_DIR/bench_transfer_keep_alive.json
  #
  #      - name: Upload artifacts
  #        uses: actions/upload-artifact@v4.3.6
  #        with:
  #          path: ./artifacts
  #          name: cargo-check-benches-${{ steps.extract_branch.outputs.branch }}-${{ github.sha }}
  #          retention-days: 10

  node-bench-regression-guard:
    timeout-minutes: 20
    if: ${{ github.ref != 'refs/heads/master' }} # no need to compare master with master
    runs-on: arc-runners-polkadot-sdk-default
    #    needs: [ set-image, cargo-check-benches ]
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.7

      - name: Extract branch name
        shell: bash
        run: echo "branch=${GITHUB_HEAD_REF:-${GITHUB_REF#refs/heads/}}" >> $GITHUB_OUTPUT
        id: extract_branch

      - name: Download artifact (master)
        uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 #v6
        with:
          name: cargo-check-benches-master
          path: ./artifacts/master
          branch: ${{ github.event.pull_request.head.ref }} # TODO: remove
          search_artifacts: true
          workflow_search: true

      - name: Download artifact (current run)
        uses: dawidd6/action-download-artifact@bf251b5aa9c2f7eeb574a96ee720e24f801b7c11 #v6
        with:
          name: cargo-check-benches-${{ steps.extract_branch.outputs.branch }}-${{ github.sha }}
          path: ./artifacts/current

      - name: script
        run: |
          echo "------- IMPORTANT -------"
          echo "node-bench-regression-guard depends on the results of a cargo-check-benches job"
          echo "In case of this job failure, check your pipeline's cargo-check-benches"
          docker run --rm \
          -v $PWD/artifacts/master:/artifacts/master \
          -v $PWD/artifacts/current:/artifacts/current \
          paritytech/node-bench-regression-guard:latest \
          node-bench-regression-guard --reference /artifacts/master --compare-with /artifacts/current


#  test-node-metrics:
#    needs: [ set-image ]
#    timeout-minutes: 30
#    runs-on: arc-runners-polkadot-sdk-beefy
#    container:
#      image: ${{ needs.set-image.outputs.IMAGE }}
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v4.1.7
#
#      - name: Run tests
#        id: tests
#        env:
#          RUST_TOOLCHAIN: stable
#          # Enable debug assertions since we are running optimized builds for testing
#          # but still want to have debug assertions.
#          RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
#        run: |
#          forklift cargo build --bin polkadot-execute-worker --bin polkadot-prepare-worker --profile testnet --verbose --locked
#          mkdir -p ./artifacts
#          forklift cargo test --profile testnet --locked --features=runtime-metrics -p polkadot-node-metrics > ./artifacts/log.txt
#          echo "Metrics test passed"
#
#      - name: Upload artifacts if failed
#        if: ${{ steps.tests.outcome != 'success' }}
#        uses: actions/upload-artifact@v4.3.6
#        with:
#          name: node-metrics-failed
#          path: ./artifacts
#
#  # more information about this job can be found here:
#  # https://github.com/paritytech/substrate/pull/6916
#  check-tracing:
#    timeout-minutes: 20
#    needs: [ set-image, test-node-metrics ]
#    runs-on: arc-runners-polkadot-sdk-beefy
#    container:
#      image: ${{ needs.set-image.outputs.IMAGE }}
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v4.1.7
#
#      - name: script
#        run: |
#          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features
#          forklift cargo test --locked --manifest-path ./substrate/primitives/tracing/Cargo.toml --no-default-features --features=with-tracing
#
#  check-metadata-hash:
#    timeout-minutes: 20
#    needs: [ set-image, check-tracing ]
#    runs-on: arc-runners-polkadot-sdk-beefy
#    container:
#      image: ${{ needs.set-image.outputs.IMAGE }}
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v4.1.7
#
#      - name: script
#        run: |
#          time forklift cargo build --locked -p westend-runtime --features metadata-hash
#
#  cargo-hfuzz:
#    timeout-minutes: 20
#    needs: [ set-image, check-metadata-hash ]
#    runs-on: arc-runners-polkadot-sdk-beefy
#    container:
#      image: ${{ needs.set-image.outputs.IMAGE }}
#    env:
#      # max 10s per iteration, 60s per file
#      HFUZZ_RUN_ARGS: |
#        --exit_upon_crash
#        --exit_code_upon_crash 1
#        --timeout 10
#        --run_time 60
#
#      # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
#      # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
#      # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
#      # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
#      HFUZZ_BUILD_ARGS: |
#        --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
#        --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v4.1.7
#
#      - name: Run honggfuzz
#        run: |
#          cd substrate/primitives/arithmetic/fuzzer
#          forklift cargo hfuzz build
#          for target in $(cargo read-manifest | jq -r '.targets | .[] | .name');
#          do
#            forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; };
#          done
#
#      - name: Upload artifacts
#        uses: actions/upload-artifact@v4.3.6
#        with:
#          path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
#          name: hfuzz-${{ github.sha }}
#
#  cargo-check-each-crate:
#    timeout-minutes: 140
#    needs: [ set-image ] # [ set-image, check-metadata-hash ]
#    runs-on: arc-runners-polkadot-sdk-beefy
#    container:
#      image: ${{ needs.set-image.outputs.IMAGE }}
#    env:
#      RUSTFLAGS: "-D warnings"
#      CI_JOB_NAME: cargo-check-each-crate
#    strategy:
#      matrix:
#        index: [ 1,2,3,4,5,6,7 ] # 7 parallel jobs
#    steps:
#      - name: Checkout
#        uses: actions/checkout@v4.1.7
#
#      - name: Check Rust
#        run: |
#          rustup show
#          rustup +nightly show
#
#      - name: script
#        run: |
#          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }}
#
  cargo-check-each-crate-macos:
    timeout-minutes: 120
    needs: [ set-image ]
    runs-on: macos-latest
    env:
      RUSTFLAGS: "-D warnings"
      CI_JOB_NAME: cargo-check-each-crate
      IMAGE: ${{ needs.set-image.outputs.IMAGE }}
    strategy:
      matrix:
        index: [ 1,2,3,4,5,6,7 ] # 7 parallel jobs
    steps:
      - name: Checkout
        uses: actions/checkout@v4.1.7

      - run: |
          VERSION=$(echo $IMAGE | sed -E 's/.*:bullseye-([^-]+)-.*/\1/')
          echo $VERSION
          echo "VERSION=$VERSION" >> $GITHUB_ENV

      - run: |
          rustup install $VERSION
          rustup default $VERSION

      - name: Check Rust
        run: |
          rustup show
          rustup +nightly show

      - name: MacOS Deps
        run: |
          brew install protobuf openssl pkg-config zlib xz zstd llvm jq curl gcc make cmake
          rustup target add wasm32-unknown-unknown --toolchain $VERSION
          rustup component add rust-src rustfmt clippy --toolchain $VERSION

      - name: script
        run: |
          PYTHONUNBUFFERED=x time .github/scripts/check-each-crate.py ${{ matrix.index }} ${{ strategy.job-total }} True