From a723cce04c05b9a3a43fae2d45b34371dab9145f Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 10:48:16 +0200 Subject: [PATCH 1/6] Improve nightly test workflow by separating binaries builds --- .../actions/prepare-binaries/README.md | 3 ++ .../actions/prepare-binaries/action.yml | 28 +++++++++++ .../workflows/build-polkadot-for-nightly.yml | 44 +++++++++++++++++ ...d-staking-miner-playground-for-nightly.yml | 49 +++++++++++++++++++ .github/workflows/nightly.yml | 36 +++----------- 5 files changed, 130 insertions(+), 30 deletions(-) create mode 100644 .github/workflows/actions/prepare-binaries/README.md create mode 100644 .github/workflows/actions/prepare-binaries/action.yml create mode 100644 .github/workflows/build-polkadot-for-nightly.yml create mode 100644 .github/workflows/build-staking-miner-playground-for-nightly.yml diff --git a/.github/workflows/actions/prepare-binaries/README.md b/.github/workflows/actions/prepare-binaries/README.md new file mode 100644 index 000000000..8bef86ca8 --- /dev/null +++ b/.github/workflows/actions/prepare-binaries/README.md @@ -0,0 +1,3 @@ +# Prepare polkadot and staking-miner-playground binaries + +This action downloads the substrate binary produced from the `build-substrate` workflow and puts it into the `$PATH`. \ No newline at end of file diff --git a/.github/workflows/actions/prepare-binaries/action.yml b/.github/workflows/actions/prepare-binaries/action.yml new file mode 100644 index 000000000..ad614b420 --- /dev/null +++ b/.github/workflows/actions/prepare-binaries/action.yml @@ -0,0 +1,28 @@ +name: Prepare polkadot and staking-miner-playground binaries +description: Downloads and configures prebuilt Polkadot and Staking Miner Playground binaries +runs: + using: composite + steps: + - name: Download polkadot binaries + uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 + with: + workflow: build-polkadot-for-nightly.yml + name: nightly-polkadot-binary + path: ./tmp + + - name: Download staking-miner-playground binary + uses: dawidd6/action-download-artifact@e7466d1a7587ed14867642c2ca74b5bcc1e19a2d # v3.0.0 + with: + workflow: build-staking-miner-playground-for-nightly.yml + name: staking-miner-playground-binary + path: ./tmp + + - name: Setup permissions and move to path + shell: bash + run: | + chmod u+x ./tmp/polkadot* ./tmp/staking-miner-playground + ./tmp/polkadot --version + ./tmp/staking-miner-playground --version + mkdir -p ~/.local/bin + mv ./tmp/polkadot* ~/.local/bin + mv ./tmp/staking-miner-playground ~/.local/bin diff --git a/.github/workflows/build-polkadot-for-nightly.yml b/.github/workflows/build-polkadot-for-nightly.yml new file mode 100644 index 000000000..17f43a8b6 --- /dev/null +++ b/.github/workflows/build-polkadot-for-nightly.yml @@ -0,0 +1,44 @@ +name: Build polkadot binary for nightly tests + +on: + # Allow it to be manually ran to rebuild binary when needed: + workflow_dispatch: {} + # Run at 22pm every day for nightly builds. + schedule: + - cron: "0 22 * * *" + +jobs: + tests: + name: Build polkadot binary + runs-on: ubuntu-latest + container: paritytech/ci-unified:bullseye-1.74.0-2023-11-01-v20231204 + steps: + - name: checkout polkadot-sdk + uses: actions/checkout@v4 + with: + repository: paritytech/polkadot-sdk + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + polkadot + cache-on-failure: true + + - name: build polkadot binary + run: | + cargo build -p polkadot --release --features fast-runtime + ls -l + ls -l ./target/release + ./target/release/polkadot --version + + - name: upload polkadot binary + uses: actions/upload-artifact@v3 + with: + name: nightly-polkadot-binary + path: | + ./target/release/polkadot + ./target/release/polkadot-execute-worker + ./target/release/polkadot-prepare-worker + retention-days: 2 + if-no-files-found: error \ No newline at end of file diff --git a/.github/workflows/build-staking-miner-playground-for-nightly.yml b/.github/workflows/build-staking-miner-playground-for-nightly.yml new file mode 100644 index 000000000..247fccb37 --- /dev/null +++ b/.github/workflows/build-staking-miner-playground-for-nightly.yml @@ -0,0 +1,49 @@ +name: Build staking miner playground binary for nightly tests + +on: + # Run when changes to staking miner playground is made + push: + branches: + - main + paths: + - 'staking-miner-playground/**' + # Allow it to be manually ran to rebuild binary when needed: + workflow_dispatch: {} + # Run at 22pm every week on Sunday's for nightly builds. + schedule: + - cron: "0 22 * * 6" + +jobs: + tests: + name: Build staking-miner-playground + runs-on: ubuntu-latest + container: paritytech/ci-unified:bullseye-1.74.0-2023-11-01-v20231204 + steps: + - name: checkout + uses: actions/checkout@v4 + + - name: Rust Cache + uses: Swatinem/rust-cache@v2 + with: + workspaces: | + staking-miner-playground + cache-on-failure: true + + - name: build staking-miner-playground binary + run: | + cd staking-miner-playground + cargo build --release --features test-trimming + ./target/release/staking-miner-playground --version + + - name: Debug + run: | + cd staking-miner-playground + ls -la ./target/release/staking-miner-playground + + - name: upload staking-miner-playground binary + uses: actions/upload-artifact@v3 + with: + name: staking-miner-playground-binary + path: ./staking-miner-playground/target/release/staking-miner-playground + retention-days: 7 + if-no-files-found: error \ No newline at end of file diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1e96dadd4..845a20223 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1,4 +1,4 @@ -name: Daily compatibility check against latest polkadot +name: Nightly test on: schedule: @@ -8,21 +8,14 @@ on: jobs: nightly-test: runs-on: ubuntu-latest + # container: paritytech/ci-unified:bullseye-1.74.0 steps: - name: Checkout repository uses: actions/checkout@v4 - - name: Free space on the runner - run: | - df -h - sudo apt -y autoremove --purge - sudo apt -y autoclean - sudo rm -rf /usr/share/dotnet - sudo rm -rf /opt/ghc - sudo rm -rf "/usr/local/share/boost" - sudo rm -rf "$AGENT_TOOLSDIRECTORY" - df -h + - name: Get Polkadot and Staking Miner Playground binaries + uses: ./.github/workflows/actions/prepare-binaries - name: Cache Rust dependencies uses: swatinem/rust-cache@v2 @@ -32,26 +25,8 @@ jobs: - name: Run nightly tests run: | - sudo apt install -y protobuf-compiler - rustup target add wasm32-unknown-unknown - rustup component add rust-src - git clone -b master --depth 1 https://github.com/paritytech/polkadot-sdk.git polkadot-sdk - cd polkadot-sdk - cargo build -p polkadot --release --features fast-runtime - mkdir -p ~/.local/bin - sudo mv ./target/release/polkadot /usr/bin - sudo mv ./target/release/polkadot-execute-worker /usr/bin - sudo mv ./target/release/polkadot-prepare-worker /usr/bin - polkadot --version - cd - - rm -rf polkadot - cd staking-miner-playground - cargo build --release --features test-trimming - sudo mv ./target/release/staking-miner-playground /usr/bin - staking-miner-playground --version - cd - RUST_LOG=info cargo test --workspace --all-features -- --nocapture - cargo clean + # cargo clean - name: Create an issue on failure if: failure() @@ -68,3 +43,4 @@ jobs: # Look for new *open* issues in this search (we want to # create a new one if we only find closed versions): search_existing: open + From 79c9c7001c7d99d86a525298dddaee7e850e1d5c Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 10:51:28 +0200 Subject: [PATCH 2/6] Fix description --- .github/workflows/actions/prepare-binaries/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/actions/prepare-binaries/README.md b/.github/workflows/actions/prepare-binaries/README.md index 8bef86ca8..11088f64d 100644 --- a/.github/workflows/actions/prepare-binaries/README.md +++ b/.github/workflows/actions/prepare-binaries/README.md @@ -1,3 +1,4 @@ # Prepare polkadot and staking-miner-playground binaries -This action downloads the substrate binary produced from the `build-substrate` workflow and puts it into the `$PATH`. \ No newline at end of file +This action downloads the Polkadot and Staking miner binaries produced from workflows +`build-polkadot-for-nightly.yml` and `build-staking-miner-playground-for-nightly.yml` and puts it into the `$PATH`. \ No newline at end of file From bae53e073578c2ed9982c122878b3beb2e1b60eb Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 10:52:18 +0200 Subject: [PATCH 3/6] Fix description --- .github/workflows/actions/prepare-binaries/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/actions/prepare-binaries/README.md b/.github/workflows/actions/prepare-binaries/README.md index 11088f64d..c4f8b3740 100644 --- a/.github/workflows/actions/prepare-binaries/README.md +++ b/.github/workflows/actions/prepare-binaries/README.md @@ -1,4 +1,4 @@ # Prepare polkadot and staking-miner-playground binaries This action downloads the Polkadot and Staking miner binaries produced from workflows -`build-polkadot-for-nightly.yml` and `build-staking-miner-playground-for-nightly.yml` and puts it into the `$PATH`. \ No newline at end of file +`build-polkadot-for-nightly.yml` and `build-staking-miner-playground-for-nightly.yml` and puts them into the `$PATH`. \ No newline at end of file From 3a59317c2f76e8fa4013b79e3a8910f9ad516f20 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 10:53:56 +0200 Subject: [PATCH 4/6] cleanup --- .github/workflows/build-polkadot-for-nightly.yml | 2 +- .../workflows/build-staking-miner-playground-for-nightly.yml | 2 +- .github/workflows/nightly.yml | 3 --- 3 files changed, 2 insertions(+), 5 deletions(-) diff --git a/.github/workflows/build-polkadot-for-nightly.yml b/.github/workflows/build-polkadot-for-nightly.yml index 17f43a8b6..9f014280c 100644 --- a/.github/workflows/build-polkadot-for-nightly.yml +++ b/.github/workflows/build-polkadot-for-nightly.yml @@ -41,4 +41,4 @@ jobs: ./target/release/polkadot-execute-worker ./target/release/polkadot-prepare-worker retention-days: 2 - if-no-files-found: error \ No newline at end of file + if-no-files-found: error diff --git a/.github/workflows/build-staking-miner-playground-for-nightly.yml b/.github/workflows/build-staking-miner-playground-for-nightly.yml index 247fccb37..c8882179d 100644 --- a/.github/workflows/build-staking-miner-playground-for-nightly.yml +++ b/.github/workflows/build-staking-miner-playground-for-nightly.yml @@ -46,4 +46,4 @@ jobs: name: staking-miner-playground-binary path: ./staking-miner-playground/target/release/staking-miner-playground retention-days: 7 - if-no-files-found: error \ No newline at end of file + if-no-files-found: error diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 845a20223..e53de1b4a 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -8,8 +8,6 @@ on: jobs: nightly-test: runs-on: ubuntu-latest - # container: paritytech/ci-unified:bullseye-1.74.0 - steps: - name: Checkout repository uses: actions/checkout@v4 @@ -26,7 +24,6 @@ jobs: - name: Run nightly tests run: | RUST_LOG=info cargo test --workspace --all-features -- --nocapture - # cargo clean - name: Create an issue on failure if: failure() From 6eed02a85f807f179caf7e49cc64561a9dfa8c3f Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 10:55:57 +0200 Subject: [PATCH 5/6] cleanup --- .github/workflows/build-polkadot-for-nightly.yml | 2 -- .../build-staking-miner-playground-for-nightly.yml | 7 +------ 2 files changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/build-polkadot-for-nightly.yml b/.github/workflows/build-polkadot-for-nightly.yml index 9f014280c..d650eba21 100644 --- a/.github/workflows/build-polkadot-for-nightly.yml +++ b/.github/workflows/build-polkadot-for-nightly.yml @@ -28,8 +28,6 @@ jobs: - name: build polkadot binary run: | cargo build -p polkadot --release --features fast-runtime - ls -l - ls -l ./target/release ./target/release/polkadot --version - name: upload polkadot binary diff --git a/.github/workflows/build-staking-miner-playground-for-nightly.yml b/.github/workflows/build-staking-miner-playground-for-nightly.yml index c8882179d..e1f37d77c 100644 --- a/.github/workflows/build-staking-miner-playground-for-nightly.yml +++ b/.github/workflows/build-staking-miner-playground-for-nightly.yml @@ -35,15 +35,10 @@ jobs: cargo build --release --features test-trimming ./target/release/staking-miner-playground --version - - name: Debug - run: | - cd staking-miner-playground - ls -la ./target/release/staking-miner-playground - - name: upload staking-miner-playground binary uses: actions/upload-artifact@v3 with: name: staking-miner-playground-binary path: ./staking-miner-playground/target/release/staking-miner-playground - retention-days: 7 + retention-days: 8 if-no-files-found: error From b02761dd633d934a9a0adbff91d3ddfe66626fa3 Mon Sep 17 00:00:00 2001 From: Sergejs Kostjucenko Date: Thu, 4 Jan 2024 11:22:29 +0200 Subject: [PATCH 6/6] change SMP rebuild schedule --- .../workflows/build-staking-miner-playground-for-nightly.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-staking-miner-playground-for-nightly.yml b/.github/workflows/build-staking-miner-playground-for-nightly.yml index e1f37d77c..52a3a1cfd 100644 --- a/.github/workflows/build-staking-miner-playground-for-nightly.yml +++ b/.github/workflows/build-staking-miner-playground-for-nightly.yml @@ -11,7 +11,7 @@ on: workflow_dispatch: {} # Run at 22pm every week on Sunday's for nightly builds. schedule: - - cron: "0 22 * * 6" + - cron: "0 22 1 * *" jobs: tests: @@ -40,5 +40,5 @@ jobs: with: name: staking-miner-playground-binary path: ./staking-miner-playground/target/release/staking-miner-playground - retention-days: 8 + retention-days: 31 if-no-files-found: error