Merge branch 'paritytech:master' into master

.cargo/config.toml

@@ -0,0 +1,14 @@
+#
+# An auto defined `clippy` feature was introduced,
+# but it was found to clash with user defined features,
+# so was renamed to `cargo-clippy`.
+#
+# If you want standard clippy run:
+# RUSTFLAGS= cargo clippy
+[target.'cfg(feature = "cargo-clippy")']
+rustflags = [
+  "-Aclippy::all",
+  "-Dclippy::correctness",
+  "-Aclippy::if-same-then-else",
+  "-Aclippy::clone-double-ref"
+]

.dockerignore

@@ -4,3 +4,4 @@ doc
 Dockerfile
 .dockerignore
 .local
+.env*

.gitignore

@@ -21,7 +21,7 @@ rls*.log
 .local
 **/hfuzz_target/
 **/hfuzz_workspace/
-.cargo/
 .cargo-remote.toml
 *.bin
 *.iml
+.maintain/node-template-release/Cargo.lock

.gitlab-ci.yml

@@ -45,6 +45,8 @@ variables:                         &default-vars
   VAULT_SERVER_URL:                "https://vault.parity-mgmt-vault.parity.io"
   VAULT_AUTH_PATH:                 "gitlab-parity-io-jwt"
   VAULT_AUTH_ROLE:                 "cicd_gitlab_parity_${CI_PROJECT_NAME}"
+  SIMNET_FEATURES_PATH:            "simnet_tests/tests"
+  PIPELINE_SCRIPTS_TAG:            "v0.1"
 
 default:
   cache:                           {}
@@ -57,6 +59,14 @@ default:
     paths:
       - artifacts/
 
+.collect-artifacts-short:          &collect-artifacts-short
+  artifacts:
+    name:                          "${CI_JOB_NAME}_${CI_COMMIT_REF_NAME}"
+    when:                          on_success
+    expire_in:                     3 hours
+    paths:
+      - artifacts/
+
 .kubernetes-env:                   &kubernetes-env
   retry:
     max: 2
@@ -168,6 +178,21 @@ default:
     | tee artifacts/benches/$CI_COMMIT_REF_NAME-$CI_COMMIT_SHORT_SHA/::trie::read::small.json'
   - sccache -s
 
+.build-linux-substrate-script:     &build-linux-substrate-script
+  - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
+  - mv ./target/release/substrate ./artifacts/substrate/.
+  - echo -n "Substrate version = "
+  - if [ "${CI_COMMIT_TAG}" ]; then
+      echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
+    else
+      ./artifacts/substrate/substrate --version |
+        sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
+          tee ./artifacts/substrate/VERSION;
+    fi
+  - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
+  - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
+  - sccache -s
+
 #### Vault secrets
 .vault-secrets:                    &vault-secrets
   secrets:
@@ -188,13 +213,13 @@ default:
       file:                        false
     AWX_TOKEN:
       vault:                       cicd/gitlab/$CI_PROJECT_PATH/AWX_TOKEN@kv
-      file:                        false    
+      file:                        false
     CRATES_TOKEN:
       vault:                       cicd/gitlab/$CI_PROJECT_PATH/CRATES_TOKEN@kv
       file:                        false
     DOCKER_CHAOS_TOKEN:
       vault:                       cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_TOKEN@kv
-      file:                        false   
+      file:                        false
     DOCKER_CHAOS_USER:
       vault:                       cicd/gitlab/$CI_PROJECT_PATH/DOCKER_CHAOS_USER@kv
       file:                        false
@@ -311,7 +336,7 @@ cargo-deny:
     when:                          always
     paths:
       - deny.log
-  # FIXME: Temorarily allow to fail.
+  # FIXME: Temporarily allow to fail.
   allow_failure:                   true
 
 cargo-fmt:
@@ -321,6 +346,13 @@ cargo-fmt:
   script:
     - cargo +nightly fmt --all -- --check
 
+cargo-clippy:
+  stage:                           test
+  <<:                              *docker-env
+  <<:                              *test-refs
+  script:
+    - SKIP_WASM_BUILD=1 env -u RUSTFLAGS cargo +nightly clippy
+
 cargo-check-benches:
   stage:                           test
   <<:                              *docker-env
@@ -522,31 +554,40 @@ cargo-check-macos:
 
 #### stage:                        build
 
-check-polkadot-companion-status:
-  stage:                           build
-  image:                           paritytech/tools:latest
-  <<:                              *kubernetes-env
-  <<:                              *vault-secrets
-  rules:
-    - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/                         # PRs
-  script:
-    - ./.maintain/gitlab/check_polkadot_companion_status.sh
-
-check-polkadot-companion-build:
+.check-dependent-project:          &check-dependent-project
   stage:                           build
   <<:                              *docker-env
   <<:                              *test-refs-no-trigger
   <<:                              *vault-secrets
-  needs:
-    - job:                         test-linux-stable-int
-      artifacts:                   false
   script:
-    - ./.maintain/gitlab/check_polkadot_companion_build.sh
-  after_script:
-    - cd polkadot && git rev-parse --abbrev-ref HEAD
-  allow_failure:                   true
+    - git clone
+        --depth=1
+        "--branch=$PIPELINE_SCRIPTS_TAG"
+        https://github.com/paritytech/pipeline-scripts
+    - ./pipeline-scripts/check_dependent_project.sh
+        paritytech
+        substrate
+        --substrate
+        "$DEPENDENT_REPO"
+        "$GITHUB_PR_TOKEN"
+    - cd "$DEPENDENT_REPO" && git rev-parse --abbrev-ref HEAD
+
+# Individual jobs are set up for each dependent project so that they can be ran in parallel.
+# Arguably we could generate a job for each companion in the PR's description using Gitlab's
+# parent-child pipelines but that's more complicated.
+
+check-dependent-polkadot:
+  <<: *check-dependent-project
+  variables:
+    DEPENDENT_REPO: polkadot
+
+check-dependent-cumulus:
+  <<: *check-dependent-project
+  variables:
+    DEPENDENT_REPO: cumulus
 
-build-linux-substrate:             &build-binary
+
+build-linux-substrate:
   stage:                           build
   <<:                              *collect-artifacts
   <<:                              *docker-env
@@ -557,21 +598,21 @@ build-linux-substrate:             &build-binary
   before_script:
     - mkdir -p ./artifacts/substrate/
   script:
-    - WASM_BUILD_NO_COLOR=1 time cargo build --release --verbose
-    - mv ./target/release/substrate ./artifacts/substrate/.
-    - echo -n "Substrate version = "
-    - if [ "${CI_COMMIT_TAG}" ]; then
-        echo "${CI_COMMIT_TAG}" | tee ./artifacts/substrate/VERSION;
-      else
-        ./artifacts/substrate/substrate --version |
-          sed -n -E 's/^substrate ([0-9.]+.*-[0-9a-f]{7,13})-.*$/\1/p' |
-            tee ./artifacts/substrate/VERSION;
-      fi
-    - sha256sum ./artifacts/substrate/substrate | tee ./artifacts/substrate/substrate.sha256
+    - *build-linux-substrate-script
     - printf '\n# building node-template\n\n'
     - ./.maintain/node-template-release.sh ./artifacts/substrate/substrate-node-template.tar.gz
-    - cp -r .maintain/docker/substrate.Dockerfile ./artifacts/substrate/
-    - sccache -s
+
+
+#Build binary for simnet quick tests.
+build-linux-substrate-simnet:
+  stage:                           build
+  <<:                              *collect-artifacts-short
+  <<:                              *docker-env
+  <<:                              *test-refs-no-trigger-prs-only
+  before_script:
+    - mkdir -p ./artifacts/substrate/
+  script:
+    - *build-linux-substrate-script
 
 build-linux-subkey:                &build-subkey
   stage:                           build
@@ -682,6 +723,25 @@ publish-docker-substrate:
       # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance
       dotenv: ./artifacts/substrate/build.env
 
+#publish artifact for quick simnet tests
+publish-docker-substrate-simnet:
+  stage:                           publish
+  <<:                              *build-push-docker-image
+  <<:                              *test-refs-no-trigger-prs-only
+  needs:
+    - job:                         build-linux-substrate-simnet
+      artifacts:                   true
+  variables:
+    <<:                            *default-vars
+    GIT_STRATEGY:                  none
+    DOCKERFILE:                    $PRODUCT.Dockerfile
+    PRODUCT:                       substrate
+    IMAGE_NAME:                    docker.io/paritypr/$PRODUCT
+  artifacts:
+    reports:
+      # this artifact is used in simnet-tests-quick job
+      dotenv: ./artifacts/substrate/build.env
+
 publish-docker-subkey:
   stage:                           publish
   <<:                              *build-push-docker-image
@@ -763,12 +823,14 @@ publish-rustdoc:
     - cp README.md /tmp/doc/
     - git checkout gh-pages
     # Remove directories no longer necessary, as specified in $RUSTDOCS_DEPLOY_REFS.
-    # Also ensure $RUSTDOCS_DEPLOY_REFS is non-space
+    # Also ensure $RUSTDOCS_DEPLOY_REFS is not just empty spaces.
+    # Even though this block spans multiple lines, they are concatenated to run as a single line
+    #   command, so note for the semi-colons in the inner-most code block.
     - if [[ ! -z ${RUSTDOCS_DEPLOY_REFS// } ]]; then
         for FILE in *; do
           if [[ ! " $RUSTDOCS_DEPLOY_REFS " =~ " $FILE " ]]; then
-            echo "Removing ${FILE}..."
-            rm -rf $FILE
+            echo "Removing ${FILE}...";
+            rm -rf $FILE;
           fi
         done
       fi
@@ -841,7 +903,7 @@ deploy-prometheus-alerting-rules:
 # Runs "quick" and "long" tests on nightly schedule and on commit / merge to master
 # A "quick" test is a smoke test where basic check-expect tests run by
 # checking values from metrics exposed by the app.
-# A "long" test is the load testing where we send 50K transactions into the 
+# A "long" test is the load testing where we send 50K transactions into the
 # network and check if all completed successfully
 simnet-tests:
   stage:                           deploy
@@ -862,12 +924,39 @@ simnet-tests:
   before_script:
     - echo "Simnet Tests Config
       docker.io/paritytech/simnet:${SIMNET_REF}
-      ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IAMGE_TAG}"
+      ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
   script:
     - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
         --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests"
         --config="simnet_tests/configs/default_local_testnet.toml"
         --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
   retry: 2
   tags:
-    - parity-simnet
+    - substrate-simnet
+
+#run quick simnet-tests for each PR
+simnet-tests-quick:
+  stage:                           deploy
+  image:                           docker.io/paritytech/simnet:${SIMNET_REF}
+  <<:                              *kubernetes-env
+  <<:                              *vault-secrets
+  <<:                              *test-refs-no-trigger-prs-only
+  variables:
+    SIMNET_FEATURES:               "${SIMNET_FEATURES_PATH}/quick"
+  needs:
+    - job:                         publish-docker-substrate-simnet
+  before_script:
+    - echo "Simnet Tests Config
+      docker.io/paritytech/simnet:${SIMNET_REF}
+      ${SUBSTRATE_IMAGE_NAME} ${SUBSTRATE_IMAGE_TAG}"
+  script:
+    - echo "Image=${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
+    - echo "Features=${SIMNET_FEATURES}"
+    - /home/nonroot/simnet/gurke/scripts/run-test-environment-manager.sh
+        --github-remote-dir="https://github.com/paritytech/substrate/tree/master/simnet_tests"
+        --config="simnet_tests/configs/default_local_testnet.toml"
+        --image="${SUBSTRATE_IMAGE_NAME}:${SUBSTRATE_IMAGE_TAG}"
+        --features="${SIMNET_FEATURES}"
+  tags:
+    - substrate-simnet
+