Merge branch 'master' into rq/expose-client

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v202407161507"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"

.github/scripts/generate-prdoc.py

@@ -67,7 +67,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 		# Go up until we find a Cargo.toml
 		p = os.path.join(workspace.path, p)
 		while not os.path.exists(os.path.join(p, "Cargo.toml")):
-			print(f"Could not find Cargo.toml in {p}")
 			if p == '/':
 				exit(1)
 			p = os.path.dirname(p)
@@ -76,7 +75,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 			manifest = toml.load(f)
 
 		if not "package" in manifest:
-			print(f"File was not in any crate: {p}")
 			continue
 
 		crate_name = manifest["package"]["name"]
@@ -85,8 +83,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 		else:
 			print(f"Skipping unpublished crate: {crate_name}")
 
-	print(f"Modified crates: {modified_crates.keys()}")
-
 	for crate_name in modified_crates.keys():
 		entry = { "name": crate_name }
 
@@ -137,4 +133,4 @@ def main(args):
 
 if __name__ == "__main__":
 	args = setup_parser().parse_args()
-	main(args)
+	main(args)

.github/workflows/check-runtime-migration.yml

@@ -17,13 +17,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   # More info can be found here: https://github.com/paritytech/polkadot/pull/5865
   check-runtime-migration:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     # We need to set this to rather long to allow the snapshot to be created, but the average time
     # should be much lower.
     timeout-minutes: 60

.github/workflows/checks.yml

@@ -15,14 +15,13 @@ concurrency:
 permissions: {}
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   cargo-clippy:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 40
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -33,31 +32,31 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
       - name: script
         run: |
-          forklift cargo clippy --all-targets --locked --workspace
-          forklift cargo clippy --all-targets --all-features --locked --workspace
+          forklift cargo clippy --all-targets --locked --workspace --quiet
+          forklift cargo clippy --all-targets --all-features --locked --workspace --quiet
   check-try-runtime:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 40
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     steps:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
       - name: script
         run: |
-          forklift cargo check --locked --all --features try-runtime
+          forklift cargo check --locked --all --features try-runtime --quiet
           # this is taken from cumulus
           # Check that parachain-template will compile with `try-runtime` feature flag.
           forklift cargo check --locked -p parachain-template-node --features try-runtime
           # add after https://github.com/paritytech/substrate/pull/14502 is merged
           # experimental code may rely on try-runtime and vice-versa
-          forklift cargo check --locked --all --features try-runtime,experimental
+          forklift cargo check --locked --all --features try-runtime,experimental --quiet
   # check-core-crypto-features works fast without forklift
   check-core-crypto-features:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 30
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

.github/workflows/command-prdoc.yml

@@ -28,7 +28,7 @@ on:
           - "Runtime Dev"
           - "Runtime User"
           - "Node Dev"
-          - "Node User"
+          - "Node Operator"
       overwrite:
         type: choice
         description: Overwrite existing PrDoc

.github/workflows/docs.yml

@@ -9,15 +9,17 @@ on:
   merge_group:
 
 concurrency:
-  group: ${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   preflight:
+    if: contains(github.event.label.name, 'GHA-migration')
     uses: ./.github/workflows/reusable-preflight.yml
 
   test-rustdoc:
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     needs: [preflight]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -38,7 +40,8 @@ jobs:
           RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
 
   build-rustdoc:
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     needs: [preflight, test-rustdoc]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

.github/workflows/tests-linux-stable.yml

@@ -13,13 +13,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   test-linux-stable-int:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     timeout-minutes: 60
     container:
@@ -40,7 +39,7 @@ jobs:
   # https://github.com/paritytech/ci_cd/issues/864
   test-linux-stable-runtime-benchmarks:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     timeout-minutes: 60
     container:
@@ -54,11 +53,11 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: forklift cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
+        run: forklift cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet --cargo-quiet
 
   test-linux-stable:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ matrix.runners }}
     timeout-minutes: 60
     strategy:
@@ -91,12 +90,13 @@ jobs:
             --locked \
             --release \
             --no-fail-fast \
+            --cargo-quiet \
             --features try-runtime,experimental,riscv,ci-only-tests \
             --partition count:${{ matrix.partition }}
       # run runtime-api tests with `enable-staging-api` feature on the 1st node
       - name: runtime-api tests
         if: ${{ matrix.partition == '1/3' }}
-        run: forklift cargo nextest run -p sp-api-test --features enable-staging-api
+        run: forklift cargo nextest run -p sp-api-test --features enable-staging-api --cargo-quiet
 
   confirm-required-jobs-passed:
     runs-on: ubuntu-latest

.github/workflows/tests-misc.yml

@@ -22,6 +22,7 @@ jobs:
   test-full-crypto-feature:
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 60
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -46,6 +47,7 @@ jobs:
     # into one job
     needs: [preflight, test-full-crypto-feature]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
@@ -67,34 +69,36 @@ jobs:
     timeout-minutes: 60
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
       # Enable debug assertions since we are running optimized builds for testing
       # but still want to have debug assertions.
       RUSTFLAGS: "-C debug-assertions -D warnings"
       RUST_BACKTRACE: 1
-      WASM_BUILD_NO_COLOR: 1
-      WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
+      SKIP_WASM_BUILD: 1
       # Ensure we run the UI tests.
       RUN_UI_TESTS: 1
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
         run: |
-          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
-          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
-          forklift cargo test --locked -q --profile testnet -p xcm-procedural
-          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
-          forklift cargo test --locked -q --profile testnet -p sp-api-test
+          cargo version
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental ui
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental ui
+          forklift cargo test --locked -q --profile testnet -p xcm-procedural ui
+          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type ui
+          forklift cargo test --locked -q --profile testnet -p sp-api-test ui
           # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
-          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
+          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml ui
 
   test-deterministic-wasm:
     timeout-minutes: 20
     needs: [preflight, test-frame-examples-compile-to-wasm]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
@@ -194,6 +198,7 @@ jobs:
     needs: [preflight]
     timeout-minutes: 30
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     steps:
@@ -226,6 +231,7 @@ jobs:
     timeout-minutes: 20
     needs: [preflight, test-node-metrics]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     steps:
@@ -241,6 +247,7 @@ jobs:
     timeout-minutes: 20
     needs: [preflight, check-tracing]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     steps:
@@ -251,50 +258,52 @@ jobs:
         run: |
           forklift cargo build --locked -p westend-runtime --features metadata-hash
 
-  cargo-hfuzz:
-    timeout-minutes: 20
-    needs: [preflight, check-metadata-hash]
-    runs-on: ${{ needs.preflight.outputs.RUNNER }}
-    container:
-      image: ${{ needs.preflight.outputs.IMAGE }}
-    env:
-      # max 10s per iteration, 60s per file
-      HFUZZ_RUN_ARGS: |
-        --exit_upon_crash
-        --exit_code_upon_crash 1
-        --timeout 10
-        --run_time 60
-
-      # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
-      # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
-      # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
-      # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
-      HFUZZ_BUILD_ARGS: |
-        --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
-        --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v4
-
-      - name: Run honggfuzz
-        run: |
-          cd substrate/primitives/arithmetic/fuzzer
-          forklift cargo hfuzz build
-          for target in $(cargo read-manifest | jq -r '.targets | .[] | .name');
-          do
-            forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; };
-          done
-
-      - name: Upload artifacts
-        uses: actions/[email protected]
-        with:
-          path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
-          name: hfuzz-${{ github.sha }}
+  # disabled until https://github.com/paritytech/polkadot-sdk/issues/5812 is resolved
+  # cargo-hfuzz:
+  #   timeout-minutes: 20
+  #   needs: [preflight, check-metadata-hash]
+  #   runs-on: ${{ needs.preflight.outputs.RUNNER }}
+  #   container:
+  #     image: ${{ needs.preflight.outputs.IMAGE }}
+  #   env:
+  #     # max 10s per iteration, 60s per file
+  #     HFUZZ_RUN_ARGS: |
+  #       --exit_upon_crash
+  #       --exit_code_upon_crash 1
+  #       --timeout 10
+  #       --run_time 60
+
+  #     # use git version of honggfuzz-rs until v0.5.56 is out, we need a few recent changes:
+  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/75 to avoid breakage on debian
+  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/81 fix to the above pr
+  #     # https://github.com/rust-fuzz/honggfuzz-rs/pull/82 fix for handling absolute CARGO_TARGET_DIR
+  #     HFUZZ_BUILD_ARGS: |
+  #       --config=patch.crates-io.honggfuzz.git="https://github.com/altaua/honggfuzz-rs"
+  #       --config=patch.crates-io.honggfuzz.rev="205f7c8c059a0d98fe1cb912cdac84f324cb6981"
+  #   steps:
+  #     - name: Checkout
+  #       uses: actions/checkout@v4
+
+  #     - name: Run honggfuzz
+  #       run: |
+  #         cd substrate/primitives/arithmetic/fuzzer
+  #         forklift cargo hfuzz build
+  #         for target in $(cargo read-manifest | jq -r '.targets | .[] | .name');
+  #         do
+  #           forklift cargo hfuzz run "$target" || { printf "fuzzing failure for %s\n" "$target"; exit 1; };
+  #         done
+
+  #     - name: Upload artifacts
+  #       uses: actions/[email protected]
+  #       with:
+  #         path: substrate/primitives/arithmetic/fuzzer/hfuzz_workspace/
+  #         name: hfuzz-${{ github.sha }}
 
   cargo-check-each-crate:
     timeout-minutes: 140
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
     env:
@@ -322,6 +331,7 @@ jobs:
     timeout-minutes: 30
     needs: [preflight]
     runs-on: ${{ needs.preflight.outputs.RUNNER_MACOS }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     env:
       SKIP_WASM_BUILD: 1
     steps: