Merge branch 'shawntabrizi-binary-trie2' of https://github.com/parity…

…tech/polkadot-sdk into shawntabrizi-binary-trie2
paritytech · Sep 27, 2024 · a25052c · a25052c
2 parents 56cd568 + 687f2b1
commit a25052c
Show file tree

Hide file tree

Showing 72 changed files with 3,738 additions and 363 deletions.
diff --git a/.github/scripts/cmd/cmd.py b/.github/scripts/cmd/cmd.py
@@ -100,7 +100,7 @@ def main():
 
         # loop over remaining runtimes to collect available pallets
         for runtime in runtimesMatrix.values():
-            os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features runtime-benchmarks")
+            os.system(f"forklift cargo build -p {runtime['package']} --profile {profile} --features={runtime['bench_features']}")
             print(f'-- listing pallets for benchmark for {runtime["name"]}')
             wasm_file = f"target/{profile}/wbuild/{runtime['package']}/{runtime['package'].replace('-', '_')}.wasm"
             output = os.popen(

diff --git a/.github/scripts/cmd/test_cmd.py b/.github/scripts/cmd/test_cmd.py
@@ -7,10 +7,10 @@
 
 # Mock data for runtimes-matrix.json
 mock_runtimes_matrix = [
-    {"name": "dev", "package": "kitchensink-runtime", "path": "substrate/frame", "header": "substrate/HEADER-APACHE2", "template": "substrate/.maintain/frame-weight-template.hbs"},
-    {"name": "westend", "package": "westend-runtime", "path": "polkadot/runtime/westend", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs"},
-    {"name": "rococo", "package": "rococo-runtime", "path": "polkadot/runtime/rococo", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs"},
-    {"name": "asset-hub-westend", "package": "asset-hub-westend-runtime", "path": "cumulus/parachains/runtimes/assets/asset-hub-westend", "header": "cumulus/file_header.txt", "template": "cumulus/templates/xcm-bench-template.hbs"},
+    {"name": "dev", "package": "kitchensink-runtime", "path": "substrate/frame", "header": "substrate/HEADER-APACHE2",  "template": "substrate/.maintain/frame-weight-template.hbs", "bench_features": "runtime-benchmarks,riscv"},
+    {"name": "westend", "package": "westend-runtime", "path": "polkadot/runtime/westend", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"},
+    {"name": "rococo", "package": "rococo-runtime", "path": "polkadot/runtime/rococo", "header": "polkadot/file_header.txt", "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs", "bench_features": "runtime-benchmarks"},
+    {"name": "asset-hub-westend", "package": "asset-hub-westend-runtime", "path": "cumulus/parachains/runtimes/assets/asset-hub-westend", "header": "cumulus/file_header.txt", "template": "cumulus/templates/xcm-bench-template.hbs", "bench_features": "runtime-benchmarks"},
 ]
 
 def get_mock_bench_output(runtime, pallets, output_path, header, template = None):
@@ -84,10 +84,10 @@ def test_bench_command_normal_execution_all_runtimes(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p kitchensink-runtime --profile release --features runtime-benchmarks"),
-                call("forklift cargo build -p westend-runtime --profile release --features runtime-benchmarks"),
-                call("forklift cargo build -p rococo-runtime --profile release --features runtime-benchmarks"),
-                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"),
+                call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
+                call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"),
+                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
 
                 call(get_mock_bench_output('kitchensink', 'pallet_balances', './substrate/frame/balances/src/weights.rs', os.path.abspath('substrate/HEADER-APACHE2'), "substrate/.maintain/frame-weight-template.hbs")),
                 call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', os.path.abspath('polkadot/file_header.txt'))),
@@ -118,7 +118,7 @@ def test_bench_command_normal_execution(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p westend-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
 
                 # Westend runtime calls
                 call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)),
@@ -149,7 +149,7 @@ def test_bench_command_normal_execution_xcm(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p westend-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
 
                 # Westend runtime calls
                 call(get_mock_bench_output(
@@ -185,8 +185,8 @@ def test_bench_command_two_runtimes_two_pallets(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p westend-runtime --profile release --features runtime-benchmarks"),
-                call("forklift cargo build -p rococo-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p westend-runtime --profile release --features=runtime-benchmarks"),
+                call("forklift cargo build -p rococo-runtime --profile release --features=runtime-benchmarks"),
                 # Westend runtime calls
                 call(get_mock_bench_output('westend', 'pallet_staking', './polkadot/runtime/westend/src/weights', header_path)),
                 call(get_mock_bench_output('westend', 'pallet_balances', './polkadot/runtime/westend/src/weights', header_path)),
@@ -220,7 +220,7 @@ def test_bench_command_one_dev_runtime(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p kitchensink-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p kitchensink-runtime --profile release --features=runtime-benchmarks,riscv"),
                 # Westend runtime calls
                 call(get_mock_bench_output(
                     'kitchensink', 
@@ -254,7 +254,7 @@ def test_bench_command_one_cumulus_runtime(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
                 # Asset-hub-westend runtime calls
                 call(get_mock_bench_output(
                     'asset-hub-westend', 
@@ -288,7 +288,7 @@ def test_bench_command_one_cumulus_runtime_xcm(self):
 
             expected_calls = [
                 # Build calls
-                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features runtime-benchmarks"),
+                call("forklift cargo build -p asset-hub-westend-runtime --profile release --features=runtime-benchmarks"),
                 # Asset-hub-westend runtime calls
                 call(get_mock_bench_output(
                     'asset-hub-westend', 

diff --git a/.github/workflows/check-runtime-migration.yml b/.github/workflows/check-runtime-migration.yml
@@ -17,13 +17,13 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   # More info can be found here: https://github.com/paritytech/polkadot/pull/5865
   check-runtime-migration:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     # We need to set this to rather long to allow the snapshot to be created, but the average time
     # should be much lower.
     timeout-minutes: 60

diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml
@@ -15,14 +15,13 @@ concurrency:
 permissions: {}
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   cargo-clippy:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 40
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -38,7 +37,7 @@ jobs:
   check-try-runtime:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 40
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -57,7 +56,7 @@ jobs:
   check-core-crypto-features:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     timeout-minutes: 30
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -9,15 +9,17 @@ on:
   merge_group:
 
 concurrency:
-  group: ${{ github.ref }}
-  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
 
 jobs:
   preflight:
+    if: contains(github.event.label.name, 'GHA-migration')
     uses: ./.github/workflows/reusable-preflight.yml
 
   test-rustdoc:
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     needs: [preflight]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}
@@ -38,7 +40,8 @@ jobs:
           RUSTFLAGS: "-Cdebug-assertions=y -Dwarnings"
 
   build-rustdoc:
-    runs-on: arc-runners-polkadot-sdk-beefy
+    runs-on: ${{ needs.preflight.outputs.RUNNER }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     needs: [preflight, test-rustdoc]
     container:
       image: ${{ needs.preflight.outputs.IMAGE }}

diff --git a/.github/workflows/runtimes-matrix.json b/.github/workflows/runtimes-matrix.json
@@ -5,6 +5,7 @@
     "path": "substrate/frame",
     "header": "substrate/HEADER-APACHE2",
     "template": "substrate/.maintain/frame-weight-template.hbs",
+    "bench_features": "runtime-benchmarks,riscv",
     "uri": null,
     "is_relay": false
   },
@@ -15,6 +16,7 @@
     "header": "polkadot/file_header.txt",
     "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs",
     "uri": "wss://try-runtime-westend.polkadot.io:443",
+    "bench_features": "runtime-benchmarks",
     "is_relay": true
   },
   {
@@ -24,6 +26,7 @@
     "header": "polkadot/file_header.txt",
     "template": "polkadot/xcm/pallet-xcm-benchmarks/template.hbs",
     "uri": "wss://try-runtime-rococo.polkadot.io:443",
+    "bench_features": "runtime-benchmarks",
     "is_relay": true
   },
   {
@@ -32,6 +35,7 @@
     "path": "cumulus/parachains/runtimes/assets/asset-hub-westend",
     "header": "cumulus/file_header.txt",
     "template": "cumulus/templates/xcm-bench-template.hbs",
+    "bench_features": "runtime-benchmarks",
     "uri": "wss://westend-asset-hub-rpc.polkadot.io:443",
     "is_relay": false
   },
@@ -49,6 +53,7 @@
     "package": "bridge-hub-rococo-runtime",
     "path": "cumulus/parachains/runtimes/bridges/bridge-hub-rococo",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-bridge-hub-rpc.polkadot.io:443",
     "is_relay": false
@@ -58,6 +63,7 @@
     "package": "bridge-hub-rococo-runtime",
     "path": "cumulus/parachains/runtimes/bridges/bridge-hub-westend",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-bridge-hub-rpc.polkadot.io:443",
     "is_relay": false
@@ -67,6 +73,7 @@
     "package": "collectives-westend-runtime",
     "path": "cumulus/parachains/runtimes/collectives/collectives-westend",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-collectives-rpc.polkadot.io:443"
   },
@@ -75,6 +82,7 @@
     "package": "contracts-rococo-runtime",
     "path": "cumulus/parachains/runtimes/contracts/contracts-rococo",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-contracts-rpc.polkadot.io:443",
     "is_relay": false
@@ -84,6 +92,7 @@
     "package": "coretime-rococo-runtime",
     "path": "cumulus/parachains/runtimes/coretime/coretime-rococo",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-coretime-rpc.polkadot.io:443",
     "is_relay": false
@@ -93,6 +102,7 @@
     "package": "coretime-westend-runtime",
     "path": "cumulus/parachains/runtimes/coretime/coretime-westend",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-coretime-rpc.polkadot.io:443",
     "is_relay": false
@@ -102,6 +112,7 @@
     "package": "glutton-westend-runtime",
     "path": "cumulus/parachains/runtimes/gluttons/glutton-westend",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": null,
     "is_relay": false
@@ -111,6 +122,7 @@
     "package": "people-rococo-runtime",
     "path": "cumulus/parachains/runtimes/people/people-rococo",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-people-rpc.polkadot.io:443",
     "is_relay": false
@@ -120,6 +132,7 @@
     "package": "people-westend-runtime",
     "path": "cumulus/parachains/runtimes/people/people-westend",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://westend-people-rpc.polkadot.io:443",
     "is_relay": false

diff --git a/.github/workflows/tests-linux-stable.yml b/.github/workflows/tests-linux-stable.yml
@@ -13,13 +13,12 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-
   preflight:
     uses: ./.github/workflows/reusable-preflight.yml
 
   test-linux-stable-int:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     timeout-minutes: 60
     container:
@@ -40,7 +39,7 @@ jobs:
   # https://github.com/paritytech/ci_cd/issues/864
   test-linux-stable-runtime-benchmarks:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     timeout-minutes: 60
     container:
@@ -58,7 +57,7 @@ jobs:
 
   test-linux-stable:
     needs: [preflight]
-    # if: ${{ needs.preflight.outputs.changes_rust }}
+    if: ${{ needs.preflight.outputs.changes_rust }}
     runs-on: ${{ matrix.runners }}
     timeout-minutes: 60
     strategy: