Merge branch 'master' into shawntabrizi-binary-trie2

.github/env

@@ -1 +1 @@
-IMAGE="docker.io/paritytech/ci-unified:bullseye-1.77.0-2024-04-10-v202407161507"
+IMAGE="docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"

.github/scripts/generate-prdoc.py

@@ -67,7 +67,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 		# Go up until we find a Cargo.toml
 		p = os.path.join(workspace.path, p)
 		while not os.path.exists(os.path.join(p, "Cargo.toml")):
-			print(f"Could not find Cargo.toml in {p}")
 			if p == '/':
 				exit(1)
 			p = os.path.dirname(p)
@@ -76,7 +75,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 			manifest = toml.load(f)
 
 		if not "package" in manifest:
-			print(f"File was not in any crate: {p}")
 			continue
 
 		crate_name = manifest["package"]["name"]
@@ -85,8 +83,6 @@ def create_prdoc(pr, audience, title, description, patch, bump, force):
 		else:
 			print(f"Skipping unpublished crate: {crate_name}")
 
-	print(f"Modified crates: {modified_crates.keys()}")
-
 	for crate_name in modified_crates.keys():
 		entry = { "name": crate_name }
 
@@ -137,4 +133,4 @@ def main(args):
 
 if __name__ == "__main__":
 	args = setup_parser().parse_args()
-	main(args)
+	main(args)

.github/workflows/checks.yml

@@ -32,8 +32,8 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
       - name: script
         run: |
-          forklift cargo clippy --all-targets --locked --workspace
-          forklift cargo clippy --all-targets --all-features --locked --workspace
+          forklift cargo clippy --all-targets --locked --workspace --quiet
+          forklift cargo clippy --all-targets --all-features --locked --workspace --quiet
   check-try-runtime:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}
     needs: [preflight]
@@ -45,13 +45,13 @@ jobs:
       - uses: actions/checkout@6d193bf28034eafb982f37bd894289fe649468fc # v4.1.7
       - name: script
         run: |
-          forklift cargo check --locked --all --features try-runtime
+          forklift cargo check --locked --all --features try-runtime --quiet
           # this is taken from cumulus
           # Check that parachain-template will compile with `try-runtime` feature flag.
           forklift cargo check --locked -p parachain-template-node --features try-runtime
           # add after https://github.com/paritytech/substrate/pull/14502 is merged
           # experimental code may rely on try-runtime and vice-versa
-          forklift cargo check --locked --all --features try-runtime,experimental
+          forklift cargo check --locked --all --features try-runtime,experimental --quiet
   # check-core-crypto-features works fast without forklift
   check-core-crypto-features:
     runs-on: ${{ needs.preflight.outputs.RUNNER }}

.github/workflows/cmd.yml

@@ -336,7 +336,9 @@ jobs:
 
       - name: Install dependencies for bench
         if: startsWith(steps.get-pr-comment.outputs.group2, 'bench')
-        run: cargo install subweight frame-omni-bencher --locked
+        run: |
+          cargo install subweight --locked
+          cargo install --path substrate/utils/frame/omni-bencher --locked
 
       - name: Run cmd
         id: cmd

.github/workflows/command-prdoc.yml

@@ -28,7 +28,7 @@ on:
           - "Runtime Dev"
           - "Runtime User"
           - "Node Dev"
-          - "Node User"
+          - "Node Operator"
       overwrite:
         type: choice
         description: Overwrite existing PrDoc

.github/workflows/runtimes-matrix.json

@@ -44,6 +44,7 @@
     "package": "asset-hub-rococo-runtime",
     "path": "cumulus/parachains/runtimes/assets/asset-hub-rococo",
     "header": "cumulus/file_header.txt",
+    "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
     "uri": "wss://rococo-asset-hub-rpc.polkadot.io:443",
     "is_relay": false

.github/workflows/tests-linux-stable.yml

@@ -53,7 +53,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: forklift cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet
+        run: forklift cargo nextest run --workspace --features runtime-benchmarks benchmark --locked --cargo-profile testnet --cargo-quiet
 
   test-linux-stable:
     needs: [preflight]
@@ -90,12 +90,13 @@ jobs:
             --locked \
             --release \
             --no-fail-fast \
+            --cargo-quiet \
             --features try-runtime,experimental,riscv,ci-only-tests \
             --partition count:${{ matrix.partition }}
       # run runtime-api tests with `enable-staging-api` feature on the 1st node
       - name: runtime-api tests
         if: ${{ matrix.partition == '1/3' }}
-        run: forklift cargo nextest run -p sp-api-test --features enable-staging-api
+        run: forklift cargo nextest run -p sp-api-test --features enable-staging-api --cargo-quiet
 
   confirm-required-jobs-passed:
     runs-on: ubuntu-latest

.github/workflows/tests-misc.yml

@@ -77,22 +77,22 @@ jobs:
       # but still want to have debug assertions.
       RUSTFLAGS: "-C debug-assertions -D warnings"
       RUST_BACKTRACE: 1
-      WASM_BUILD_NO_COLOR: 1
-      WASM_BUILD_RUSTFLAGS: "-C debug-assertions -D warnings"
+      SKIP_WASM_BUILD: 1
       # Ensure we run the UI tests.
       RUN_UI_TESTS: 1
     steps:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
         run: |
-          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental
-          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental
-          forklift cargo test --locked -q --profile testnet -p xcm-procedural
-          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type
-          forklift cargo test --locked -q --profile testnet -p sp-api-test
+          cargo version
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,no-metadata-docs,try-runtime,experimental ui
+          forklift cargo test --locked -q --profile testnet -p frame-support-test --features=frame-feature-testing,frame-feature-testing-2,no-metadata-docs,try-runtime,experimental ui
+          forklift cargo test --locked -q --profile testnet -p xcm-procedural ui
+          forklift cargo test --locked -q --profile testnet -p frame-election-provider-solution-type ui
+          forklift cargo test --locked -q --profile testnet -p sp-api-test ui
           # There is multiple version of sp-runtime-interface in the repo. So we point to the manifest.
-          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml
+          forklift cargo test --locked -q --profile testnet --manifest-path substrate/primitives/runtime-interface/Cargo.toml ui
 
   test-deterministic-wasm:
     timeout-minutes: 20

.github/workflows/tests.yml

@@ -32,7 +32,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
+        run: forklift cargo run --locked --release -p staging-node-cli --bin substrate-node --features runtime-benchmarks --quiet -- benchmark pallet --chain dev --pallet "*" --extrinsic "*" --steps 2 --repeat 1 --quiet
 
   # cf https://github.com/paritytech/polkadot-sdk/issues/1652
   test-syscalls:
@@ -51,7 +51,7 @@ jobs:
       - name: script
         id: test
         run: |
-          forklift cargo build --locked --profile production --target x86_64-unknown-linux-musl --bin polkadot-execute-worker --bin polkadot-prepare-worker
+          forklift cargo build --locked --profile production --target x86_64-unknown-linux-musl --bin polkadot-execute-worker --bin polkadot-prepare-worker --quiet
           cd polkadot/scripts/list-syscalls
           ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-execute-worker --only-used-syscalls | diff -u execute-worker-syscalls -
           ./list-syscalls.rb ../../../target/x86_64-unknown-linux-musl/production/polkadot-prepare-worker --only-used-syscalls | diff -u prepare-worker-syscalls -
@@ -73,4 +73,4 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: script
-        run: forklift cargo check --all --benches
+        run: forklift cargo check --all --benches --quiet

.gitlab-ci.yml

@@ -21,7 +21,8 @@ workflow:
     - if: $CI_COMMIT_BRANCH
 
 variables:
-  CI_IMAGE: !reference [.ci-unified, variables, CI_IMAGE]
+  # CI_IMAGE: !reference [ .ci-unified, variables, CI_IMAGE ]
+  CI_IMAGE: "docker.io/paritytech/ci-unified:bullseye-1.81.0-2024-09-11-v202409111034"
   # BUILDAH_IMAGE is defined in group variables
   BUILDAH_COMMAND: "buildah --storage-driver overlay2"
   RELENG_SCRIPTS_BRANCH: "master"

.gitlab/pipeline/zombienet/polkadot.yml

@@ -108,7 +108,7 @@ zombienet-polkadot-functional-0004-parachains-disputes-garbage-candidate:
       --local-dir="${LOCAL_DIR}/functional"
       --test="0004-parachains-garbage-candidate.zndsl"
 
-zombienet-polkadot-functional-0005-parachains-disputes-past-session:
+.zombienet-polkadot-functional-0005-parachains-disputes-past-session:
   extends:
     - .zombienet-polkadot-common
   script:
@@ -351,7 +351,7 @@ zombienet-polkadot-malus-0001-dispute-valid:
       --local-dir="${LOCAL_DIR}/integrationtests"
       --test="0001-dispute-valid-block.zndsl"
 
-zombienet-polkadot-coretime-revenue:
+.zombienet-polkadot-coretime-revenue:
   extends:
     - .zombienet-polkadot-common
   needs:

Cargo.toml

@@ -552,7 +552,7 @@ default-members = [
 [workspace.lints.rust]
 suspicious_double_ref_op = { level = "allow", priority = 2 }
 # `substrate_runtime` is a common `cfg` condition name used in the repo.
-unexpected_cfgs = { level = "warn", check-cfg = ['cfg(substrate_runtime)'] }
+unexpected_cfgs = { level = "warn", check-cfg = ['cfg(build_opt_level, values("3"))', 'cfg(build_profile, values("debug", "release"))', 'cfg(enable_alloc_error_handler)', 'cfg(fuzzing)', 'cfg(substrate_runtime)'] }
 
 [workspace.lints.clippy]
 all = { level = "allow", priority = 0 }

bridges/modules/grandpa/src/lib.rs

@@ -728,15 +728,13 @@ pub mod pallet {
 			init_params;
 		let authority_set_length = authority_list.len();
 		let authority_set = StoredAuthoritySet::<T, I>::try_new(authority_list, set_id)
-			.map_err(|e| {
+			.inspect_err(|_| {
 				log::error!(
 					target: LOG_TARGET,
 					"Failed to initialize bridge. Number of authorities in the set {} is larger than the configured value {}",
 					authority_set_length,
 					T::BridgedChain::MAX_AUTHORITIES_COUNT,
 				);
-
-				e
 			})?;
 		let initial_hash = header.hash();
 

bridges/modules/xcm-bridge-hub/src/lib.rs

@@ -415,15 +415,14 @@ pub mod pallet {
 				bridge.deposit,
 				Precision::BestEffort,
 			)
-			.map_err(|e| {
+			.inspect_err(|e| {
 				// we can't do anything here - looks like funds have been (partially) unreserved
 				// before by someone else. Let's not fail, though - it'll be worse for the caller
 				log::error!(
 					target: LOG_TARGET,
 					"Failed to unreserve during the bridge {:?} closure with error: {e:?}",
 					locations.bridge_id(),
 				);
-				e
 			})
 			.ok()
 			.unwrap_or(BalanceOf::<ThisChainOf<T, I>>::zero());
@@ -1456,25 +1455,26 @@ mod tests {
 		let lane_id = TestLaneIdType::try_new(1, 2).unwrap();
 		let lane_id_mismatch = TestLaneIdType::try_new(3, 4).unwrap();
 
-		let test_bridge_state = |id,
-		                         bridge,
-		                         (lane_id, bridge_id),
-		                         (inbound_lane_id, outbound_lane_id),
-		                         expected_error: Option<TryRuntimeError>| {
-			Bridges::<TestRuntime, ()>::insert(id, bridge);
-			LaneToBridge::<TestRuntime, ()>::insert(lane_id, bridge_id);
-
-			let lanes_manager = LanesManagerOf::<TestRuntime, ()>::new();
-			lanes_manager.create_inbound_lane(inbound_lane_id).unwrap();
-			lanes_manager.create_outbound_lane(outbound_lane_id).unwrap();
+		let test_bridge_state =
+			|id,
+			 bridge,
+			 (lane_id, bridge_id),
+			 (inbound_lane_id, outbound_lane_id),
+			 expected_error: Option<TryRuntimeError>| {
+				Bridges::<TestRuntime, ()>::insert(id, bridge);
+				LaneToBridge::<TestRuntime, ()>::insert(lane_id, bridge_id);
 
-			let result = XcmOverBridge::do_try_state();
-			if let Some(e) = expected_error {
-				assert_err!(result, e);
-			} else {
-				assert_ok!(result);
-			}
-		};
+				let lanes_manager = LanesManagerOf::<TestRuntime, ()>::new();
+				lanes_manager.create_inbound_lane(inbound_lane_id).unwrap();
+				lanes_manager.create_outbound_lane(outbound_lane_id).unwrap();
+
+				let result = XcmOverBridge::do_try_state();
+				if let Some(e) = expected_error {
+					assert_err!(result, e);
+				} else {
+					assert_ok!(result);
+				}
+			};
 		let cleanup = |bridge_id, lane_ids| {
 			Bridges::<TestRuntime, ()>::remove(bridge_id);
 			for lane_id in lane_ids {

bridges/primitives/header-chain/src/justification/mod.rs

@@ -32,7 +32,6 @@ pub use verification::{
 
 use bp_runtime::{BlockNumberOf, Chain, HashOf, HeaderId};
 use codec::{Decode, Encode, MaxEncodedLen};
-use frame_support::RuntimeDebugNoBound;
 use scale_info::TypeInfo;
 use sp_consensus_grandpa::{AuthorityId, AuthoritySignature};
 use sp_runtime::{traits::Header as HeaderT, RuntimeDebug, SaturatedConversion};
@@ -43,7 +42,8 @@ use sp_std::prelude::*;
 ///
 /// This particular proof is used to prove that headers on a bridged chain
 /// (so not our chain) have been finalized correctly.
-#[derive(Encode, Decode, Clone, PartialEq, Eq, TypeInfo, RuntimeDebugNoBound)]
+#[derive(Encode, Decode, Clone, PartialEq, Eq, TypeInfo)]
+#[cfg_attr(feature = "std", derive(Debug))]
 pub struct GrandpaJustification<Header: HeaderT> {
 	/// The round (voting period) this justification is valid for.
 	pub round: u64,
@@ -54,6 +54,17 @@ pub struct GrandpaJustification<Header: HeaderT> {
 	pub votes_ancestries: Vec<Header>,
 }
 
+// A proper Debug impl for no-std is not possible for the `GrandpaJustification` since the `Commit`
+// type only implements Debug that for `std` here:
+// https://github.com/paritytech/finality-grandpa/blob/8c45a664c05657f0c71057158d3ba555ba7d20de/src/lib.rs#L275
+// so we do a manual impl.
+#[cfg(not(feature = "std"))]
+impl<H: HeaderT> core::fmt::Debug for GrandpaJustification<H> {
+	fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result {
+		write!(f, "GrandpaJustification {{ round: {:?}, commit: <wasm:stripped>, votes_ancestries: {:?} }}", self.round, self.votes_ancestries)
+	}
+}
+
 impl<H: HeaderT> GrandpaJustification<H> {
 	/// Returns reasonable size of justification using constants from the provided chain.
 	///

bridges/primitives/messages/src/lane.rs

@@ -71,7 +71,7 @@ impl LaneIdType for LegacyLaneId {
 	}
 }
 
-#[cfg(any(feature = "std", feature = "runtime-benchmarks", test))]
+#[cfg(feature = "std")]
 impl TryFrom<Vec<u8>> for LegacyLaneId {
 	type Error = ();
 
@@ -140,7 +140,7 @@ impl HashedLaneId {
 	/// There's no `From<H256>` implementation for the `LaneId`, because using this conversion
 	/// in a wrong way (i.e. computing hash of endpoints manually) may lead to issues. So we
 	/// want the call to be explicit.
-	#[cfg(any(feature = "std", feature = "runtime-benchmarks", test))]
+	#[cfg(feature = "std")]
 	pub const fn from_inner(inner: H256) -> Self {
 		Self(inner)
 	}
@@ -184,7 +184,7 @@ impl LaneIdType for HashedLaneId {
 	}
 }
 
-#[cfg(any(feature = "std", feature = "runtime-benchmarks", test))]
+#[cfg(feature = "std")]
 impl TryFrom<Vec<u8>> for HashedLaneId {
 	type Error = ();
 

bridges/relays/lib-substrate-relay/src/cli/relay_headers_and_messages/mod.rs

@@ -325,7 +325,9 @@ where
 				_,
 				Self::Left,
 				MessagesLaneIdOf<Self::R2L>,
-			>(common.right.client.clone(), &common.metrics_params, &common.right.accounts, &lanes)
+			>(
+				common.right.client.clone(), &common.metrics_params, &common.right.accounts, &lanes
+			)
 			.await?;
 		}
 

bridges/relays/lib-substrate-relay/src/messages/metrics.rs

@@ -53,9 +53,8 @@ where
 	let token_decimals = client
 		.token_decimals()
 		.await?
-		.map(|token_decimals| {
+		.inspect(|token_decimals| {
 			log::info!(target: "bridge", "Read `tokenDecimals` for {}: {}", C::NAME, token_decimals);
-			token_decimals
 		})
 		.unwrap_or_else(|| {
 			// turns out it is normal not to have this property - e.g. when polkadot binary is

bridges/relays/lib-substrate-relay/src/on_demand/parachains.rs

@@ -664,7 +664,8 @@ impl<'a, P: SubstrateParachainsPipeline, SourceRelayClnt, TargetClnt>
 	for (
 		&'a OnDemandParachainsRelay<P, SourceRelayClnt, TargetClnt>,
 		&'a ParachainsSource<P, SourceRelayClnt>,
-	) where
+	)
+where
 	SourceRelayClnt: Client<P::SourceRelayChain>,
 	TargetClnt: Client<P::TargetChain>,
 {