Merge branch 'master' into rq/expose-client

.github/workflows/runtimes-matrix.json

@@ -52,7 +52,7 @@
   {
     "name": "bridge-hub-rococo",
     "package": "bridge-hub-rococo-runtime",
-    "path": "cumulus/parachains/runtimes/bridges/bridge-hub-rococo",
+    "path": "cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",
@@ -62,7 +62,7 @@
   {
     "name": "bridge-hub-westend",
     "package": "bridge-hub-rococo-runtime",
-    "path": "cumulus/parachains/runtimes/bridges/bridge-hub-westend",
+    "path": "cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend",
     "header": "cumulus/file_header.txt",
     "bench_features": "runtime-benchmarks",
     "template": "cumulus/templates/xcm-bench-template.hbs",

Cargo.toml

@@ -1200,6 +1200,7 @@ sha1 = { version = "0.10.6" }
 sha2 = { version = "0.10.7", default-features = false }
 sha3 = { version = "0.10.0", default-features = false }
 shell-runtime = { path = "cumulus/parachains/runtimes/starters/shell" }
+shlex = { version = "1.3.0" }
 slot-range-helper = { path = "polkadot/runtime/common/slot_range_helper", default-features = false }
 slotmap = { version = "1.0" }
 smallvec = { version = "1.11.0", default-features = false }

prdoc/pr_5756.prdoc

@@ -0,0 +1,16 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: Improve APIs for Tries in Runtime
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      This PR introduces a trait `ProvingTrie` which has all the function you need to use tries in the runtime.
+      This trait includes the ability to create, query, and prove data in a trie. Another trait `ProofToHashes`
+      allows developers to express the computational complexity of proof verification using the proof data.
+crates:
+  - name: sp-runtime
+    bump: major
+  - name: frame-support
+    bump: major

prdoc/pr_5811.prdoc

@@ -0,0 +1,13 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: "Improve `import_notification_stream` documentation" 
+
+doc:
+  - audience: Node Dev
+    description: |
+      "Updates the doc comment on the `import_notification_stream` to make its behaviour clearer. Now it specifically states that this notification stream is fired on every import notification after the initial sync, and only when there are re-orgs in the initial sync."
+
+crates:
+  - name: sc-client-api
+    bump: patch

prdoc/pr_5838.prdoc

@@ -0,0 +1,20 @@
+# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0
+# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json
+
+title: enable wasm builder diagnostics propagation
+
+doc:
+  - audience: Runtime Dev
+    description: |
+      `substrate-wasm-builder` is used as a build dependency by crates that implement FRAME runtimes.
+      Errors that occur in these crates can not be detected by IDEs that use rust-analyzer as a language
+      server because rust-analyzer needs the errors to be reported as diagnostic message in json format to
+      be able to publish them to language server clients. This PR adds `WASM_BUILD_CARGO_ARGS` environment
+      variable, which can hold a space separated list of args that will be parsed and passed to the `cargo`
+      command that it is used for building against wasm target. It can be used for the stated initial case,
+      but it is also flexible enough to allow passing other arguments or formatting the messages using another
+      available type.
+crates:
+  - name: substrate-wasm-builder 
+    bump: patch
+

substrate/client/api/src/client.rs

@@ -65,9 +65,16 @@ pub trait BlockOf {
 pub trait BlockchainEvents<Block: BlockT> {
 	/// Get block import event stream.
 	///
-	/// Not guaranteed to be fired for every imported block, only fired when the node
-	/// has synced to the tip or there is a re-org. Use `every_import_notification_stream()`
-	/// if you want a notification of every imported block regardless.
+	/// Not guaranteed to be fired for every imported block. Use
+	/// `every_import_notification_stream()` if you want a notification of every imported block
+	/// regardless.
+	///
+	/// The events for this notification stream are emitted:
+	/// - During initial sync process: if there is a re-org while importing blocks. See
+	/// [here](https://github.com/paritytech/substrate/pull/7118#issuecomment-694091901) for the
+	/// rationale behind this.
+	/// - After initial sync process: on every imported block, regardless of whether it is
+	/// the new best block or not, causes a re-org or not.
 	fn import_notification_stream(&self) -> ImportNotifications<Block>;
 
 	/// Get a stream of every imported block.

substrate/frame/support/src/traits/proving.rs

@@ -20,6 +20,10 @@
 use alloc::vec::Vec;
 use codec::{Decode, Encode};
 use sp_core::Hasher;
+use sp_runtime::DispatchError;
+
+// Re-export the `proving_trie` types and traits.
+pub use sp_runtime::proving_trie::*;
 
 /// Something that can verify the existence of some data in a given proof.
 pub trait VerifyExistenceProof {
@@ -31,7 +35,7 @@ pub trait VerifyExistenceProof {
 	/// Verify the given `proof`.
 	///
 	/// Ensures that the `proof` was build for `root` and returns the proved data.
-	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, ()>;
+	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, DispatchError>;
 }
 
 /// Implements [`VerifyExistenceProof`] using a binary merkle tree.
@@ -44,9 +48,9 @@ where
 	type Proof = binary_merkle_tree::MerkleProof<H::Out, Vec<u8>>;
 	type Hash = H::Out;
 
-	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, ()> {
+	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, DispatchError> {
 		if proof.root != *root {
-			return Err(());
+			return Err(TrieError::RootMismatch.into());
 		}
 
 		if binary_merkle_tree::verify_proof::<H, _, _>(
@@ -58,13 +62,25 @@ where
 		) {
 			Ok(proof.leaf)
 		} else {
-			Err(())
+			Err(TrieError::IncompleteProof.into())
 		}
 	}
 }
 
+impl<H: Hasher> ProofToHashes for BinaryMerkleTreeProver<H> {
+	type Proof = binary_merkle_tree::MerkleProof<H::Out, Vec<u8>>;
+
+	// This base 2 merkle trie includes a `proof` field which is a `Vec<Hash>`.
+	// The length of this vector tells us the depth of the proof, and how many
+	// hashes we need to calculate.
+	fn proof_to_hashes(proof: &Self::Proof) -> Result<u32, DispatchError> {
+		let depth = proof.proof.len();
+		Ok(depth as u32)
+	}
+}
+
 /// Proof used by [`SixteenPatriciaMerkleTreeProver`] for [`VerifyExistenceProof`].
-#[derive(Encode, Decode)]
+#[derive(Encode, Decode, Clone)]
 pub struct SixteenPatriciaMerkleTreeExistenceProof {
 	/// The key of the value to prove.
 	pub key: Vec<u8>,
@@ -81,21 +97,35 @@ impl<H: Hasher> VerifyExistenceProof for SixteenPatriciaMerkleTreeProver<H> {
 	type Proof = SixteenPatriciaMerkleTreeExistenceProof;
 	type Hash = H::Out;
 
-	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, ()> {
+	fn verify_proof(proof: Self::Proof, root: &Self::Hash) -> Result<Vec<u8>, DispatchError> {
 		sp_trie::verify_trie_proof::<sp_trie::LayoutV1<H>, _, _, _>(
 			&root,
 			&proof.proof,
 			[&(&proof.key, Some(&proof.value))],
 		)
-		.map_err(drop)
+		.map_err(|err| TrieError::from(err).into())
 		.map(|_| proof.value)
 	}
 }
 
+impl<H: Hasher> ProofToHashes for SixteenPatriciaMerkleTreeProver<H> {
+	type Proof = SixteenPatriciaMerkleTreeExistenceProof;
+
+	// This base 16 trie uses a raw proof of `Vec<Vec<u8>`, where the length of the first `Vec`
+	// is the depth of the trie. We can use this to predict the number of hashes.
+	fn proof_to_hashes(proof: &Self::Proof) -> Result<u32, DispatchError> {
+		let depth = proof.proof.len();
+		Ok(depth as u32)
+	}
+}
+
 #[cfg(test)]
 mod tests {
 	use super::*;
-	use sp_runtime::{proving_trie::BasicProvingTrie, traits::BlakeTwo256};
+	use sp_runtime::{
+		proving_trie::{base16::BasicProvingTrie, ProvingTrie},
+		traits::BlakeTwo256,
+	};
 
 	#[test]
 	fn verify_binary_merkle_tree_prover_works() {
@@ -113,23 +143,87 @@ mod tests {
 
 	#[test]
 	fn verify_sixteen_patricia_merkle_tree_prover_works() {
-		let trie = BasicProvingTrie::<BlakeTwo256, u32, &[u8]>::generate_for(vec![
-			(0u32, &b"hey"[..]),
-			(1u32, &b"yes"[..]),
+		let trie = BasicProvingTrie::<BlakeTwo256, u32, _>::generate_for(vec![
+			(0u32, String::from("hey")),
+			(1u32, String::from("yes")),
 		])
 		.unwrap();
-		let proof = trie.create_single_value_proof(1u32).unwrap();
+		let proof = trie.create_proof(&1u32).unwrap();
+		let structured_proof: Vec<Vec<u8>> = Decode::decode(&mut &proof[..]).unwrap();
 		let root = *trie.root();
 
 		let proof = SixteenPatriciaMerkleTreeExistenceProof {
 			key: 1u32.encode(),
-			value: b"yes"[..].encode(),
-			proof,
+			value: String::from("yes").encode(),
+			proof: structured_proof,
 		};
 
 		assert_eq!(
 			SixteenPatriciaMerkleTreeProver::<BlakeTwo256>::verify_proof(proof, &root).unwrap(),
-			b"yes"[..].encode()
+			String::from("yes").encode()
 		);
 	}
+
+	#[test]
+	fn proof_to_hashes_sixteen() {
+		let mut i: u32 = 1;
+
+		// Compute log base 16 and round up
+		let log16 = |x: u32| -> u32 {
+			let x_f64 = x as f64;
+			let log16_x = (x_f64.ln() / 16_f64.ln()).ceil();
+			log16_x as u32
+		};
+
+		while i < 10_000_000 {
+			let trie = BasicProvingTrie::<BlakeTwo256, u32, _>::generate_for(
+				(0..i).map(|i| (i, u128::from(i))),
+			)
+			.unwrap();
+			let proof = trie.create_proof(&0).unwrap();
+			let structured_proof: Vec<Vec<u8>> = Decode::decode(&mut &proof[..]).unwrap();
+			let root = *trie.root();
+
+			let proof = SixteenPatriciaMerkleTreeExistenceProof {
+				key: 0u32.encode(),
+				value: 0u128.encode(),
+				proof: structured_proof,
+			};
+			let hashes =
+				SixteenPatriciaMerkleTreeProver::<BlakeTwo256>::proof_to_hashes(&proof).unwrap();
+			let log16 = log16(i).max(1);
+			assert_eq!(hashes, log16);
+
+			assert_eq!(
+				SixteenPatriciaMerkleTreeProver::<BlakeTwo256>::verify_proof(proof.clone(), &root)
+					.unwrap(),
+				proof.value
+			);
+
+			i = i * 10;
+		}
+	}
+
+	#[test]
+	fn proof_to_hashes_binary() {
+		let mut i: u32 = 1;
+		while i < 10_000_000 {
+			let proof = binary_merkle_tree::merkle_proof::<BlakeTwo256, _, _>(
+				(0..i).map(|i| u128::from(i).encode()),
+				0,
+			);
+			let root = proof.root;
+
+			let hashes = BinaryMerkleTreeProver::<BlakeTwo256>::proof_to_hashes(&proof).unwrap();
+			let log2 = (i as f64).log2().ceil() as u32;
+			assert_eq!(hashes, log2);
+
+			assert_eq!(
+				BinaryMerkleTreeProver::<BlakeTwo256>::verify_proof(proof, &root).unwrap(),
+				0u128.encode()
+			);
+
+			i = i * 10;
+		}
+	}
 }

substrate/primitives/runtime/Cargo.toml

@@ -36,6 +36,7 @@ sp-trie = { workspace = true }
 sp-weights = { workspace = true }
 docify = { workspace = true }
 tracing = { workspace = true, features = ["log"], default-features = false }
+binary-merkle-tree = { workspace = true }
 
 simple-mermaid = { version = "0.1.1", optional = true }
 
@@ -53,6 +54,7 @@ runtime-benchmarks = []
 try-runtime = []
 default = ["std"]
 std = [
+	"binary-merkle-tree/std",
 	"codec/std",
 	"either/use_std",
 	"hash256-std-hasher/std",