diff --git a/Cargo.lock b/Cargo.lock index aa2ce3aa38..a6d0ad8b9a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2488,6 +2488,7 @@ dependencies = [ "sp-core", "sp-domains", "sp-executive", + "sp-externalities", "sp-inherents", "sp-keyring", "sp-messenger", @@ -2524,10 +2525,10 @@ dependencies = [ "sc-client-api", "sc-state-db", "sc-utils", - "scale-info", "sp-api", "sp-blockchain", "sp-consensus", + "sp-core", "sp-domains", "sp-messenger", "sp-mmr-primitives", @@ -2550,7 +2551,10 @@ dependencies = [ "futures-timer", "pallet-balances", "pallet-domains", + "pallet-messenger", + "pallet-sudo", "pallet-timestamp", + "pallet-transporter", "parity-scale-codec", "parking_lot 0.12.1", "sc-cli", @@ -2572,6 +2576,7 @@ dependencies = [ "sp-inherents", "sp-keystore", "sp-messenger", + "sp-mmr-primitives", "sp-runtime", "sp-state-machine", "sp-transaction-pool", @@ -12028,6 +12033,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "subspace-test-primitives" +version = "0.1.0" +dependencies = [ + "parity-scale-codec", + "sp-api", + "sp-messenger", + "subspace-runtime-primitives", +] + [[package]] name = "subspace-test-runtime" version = "0.1.0" @@ -12076,6 +12091,7 @@ dependencies = [ "static_assertions", "subspace-core-primitives", "subspace-runtime-primitives", + "subspace-test-primitives", "substrate-wasm-builder", ] @@ -12085,9 +12101,11 @@ version = "0.1.0" dependencies = [ "async-trait", "cross-domain-message-gossip", + "domain-client-message-relayer", "domain-runtime-primitives", "futures", "jsonrpsee", + "mmr-gadget", "pallet-domains", "parity-scale-codec", "parking_lot 0.12.1", @@ -12117,6 +12135,7 @@ dependencies = [ "sp-inherents", "sp-keyring", "sp-messenger", + "sp-messenger-host-functions", "sp-mmr-primitives", "sp-runtime", "sp-subspace-mmr", @@ -12125,6 +12144,7 @@ dependencies = [ "subspace-runtime-primitives", "subspace-service", "subspace-test-client", + "subspace-test-primitives", "subspace-test-runtime", "tokio", "tracing", diff --git a/crates/pallet-domains/src/lib.rs b/crates/pallet-domains/src/lib.rs index bfe9a1bbd2..17fbe9d8d1 100644 --- a/crates/pallet-domains/src/lib.rs +++ b/crates/pallet-domains/src/lib.rs @@ -1986,6 +1986,13 @@ impl Pallet { .unwrap_or_default() } + pub fn latest_confirmed_domain_block( + domain_id: DomainId, + ) -> Option<(DomainBlockNumberFor, T::DomainHash)> { + LatestConfirmedDomainBlock::::get(domain_id) + .map(|block| (block.block_number, block.block_hash)) + } + /// Returns the domain block limit of the given domain. pub fn domain_block_limit(domain_id: DomainId) -> Option { DomainRegistry::::get(domain_id).map(|domain_obj| DomainBlockLimit { diff --git a/crates/pallet-domains/src/tests.rs b/crates/pallet-domains/src/tests.rs index 2f4dc1638e..4cf09cea49 100644 --- a/crates/pallet-domains/src/tests.rs +++ b/crates/pallet-domains/src/tests.rs @@ -353,6 +353,7 @@ pub(crate) struct MockDomainFraudProofExtension { bundle_slot_probability: (u64, u64), operator_stake: Balance, maybe_illegal_extrinsic_index: Option, + is_valid_xdm: Option, } impl FraudProofHostFunctions for MockDomainFraudProofExtension { @@ -432,6 +433,9 @@ impl FraudProofHostFunctions for MockDomainFraudProofExtension { FraudProofVerificationInfoRequest::StorageKey { .. } => { FraudProofVerificationInfoResponse::StorageKey(None) } + FraudProofVerificationInfoRequest::XDMValidationCheck { .. } => { + FraudProofVerificationInfoResponse::XDMValidationCheck(self.is_valid_xdm) + } }; Some(response) @@ -1016,6 +1020,7 @@ fn test_invalid_domain_extrinsic_root_proof() { operator_stake: 10 * SSC, bundle_slot_probability: (0, 0), maybe_illegal_extrinsic_index: None, + is_valid_xdm: None, })); ext.register_extension(fraud_proof_ext); @@ -1097,6 +1102,7 @@ fn test_true_invalid_bundles_inherent_extrinsic_proof() { operator_stake: 10 * SSC, bundle_slot_probability: (0, 0), maybe_illegal_extrinsic_index: None, + is_valid_xdm: None, })); ext.register_extension(fraud_proof_ext); @@ -1164,6 +1170,7 @@ fn test_false_invalid_bundles_inherent_extrinsic_proof() { operator_stake: 10 * SSC, bundle_slot_probability: (0, 0), maybe_illegal_extrinsic_index: None, + is_valid_xdm: None, })); ext.register_extension(fraud_proof_ext); diff --git a/crates/sp-domains-fraud-proof/src/fraud_proof.rs b/crates/sp-domains-fraud-proof/src/fraud_proof.rs index 2bfebba371..85f9679ab2 100644 --- a/crates/sp-domains-fraud-proof/src/fraud_proof.rs +++ b/crates/sp-domains-fraud-proof/src/fraud_proof.rs @@ -386,6 +386,9 @@ pub enum VerificationError { error("Failed to check if a given extrinsic is inherent or not") )] FailedToCheckInherentExtrinsic, + /// Failed to check if a given extrinsic is inherent or not. + #[cfg_attr(feature = "thiserror", error("Failed to validate given XDM"))] + FailedToValidateXDM, /// Failed to check if a given extrinsic is decodable or not. #[cfg_attr( feature = "thiserror", diff --git a/crates/sp-domains-fraud-proof/src/host_functions.rs b/crates/sp-domains-fraud-proof/src/host_functions.rs index cbf39cc5dd..b4f7c11c1d 100644 --- a/crates/sp-domains-fraud-proof/src/host_functions.rs +++ b/crates/sp-domains-fraud-proof/src/host_functions.rs @@ -85,33 +85,33 @@ impl FraudProofExtension { } /// Trait Impl to query and verify Domains Fraud proof. -pub struct FraudProofHostFunctionsImpl { +pub struct FraudProofHostFunctionsImpl { consensus_client: Arc, executor: Arc, - domain_extensions_factory: Box>, + domain_extensions_factory_creator: EFC, _phantom: PhantomData<(Block, DomainBlock)>, } -impl - FraudProofHostFunctionsImpl +impl + FraudProofHostFunctionsImpl { pub fn new( consensus_client: Arc, executor: Arc, - domain_extensions_factory: Box>, + domain_extensions_factory_creator: EFC, ) -> Self { FraudProofHostFunctionsImpl { consensus_client, executor, - domain_extensions_factory, + domain_extensions_factory_creator, _phantom: Default::default(), } } } // TODO: Revisit the host function implementation once we decide best strategy to structure them. -impl - FraudProofHostFunctionsImpl +impl + FraudProofHostFunctionsImpl where Block: BlockT, Block::Hash: From, @@ -120,6 +120,7 @@ where Client: BlockBackend + HeaderBackend + ProvideRuntimeApi, Client::Api: DomainsApi + BundleProducerElectionApi, Executor: CodeExecutor + RuntimeVersionOf, + EFC: Fn(Arc, Arc) -> Box> + Send + Sync, { fn get_block_randomness(&self, consensus_block_hash: H256) -> Option { let runtime_api = self.consensus_client.runtime_api(); @@ -297,6 +298,36 @@ where .ok() } + fn is_valid_xdm( + &self, + consensus_block_hash: H256, + domain_id: DomainId, + opaque_extrinsic: OpaqueExtrinsic, + ) -> Option { + let runtime_code = self.get_domain_runtime_code(consensus_block_hash, domain_id)?; + let mut domain_stateless_runtime = + StatelessRuntime::::new(self.executor.clone(), runtime_code.into()); + let extension_factory = (self.domain_extensions_factory_creator)( + self.consensus_client.clone(), + self.executor.clone(), + ); + domain_stateless_runtime.set_extension_factory(extension_factory); + + let consensus_api = self.consensus_client.runtime_api(); + let domain_initial_state = consensus_api + .domain_instance_data(consensus_block_hash.into(), domain_id) + .expect("Runtime Api must not fail. This is unrecoverable error")? + .0 + .raw_genesis + .into_storage(); + domain_stateless_runtime.set_storage(domain_initial_state); + + let encoded_extrinsic = opaque_extrinsic.encode(); + domain_stateless_runtime + .is_valid_xdm(encoded_extrinsic) + .expect("Runtime api must not fail. This is an unrecoverable error") + } + fn is_decodable_extrinsic( &self, consensus_block_hash: H256, @@ -389,8 +420,8 @@ where } } -impl FraudProofHostFunctions - for FraudProofHostFunctionsImpl +impl FraudProofHostFunctions + for FraudProofHostFunctionsImpl where Block: BlockT, Block::Hash: From, @@ -400,6 +431,7 @@ where Client: BlockBackend + HeaderBackend + ProvideRuntimeApi, Client::Api: DomainsApi + BundleProducerElectionApi, Executor: CodeExecutor + RuntimeVersionOf, + EFC: Fn(Arc, Arc) -> Box> + Send + Sync, { fn get_fraud_proof_verification_info( &self, @@ -515,6 +547,12 @@ where self.storage_key(consensus_block_hash, domain_id, req), )) } + FraudProofVerificationInfoRequest::XDMValidationCheck { + domain_id, + opaque_extrinsic, + } => Some(FraudProofVerificationInfoResponse::XDMValidationCheck( + self.is_valid_xdm(consensus_block_hash, domain_id, opaque_extrinsic), + )), } } @@ -574,9 +612,11 @@ where }; let (domain_block_number, domain_block_hash) = domain_block_id; - let mut domain_extensions = self - .domain_extensions_factory - .extensions_for(domain_block_hash.into(), domain_block_number.into()); + let mut domain_extensions = (self.domain_extensions_factory_creator)( + self.consensus_client.clone(), + self.executor.clone(), + ) + .extensions_for(domain_block_hash.into(), domain_block_number.into()); execution_proof_check::<::Hashing, _>( pre_state_root.into(), diff --git a/crates/sp-domains-fraud-proof/src/lib.rs b/crates/sp-domains-fraud-proof/src/lib.rs index b118b01698..832d23963f 100644 --- a/crates/sp-domains-fraud-proof/src/lib.rs +++ b/crates/sp-domains-fraud-proof/src/lib.rs @@ -128,6 +128,12 @@ pub enum FraudProofVerificationInfoRequest { /// Extrinsic for which we need to if it is decodable or not. opaque_extrinsic: OpaqueExtrinsic, }, + /// Request to check if the XDM is valid + XDMValidationCheck { + domain_id: DomainId, + /// Encoded XDM extrinsic that needs to be validated. + opaque_extrinsic: OpaqueExtrinsic, + }, /// Request to get Domain election params. DomainElectionParams { domain_id: DomainId }, /// Request to get Operator stake. @@ -186,6 +192,9 @@ pub enum FraudProofVerificationInfoResponse { TxRangeCheck(bool), /// If the particular extrinsic provided is either inherent or not. InherentExtrinsicCheck(bool), + /// If the particular xdm extrinsic is valid or not. + /// Returns None if extrinsic is not an XDM + XDMValidationCheck(Option), /// If the domain extrinsic is decodable or not. ExtrinsicDecodableCheck(bool), /// Domain's total stake at a given Consensus hash. @@ -264,6 +273,13 @@ impl FraudProofVerificationInfoResponse { } } + pub fn into_xdm_validation_check(self) -> Option { + match self { + FraudProofVerificationInfoResponse::XDMValidationCheck(maybe_valid) => maybe_valid, + _ => None, + } + } + pub fn into_extrinsic_decodable_check(self) -> Option { match self { FraudProofVerificationInfoResponse::ExtrinsicDecodableCheck(is_decodable) => { diff --git a/crates/sp-domains-fraud-proof/src/verification.rs b/crates/sp-domains-fraud-proof/src/verification.rs index 0bf14e34e5..bdbf1a8e62 100644 --- a/crates/sp-domains-fraud-proof/src/verification.rs +++ b/crates/sp-domains-fraud-proof/src/verification.rs @@ -633,9 +633,32 @@ where } Ok(()) } + InvalidBundleType::InvalidXDM(extrinsic_index) => { + let extrinsic = get_extrinsic_from_proof::( + *extrinsic_index, + invalid_bundle_entry.extrinsics_root, + invalid_bundles_fraud_proof.proof_data.clone(), + )?; - // TODO: implement the other invalid bundle types - _ => Err(VerificationError::InvalidProof), + let is_valid_xdm = get_fraud_proof_verification_info( + H256::from_slice(bad_receipt.consensus_block_hash.as_ref()), + FraudProofVerificationInfoRequest::XDMValidationCheck { + domain_id: invalid_bundles_fraud_proof.domain_id, + opaque_extrinsic: extrinsic, + }, + ) + .and_then(FraudProofVerificationInfoResponse::into_xdm_validation_check) + .ok_or(VerificationError::FailedToValidateXDM)?; + + // Proof to be considered valid only, + // If it is true invalid fraud proof then extrinsic must be an invalid xdm and + // If it is false invalid fraud proof then extrinsic must be a valid xdm + if is_valid_xdm != invalid_bundles_fraud_proof.is_true_invalid_fraud_proof { + Ok(()) + } else { + Err(VerificationError::InvalidProof) + } + } } } diff --git a/crates/sp-domains/src/lib.rs b/crates/sp-domains/src/lib.rs index 824b08abd3..b8e4af3bea 100644 --- a/crates/sp-domains/src/lib.rs +++ b/crates/sp-domains/src/lib.rs @@ -1007,8 +1007,8 @@ impl InvalidBundleType { Self::UndecodableTx(_) => 1, Self::OutOfRangeTx(_) => 2, Self::InherentExtrinsic(_) => 3, - Self::IllegalTx(_) => 4, - Self::InvalidXDM(_) => 5, + Self::InvalidXDM(_) => 4, + Self::IllegalTx(_) => 5, } } @@ -1248,10 +1248,13 @@ sp_api::decl_runtime_apis! { /// Returns the execution receipt hash of the given domain and domain block number fn receipt_hash(domain_id: DomainId, domain_number: HeaderNumberFor) -> Option>; - /// Reture the consensus chain byte fee that will used to charge the domain transaction for consensus + /// Return the consensus chain byte fee that will used to charge the domain transaction for consensus /// chain storage fee fn consensus_chain_byte_fee() -> Balance; + /// Returns the latest confirmed domain block number and hash + fn latest_confirmed_domain_block(domain_id: DomainId) -> Option<(HeaderNumberFor, HeaderHashFor)>; + /// Return if the receipt is exist and pending to prune fn is_bad_er_pending_to_prune(domain_id: DomainId, receipt_hash: HeaderHashFor) -> bool; } diff --git a/crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs b/crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs index 2584a54788..ca2fc3c653 100644 --- a/crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs +++ b/crates/subspace-malicious-operator/src/bin/subspace-malicious-operator.rs @@ -119,7 +119,7 @@ fn main() -> Result<(), Error> { })? .unwrap_or_default(); - let consensus_state_pruning_mode = consensus_chain_config + let consensus_state_pruning = consensus_chain_config .state_pruning .clone() .unwrap_or_default(); @@ -273,7 +273,7 @@ fn main() -> Result<(), Error> { let relayer_worker = domain_client_message_relayer::worker::relay_consensus_chain_messages( consensus_chain_node.client.clone(), - consensus_state_pruning_mode, + consensus_state_pruning.clone(), consensus_chain_node.sync_service.clone(), xdm_gossip_worker_builder.gossip_msg_sink(), ); @@ -338,6 +338,7 @@ fn main() -> Result<(), Error> { consensus_sync_service: consensus_chain_node.sync_service.clone(), domain_message_receiver, gossip_message_sink: xdm_gossip_worker_builder.gossip_msg_sink(), + consensus_state_pruning, }; consensus_chain_node diff --git a/crates/subspace-malicious-operator/src/malicious_domain_instance_starter.rs b/crates/subspace-malicious-operator/src/malicious_domain_instance_starter.rs index f0d49d65b7..24f5d173f3 100644 --- a/crates/subspace-malicious-operator/src/malicious_domain_instance_starter.rs +++ b/crates/subspace-malicious-operator/src/malicious_domain_instance_starter.rs @@ -13,6 +13,7 @@ use sc_consensus_subspace::block_import::BlockImportingNotification; use sc_consensus_subspace::notification::SubspaceNotificationStream; use sc_consensus_subspace::slot_worker::NewSlotNotification; use sc_network::NetworkPeers; +use sc_service::PruningMode; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::traits::SpawnEssentialNamed; @@ -40,6 +41,7 @@ pub struct DomainInstanceStarter { pub domain_message_receiver: TracingUnboundedReceiver, pub gossip_message_sink: TracingUnboundedSender, pub consensus_network: Arc, + pub consensus_state_pruning: PruningMode, } impl DomainInstanceStarter @@ -74,6 +76,7 @@ where domain_message_receiver, gossip_message_sink, consensus_network, + consensus_state_pruning, } = self; let domain_id = domain_cli.domain_id.into(); @@ -151,6 +154,7 @@ where skip_out_of_order_slot: false, // Always set it to `None` to not running the normal bundle producer maybe_operator_id: None, + consensus_state_pruning, }; let mut domain_node = domain_service::new_full::< diff --git a/crates/subspace-node/src/commands/run.rs b/crates/subspace-node/src/commands/run.rs index d02ae9c1f2..fc3b84abe7 100644 --- a/crates/subspace-node/src/commands/run.rs +++ b/crates/subspace-node/src/commands/run.rs @@ -110,11 +110,11 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { info!("🏷 Node name: {}", subspace_configuration.network.node_name); info!("💾 Node path: {}", base_path.display()); + let consensus_state_pruning = subspace_configuration + .state_pruning + .clone() + .unwrap_or_default(); let mut task_manager = { - let consensus_state_pruning_mode = subspace_configuration - .state_pruning - .clone() - .unwrap_or_default(); let consensus_chain_node = { let span = info_span!("Consensus"); let _enter = span.enter(); @@ -178,7 +178,7 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { Box::pin( domain_client_message_relayer::worker::relay_consensus_chain_messages( consensus_chain_node.client.clone(), - consensus_state_pruning_mode, + consensus_state_pruning.clone(), consensus_chain_node.sync_service.clone(), xdm_gossip_worker_builder.gossip_msg_sink(), ), @@ -242,6 +242,7 @@ pub async fn run(run_options: RunOptions) -> Result<(), Error> { consensus_network_sync_oracle: consensus_chain_node.sync_service, domain_message_receiver, gossip_message_sink, + consensus_state_pruning, }; consensus_chain_node diff --git a/crates/subspace-node/src/commands/run/domain.rs b/crates/subspace-node/src/commands/run/domain.rs index 76584418f4..76c1d6380b 100644 --- a/crates/subspace-node/src/commands/run/domain.rs +++ b/crates/subspace-node/src/commands/run/domain.rs @@ -25,7 +25,7 @@ use sc_informant::OutputFormat; use sc_network::config::{MultiaddrWithPeerId, NonReservedPeerMode, SetConfig, TransportConfig}; use sc_network::NetworkPeers; use sc_service::config::KeystoreConfig; -use sc_service::Configuration; +use sc_service::{Configuration, PruningMode}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender}; use sp_core::crypto::SecretString; @@ -384,6 +384,7 @@ pub(super) struct DomainStartOptions { pub(super) domain_message_receiver: TracingUnboundedReceiver, pub(super) gossip_message_sink: TracingUnboundedSender, + pub(super) consensus_state_pruning: PruningMode, } pub(super) async fn run_evm_domain( @@ -426,6 +427,7 @@ where consensus_network_sync_oracle, domain_message_receiver, gossip_message_sink, + consensus_state_pruning, } = domain_start_options; let block_importing_notification_stream = block_importing_notification_stream.subscribe().then( @@ -485,6 +487,7 @@ where skip_empty_bundle_production: true, skip_out_of_order_slot: false, maybe_operator_id: operator_id, + consensus_state_pruning, }; let mut domain_node = domain_service::new_full::< diff --git a/crates/subspace-runtime/src/lib.rs b/crates/subspace-runtime/src/lib.rs index b055cc6901..fd87745f64 100644 --- a/crates/subspace-runtime/src/lib.rs +++ b/crates/subspace-runtime/src/lib.rs @@ -65,7 +65,9 @@ use sp_domains::{ }; use sp_domains_fraud_proof::fraud_proof::FraudProof; use sp_messenger::endpoint::{Endpoint, EndpointHandler as EndpointHandlerT, EndpointId}; -use sp_messenger::messages::{BlockMessagesWithStorageKey, ChainId, CrossDomainMessage, MessageId}; +use sp_messenger::messages::{ + BlockMessagesWithStorageKey, ChainId, CrossDomainMessage, MessageId, MessageKey, +}; use sp_messenger_host_functions::{get_storage_key, StorageKeyRequest}; use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof}; use sp_runtime::traits::{ @@ -499,17 +501,17 @@ impl sp_messenger::StorageKeys for StorageKeys { Some(Domains::confirmed_domain_block_storage_key(domain_id)) } - fn outbox_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn outbox_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::OutboxStorageKey { chain_id, - message_id, + message_key, }) } - fn inbox_responses_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn inbox_responses_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::InboxResponseStorageKey { chain_id, - message_id, + message_key, }) } } @@ -1111,6 +1113,10 @@ impl_runtime_apis! { DOMAIN_STORAGE_FEE_MULTIPLIER * TransactionFees::transaction_byte_fee() } + fn latest_confirmed_domain_block(domain_id: DomainId) -> Option<(DomainNumber, DomainHash)>{ + Domains::latest_confirmed_domain_block(domain_id) + } + fn is_bad_er_pending_to_prune(domain_id: DomainId, receipt_hash: DomainHash) -> bool { Domains::execution_receipt(receipt_hash).map( |er| Domains::is_bad_er_pending_to_prune(domain_id, er.domain_block_number) @@ -1179,24 +1185,16 @@ impl_runtime_apis! { Domains::confirmed_domain_block_storage_key(domain_id) } - fn outbox_storage_key(message_id: MessageId) -> Vec { - Messenger::outbox_storage_key(message_id) + fn outbox_storage_key(message_key: MessageKey) -> Vec { + Messenger::outbox_storage_key(message_key) } - fn inbox_response_storage_key(message_id: MessageId) -> Vec { - Messenger::inbox_response_storage_key(message_id) + fn inbox_response_storage_key(message_key: MessageKey) -> Vec { + Messenger::inbox_response_storage_key(message_key) } } - impl sp_messenger::RelayerApi for Runtime { - fn chain_id() -> ChainId { - SelfChainId::get() - } - - fn relay_confirmation_depth() -> BlockNumber { - RelayConfirmationDepth::get() - } - + impl sp_messenger::RelayerApi::Hash> for Runtime { fn block_messages() -> BlockMessagesWithStorageKey { Messenger::get_block_messages() } diff --git a/crates/subspace-service/src/lib.rs b/crates/subspace-service/src/lib.rs index 1589fbff80..c37a720326 100644 --- a/crates/subspace-service/src/lib.rs +++ b/crates/subspace-service/src/lib.rs @@ -87,6 +87,8 @@ use sp_consensus_subspace::digests::extract_pre_digest; use sp_consensus_subspace::{ FarmerPublicKey, KzgExtension, PosExtension, PotExtension, PotNextSlotInput, SubspaceApi, }; +use sp_core::offchain::storage::OffchainDb; +use sp_core::offchain::OffchainDbExt; use sp_core::traits::SpawnEssentialNamed; use sp_core::H256; use sp_domains::{BundleProducerElectionApi, DomainsApi}; @@ -227,6 +229,7 @@ pub type FullSelectChain = sc_consensus::LongestChain; struct SubspaceExtensionsFactory { kzg: Kzg, client: Arc, + backend: Arc, pot_verifier: PotVerifier, executor: Arc, domains_executor: Arc, @@ -368,13 +371,14 @@ where })); exts.register(FraudProofExtension::new(Arc::new( - FraudProofHostFunctionsImpl::<_, _, DomainBlock, _>::new( + FraudProofHostFunctionsImpl::<_, _, DomainBlock, _, _>::new( self.client.clone(), self.domains_executor.clone(), - Box::new(DomainsExtensionFactory::<_, Block, DomainBlock, _>::new( - self.client.clone(), - self.domains_executor.clone(), - )), + |client, executor| { + let extension_factory = + DomainsExtensionFactory::<_, Block, DomainBlock, _>::new(client, executor); + Box::new(extension_factory) as Box> + }, ), ))); @@ -389,6 +393,13 @@ where ), ))); + // if the offchain storage is available, then add offchain extension + // to generate and verify MMR proofs + if let Some(offchain_storage) = self.backend.offchain_storage() { + let offchain_db = OffchainDb::new(offchain_storage); + exts.register(OffchainDbExt::new(offchain_db)); + } + exts } } @@ -485,6 +496,7 @@ where pot_verifier: pot_verifier.clone(), executor: executor.clone(), domains_executor: Arc::new(domains_executor), + backend: backend.clone(), _pos_table: PhantomData, }); diff --git a/domains/client/block-preprocessor/Cargo.toml b/domains/client/block-preprocessor/Cargo.toml index f6f181d17d..965cd62eda 100644 --- a/domains/client/block-preprocessor/Cargo.toml +++ b/domains/client/block-preprocessor/Cargo.toml @@ -24,6 +24,7 @@ sp-block-fees = { version = "0.1.0", path = "../../primitives/block-fees" } sp-core = { version = "21.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-domains = { version = "0.1.0", path = "../../../crates/sp-domains" } sp-executive = { version = "0.1.0", path = "../../primitives/executive" } +sp-externalities = { version = "0.19.0", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-inherents = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-messenger = { version = "0.1.0", path = "../../primitives/messenger" } sp-runtime = { version = "24.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } diff --git a/domains/client/block-preprocessor/src/lib.rs b/domains/client/block-preprocessor/src/lib.rs index fe3df98b98..5353fc9290 100644 --- a/domains/client/block-preprocessor/src/lib.rs +++ b/domains/client/block-preprocessor/src/lib.rs @@ -297,6 +297,21 @@ where )); } + // TODO: remove version check before next network + let messenger_api_version = runtime_api + .api_version::>>(at)? + // safe to return default version as 1 since there will always be version 1. + .unwrap_or(1); + + if messenger_api_version >= 2 { + // check if the extrinsic is an XDM and is valid + if let Some(false) = runtime_api.is_xdm_valid(at, extrinsic.encode())? { + return Ok(BundleValidity::Invalid(InvalidBundleType::InvalidXDM( + index as u32, + ))); + } + } + // Using one instance of runtime_api throughout the loop in order to maintain context // between them. // Using `check_extrinsics_and_do_pre_dispatch` instead of `check_transaction_validity` @@ -316,24 +331,6 @@ where ))); } - // TODO: remove version check before next network - let messenger_api_version = runtime_api - .api_version::>>(at)? - // safe to return default version as 1 since there will always be version 1. - .unwrap_or(1); - - if messenger_api_version >= 2 { - // TODO: the behavior is changed, as before invalid XDM will be dropped silently, - // and the other extrinsic of the bundle will be continue processed, now the whole - // bundle is considered as invalid and excluded from further processing. - if let Some(false) = runtime_api.is_xdm_valid(at, extrinsic.encode())? { - // TODO: Generate a fraud proof for this invalid bundle - return Ok(BundleValidity::Invalid(InvalidBundleType::InvalidXDM( - index as u32, - ))); - } - } - extrinsics.push(extrinsic); } diff --git a/domains/client/block-preprocessor/src/stateless_runtime.rs b/domains/client/block-preprocessor/src/stateless_runtime.rs index 81bed75677..ae34fc989c 100644 --- a/domains/client/block-preprocessor/src/stateless_runtime.rs +++ b/domains/client/block-preprocessor/src/stateless_runtime.rs @@ -1,12 +1,13 @@ use codec::{Codec, Encode}; use domain_runtime_primitives::opaque::AccountId; use domain_runtime_primitives::{Balance, CheckExtrinsicsValidityError, DecodeExtrinsicError}; +use sc_client_api::execution_extensions::ExtensionsFactory; use sc_executor::RuntimeVersionOf; use sp_api::{ApiError, Core}; use sp_core::traits::{CallContext, CodeExecutor, FetchRuntimeCode, RuntimeCode}; use sp_core::Hasher; use sp_domains::core_api::DomainCoreApi; -use sp_messenger::messages::MessageId; +use sp_messenger::messages::MessageKey; use sp_messenger::MessengerApi; use sp_runtime::traits::{Block as BlockT, NumberFor}; use sp_runtime::Storage; @@ -30,6 +31,7 @@ pub struct StatelessRuntime { executor: Arc, runtime_code: Cow<'static, [u8]>, storage: Storage, + extension_factory: Box>, _marker: PhantomData, } @@ -98,6 +100,7 @@ where executor, runtime_code, storage: Storage::default(), + extension_factory: Box::new(()), _marker: Default::default(), } } @@ -109,6 +112,13 @@ where self.storage = storage; } + /// Set the extensions. + /// + /// Inject the necessary extensions for Domain. + pub fn set_extension_factory(&mut self, extension_factory: Box>) { + self.extension_factory = extension_factory; + } + fn runtime_code(&self) -> RuntimeCode<'_> { let code_hash = sp_core::Blake2Hasher::hash(&self.runtime_code); RuntimeCode { @@ -124,6 +134,11 @@ where input: Vec, ) -> Result, ApiError> { let mut ext = BasicExternalities::new(self.storage.clone()); + let ext_extensions = ext.extensions(); + ext_extensions.merge( + self.extension_factory + .extensions_for(Default::default(), Default::default()), + ); let runtime_code = self.runtime_code(); let runtime_version = self .executor @@ -148,20 +163,20 @@ where }) } - pub fn outbox_storage_key(&self, message_id: MessageId) -> Result, ApiError> { + pub fn outbox_storage_key(&self, message_key: MessageKey) -> Result, ApiError> { let storage_key = >::outbox_storage_key( self, Default::default(), - message_id, + message_key, )?; Ok(storage_key) } - pub fn inbox_response_storage_key(&self, message_id: MessageId) -> Result, ApiError> { + pub fn inbox_response_storage_key(&self, message_key: MessageKey) -> Result, ApiError> { let storage_key = >::inbox_response_storage_key( self, Default::default(), - message_id, + message_key, )?; Ok(storage_key) } @@ -210,6 +225,14 @@ where >::is_inherent_extrinsic(self, Default::default(), extrinsic) } + pub fn is_valid_xdm(&self, extrinsic: Vec) -> Result, ApiError> { + >>::is_xdm_valid( + self, + Default::default(), + extrinsic, + ) + } + pub fn decode_extrinsic( &self, opaque_extrinsic: sp_runtime::OpaqueExtrinsic, diff --git a/domains/client/domain-operator/Cargo.toml b/domains/client/domain-operator/Cargo.toml index 0666ef70a3..e28e72aad8 100644 --- a/domains/client/domain-operator/Cargo.toml +++ b/domains/client/domain-operator/Cargo.toml @@ -47,10 +47,14 @@ evm-domain-test-runtime = { version = "0.1.0", path = "../../test/runtime/evm" } frame-system = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } pallet-balances = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } pallet-domains = { version = "0.1.0", path = "../../../crates/pallet-domains" } +pallet-messenger = { version = "0.1.0", path = "../../../domains/pallets/messenger" } +pallet-sudo = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } pallet-timestamp = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } +pallet-transporter = { version = "0.1.0", path = "../../../domains/pallets/transporter" } sc-cli = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false } sc-service = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false } sc-transaction-pool = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } +sp-mmr-primitives = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-state-machine = { version = "0.28.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../../../crates/subspace-core-primitives" } subspace-test-runtime = { version = "0.1.0", path = "../../../test/subspace-test-runtime" } diff --git a/domains/client/domain-operator/src/domain_block_processor.rs b/domains/client/domain-operator/src/domain_block_processor.rs index b8ddab8678..33cf85fff8 100644 --- a/domains/client/domain-operator/src/domain_block_processor.rs +++ b/domains/client/domain-operator/src/domain_block_processor.rs @@ -1110,10 +1110,11 @@ mod tests { ) .unwrap(), Some(InboxedBundleMismatchInfo { - mismatch_type: BundleMismatchType::TrueInvalid(InvalidBundleType::IllegalTx(3)), + mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::InvalidXDM(3)), bundle_index: 1, }) ); + assert_eq!( find_inboxed_bundles_mismatch::( &create_test_execution_receipt(vec![ @@ -1127,7 +1128,7 @@ mod tests { ) .unwrap(), Some(InboxedBundleMismatchInfo { - mismatch_type: BundleMismatchType::FalseInvalid(InvalidBundleType::IllegalTx(3)), + mismatch_type: BundleMismatchType::TrueInvalid(InvalidBundleType::InvalidXDM(3)), bundle_index: 1, }) ); diff --git a/domains/client/domain-operator/src/tests.rs b/domains/client/domain-operator/src/tests.rs index 455a483840..45a44d4d9b 100644 --- a/domains/client/domain-operator/src/tests.rs +++ b/domains/client/domain-operator/src/tests.rs @@ -5,7 +5,7 @@ use crate::fraud_proof::{FraudProofGenerator, TraceDiffType}; use crate::tests::TxPoolError::InvalidTransaction as TxPoolInvalidTransaction; use crate::OperatorSlotInfo; use codec::{Decode, Encode}; -use domain_runtime_primitives::Hash; +use domain_runtime_primitives::{AccountIdConverter, Hash}; use domain_test_primitives::{OnchainStateApi, TimestampApi}; use domain_test_service::evm_domain_test_runtime::{Header, UncheckedExtrinsic}; use domain_test_service::EcdsaKeyring::{Alice, Bob, Charlie, Eve}; @@ -18,7 +18,7 @@ use sc_service::{BasePath, Role}; use sc_transaction_pool::error::Error as PoolError; use sc_transaction_pool_api::error::Error as TxPoolError; use sc_transaction_pool_api::TransactionPool; -use sp_api::ProvideRuntimeApi; +use sp_api::{ProvideRuntimeApi, StorageProof}; use sp_consensus::SyncOracle; use sp_core::storage::StateVersion; use sp_core::traits::FetchRuntimeCode; @@ -36,8 +36,14 @@ use sp_domains_fraud_proof::fraud_proof::{ InvalidTransfersProof, }; use sp_domains_fraud_proof::InvalidTransactionCode; +use sp_messenger::messages::{ + ConsensusChainMmrLeafProof, CrossDomainMessage, FeeModel, InitiateChannelParams, Proof, +}; +use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof as MmrProof}; use sp_runtime::generic::{BlockId, DigestItem}; -use sp_runtime::traits::{BlakeTwo256, Block as BlockT, Hash as HashT, Header as HeaderT, Zero}; +use sp_runtime::traits::{ + BlakeTwo256, Block as BlockT, Convert, Hash as HashT, Header as HeaderT, Zero, +}; use sp_runtime::transaction_validity::InvalidTransaction; use sp_runtime::OpaqueExtrinsic; use sp_state_machine::backend::AsTrieBackend; @@ -1518,6 +1524,170 @@ async fn test_false_invalid_bundles_inherent_extrinsic_proof_creation_and_verifi assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); } +#[tokio::test(flavor = "multi_thread")] +async fn test_invalid_xdm_proof_creation_and_verification() { + let directory = TempDir::new().expect("Must be able to create temporary directory"); + + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(false); + let _ = builder.init(); + + let tokio_handle = tokio::runtime::Handle::current(); + + // Start Ferdie + let mut ferdie = MockConsensusNode::run( + tokio_handle.clone(), + Ferdie, + BasePath::new(directory.path().join("ferdie")), + ); + + // Run Alice (a evm domain authority node) + let alice = domain_test_service::DomainNodeBuilder::new( + tokio_handle.clone(), + Alice, + BasePath::new(directory.path().join("alice")), + ) + .build_evm_node(Role::Authority, GENESIS_DOMAIN_ID, &mut ferdie) + .await; + + let bundle_to_tx = |opaque_bundle| { + subspace_test_runtime::UncheckedExtrinsic::new_unsigned( + pallet_domains::Call::submit_bundle { opaque_bundle }.into(), + ) + .into() + }; + + // Produce a bundle that contains the previously sent extrinsic and record that bundle for later use + let (slot, target_bundle) = ferdie.produce_slot_and_wait_for_bundle_submission().await; + assert_eq!(target_bundle.extrinsics.len(), 0); + produce_block_with!(ferdie.produce_block_with_slot(slot), alice) + .await + .unwrap(); + + // Get a bundle from the txn pool and modify the receipt of the target bundle to an invalid one + let (slot, mut opaque_bundle) = ferdie.produce_slot_and_wait_for_bundle_submission().await; + let original_submit_bundle_tx = bundle_to_tx(opaque_bundle.clone()); + + let bundle_extrinsic_root; + let bad_submit_bundle_tx = { + let invalid_xdm = evm_domain_test_runtime::UncheckedExtrinsic::new_unsigned( + pallet_messenger::Call::relay_message { + msg: CrossDomainMessage { + src_chain_id: ChainId::Consensus, + dst_chain_id: ChainId::Domain(GENESIS_DOMAIN_ID), + channel_id: Default::default(), + nonce: Default::default(), + proof: Proof::Domain { + consensus_chain_mmr_proof: ConsensusChainMmrLeafProof { + consensus_block_hash: Default::default(), + opaque_mmr_leaf: EncodableOpaqueLeaf(vec![0, 1, 2]), + proof: MmrProof { + leaf_indices: vec![], + leaf_count: 0, + items: vec![], + }, + }, + domain_proof: StorageProof::empty(), + message_proof: StorageProof::empty(), + }, + weight_tag: Default::default(), + }, + } + .into(), + ) + .into(); + opaque_bundle.extrinsics = vec![invalid_xdm]; + let extrinsics: Vec> = opaque_bundle + .extrinsics + .clone() + .into_iter() + .map(|ext| ext.encode()) + .collect(); + bundle_extrinsic_root = + BlakeTwo256::ordered_trie_root(extrinsics.clone(), StateVersion::V1); + opaque_bundle.sealed_header.header.bundle_extrinsics_root = bundle_extrinsic_root; + opaque_bundle.sealed_header.signature = Sr25519Keyring::Alice + .pair() + .sign(opaque_bundle.sealed_header.pre_hash().as_ref()) + .into(); + bundle_to_tx(opaque_bundle) + }; + + // Replace `original_submit_bundle_tx` with `bad_submit_bundle_tx` in the tx pool + ferdie + .prune_tx_from_pool(&original_submit_bundle_tx) + .await + .unwrap(); + assert!(ferdie.get_bundle_from_tx_pool(slot).is_none()); + + ferdie + .submit_transaction(bad_submit_bundle_tx) + .await + .unwrap(); + + produce_block_with!(ferdie.produce_block_with_slot(slot), alice) + .await + .unwrap(); + + // produce another bundle that marks the previous extrinsic as invalid. + let (slot, mut opaque_bundle) = ferdie.produce_slot_and_wait_for_bundle_submission().await; + let original_submit_bundle_tx = bundle_to_tx(opaque_bundle.clone()); + + let (bad_receipt_hash, bad_submit_bundle_tx) = { + let bad_receipt = &mut opaque_bundle.sealed_header.header.receipt; + // bad receipt marks this particular bundle as valid even though bundle contains illegal extrinsic + bad_receipt.inboxed_bundles = + vec![InboxedBundle::valid(H256::random(), bundle_extrinsic_root)]; + + opaque_bundle.sealed_header.signature = Sr25519Keyring::Alice + .pair() + .sign(opaque_bundle.sealed_header.pre_hash().as_ref()) + .into(); + ( + opaque_bundle.receipt().hash::(), + bundle_to_tx(opaque_bundle), + ) + }; + + // Replace `original_submit_bundle_tx` with `bad_submit_bundle_tx` in the tx pool + ferdie + .prune_tx_from_pool(&original_submit_bundle_tx) + .await + .unwrap(); + assert!(ferdie.get_bundle_from_tx_pool(slot).is_none()); + + ferdie + .submit_transaction(bad_submit_bundle_tx) + .await + .unwrap(); + + // Wait for the fraud proof that target the bad ER + let wait_for_fraud_proof_fut = ferdie.wait_for_fraud_proof(move |fp| { + if let FraudProof::InvalidBundles(proof) = fp { + if let InvalidBundleType::InvalidXDM(extrinsic_index) = proof.invalid_bundle_type { + assert!(proof.is_true_invalid_fraud_proof); + assert_eq!(extrinsic_index, 0); + return true; + } + } + false + }); + + // Produce a consensus block that contains the `bad_submit_bundle_tx` and the bad receipt should + // be added to the consensus chain block tree + produce_block_with!(ferdie.produce_block_with_slot(slot), alice) + .await + .unwrap(); + assert!(ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); + + let _ = wait_for_fraud_proof_fut.await; + + // Produce a consensus block that contains the fraud proof, the fraud proof wil be verified + // and executed, thus pruned the bad receipt from the block tree + ferdie.produce_blocks(1).await.unwrap(); + assert!(!ferdie.does_receipt_exist(bad_receipt_hash).unwrap()); +} + #[tokio::test(flavor = "multi_thread")] async fn test_true_invalid_bundles_illegal_extrinsic_proof_creation_and_verification() { let directory = TempDir::new().expect("Must be able to create temporary directory"); @@ -3117,6 +3287,90 @@ async fn existing_bundle_can_be_resubmitted_to_new_fork() { assert_eq!(alice.client.info().best_number, pre_alice_best_number + 2); } +#[tokio::test(flavor = "multi_thread")] +async fn test_cross_domains_messages_should_work() { + let directory = TempDir::new().expect("Must be able to create temporary directory"); + + let mut builder = sc_cli::LoggerBuilder::new(""); + builder.with_colors(false); + let _ = builder.init(); + + let tokio_handle = tokio::runtime::Handle::current(); + + // Start Ferdie + let mut ferdie = MockConsensusNode::run_with_finalization_depth( + tokio_handle.clone(), + Ferdie, + BasePath::new(directory.path().join("ferdie")), + // finalization depth + Some(10), + ); + + // Run Alice (an evm domain) + let mut alice = domain_test_service::DomainNodeBuilder::new( + tokio_handle.clone(), + Alice, + BasePath::new(directory.path().join("alice")), + ) + .build_evm_node(Role::Authority, GENESIS_DOMAIN_ID, &mut ferdie) + .await; + + // Run the cross domain gossip message worker + ferdie.start_cross_domain_gossip_message_worker(); + + produce_blocks!(ferdie, alice, 3).await.unwrap(); + + // Open channel between the Consensus chain and EVM domains + let fee_model = FeeModel { relay_fee: 1 }; + alice + .construct_and_send_extrinsic(pallet_sudo::Call::sudo { + call: Box::new(evm_domain_test_runtime::RuntimeCall::Messenger( + pallet_messenger::Call::initiate_channel { + dst_chain_id: ChainId::Consensus, + params: InitiateChannelParams { + max_outgoing_messages: 100, + fee_model, + }, + }, + )), + }) + .await + .expect("Failed to construct and send extrinsic"); + // Wait until channel open + produce_blocks_until!(ferdie, alice, { + alice + .get_open_channel_for_chain(ChainId::Consensus) + .is_some() + }) + .await + .unwrap(); + + // Transfer balance from + let pre_alice_free_balance = alice.free_balance(alice.key.to_account_id()); + let pre_ferdie_free_balance = ferdie.free_balance(ferdie.key.to_account_id()); + let transfer_amount = 10; + alice + .construct_and_send_extrinsic(pallet_transporter::Call::transfer { + dst_location: pallet_transporter::Location { + chain_id: ChainId::Consensus, + account_id: AccountIdConverter::convert(Ferdie.into()), + }, + amount: transfer_amount, + }) + .await + .expect("Failed to construct and send extrinsic"); + // Wait until transfer succeed + produce_blocks_until!(ferdie, alice, { + let post_alice_free_balance = alice.free_balance(alice.key.to_account_id()); + let post_ferdie_free_balance = ferdie.free_balance(ferdie.key.to_account_id()); + + post_alice_free_balance < pre_alice_free_balance - transfer_amount + && post_ferdie_free_balance == pre_ferdie_free_balance + transfer_amount + }) + .await + .unwrap(); +} + // TODO: Unlock test when multiple domains are supported in DecEx v2. // #[tokio::test(flavor = "multi_thread")] // async fn test_cross_domains_message_should_work() { diff --git a/domains/client/relayer/Cargo.toml b/domains/client/relayer/Cargo.toml index 9974f9b07e..7b62100ddf 100644 --- a/domains/client/relayer/Cargo.toml +++ b/domains/client/relayer/Cargo.toml @@ -16,13 +16,13 @@ async-channel = "1.9.0" cross-domain-message-gossip = { path = "../../client/cross-domain-message-gossip" } futures = "0.3.29" parity-scale-codec = { version = "3.6.9", features = ["derive"] } -scale-info = { version = "2.7.0", features = ["derive"] } sc-client-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sc-state-db = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sc-utils = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-blockchain = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-consensus = { version = "0.10.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } +sp-core = { version = "21.0.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-domains = { version = "0.1.0", path = "../../../crates/sp-domains" } sp-messenger = { version = "0.1.0", path = "../../primitives/messenger" } sp-mmr-primitives = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } diff --git a/domains/client/relayer/src/lib.rs b/domains/client/relayer/src/lib.rs index 88106ed8eb..01f7e7fd63 100644 --- a/domains/client/relayer/src/lib.rs +++ b/domains/client/relayer/src/lib.rs @@ -1,21 +1,22 @@ #![warn(rust_2018_idioms)] +#![deny(unused_crate_dependencies)] pub mod worker; use async_channel::TrySendError; use cross_domain_message_gossip::Message as GossipMessage; -use parity_scale_codec::{Decode, Encode, FullCodec}; +use parity_scale_codec::{Codec, Decode, Encode}; use sc_client_api::{AuxStore, HeaderBackend, ProofProvider, StorageProof}; use sc_utils::mpsc::TracingUnboundedSender; -use scale_info::TypeInfo; use sp_api::ProvideRuntimeApi; -use sp_domains::DomainsApi; +use sp_core::H256; +use sp_domains::{DomainId, DomainsApi}; use sp_messenger::messages::{ BlockMessageWithStorageKey, BlockMessagesWithStorageKey, ChainId, ConsensusChainMmrLeafProof, CrossDomainMessage, Proof, }; use sp_messenger::{MessengerApi, RelayerApi}; -use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof as MmrProof}; +use sp_mmr_primitives::{EncodableOpaqueLeaf, MmrApi}; use sp_runtime::traits::{Block as BlockT, CheckedSub, Header as HeaderT, NumberFor, One, Zero}; use sp_runtime::ArithmeticError; use std::marker::PhantomData; @@ -51,12 +52,18 @@ pub enum Error { ArithmeticError(ArithmeticError), /// Api related error. ApiError(sp_api::ApiError), - /// Emits when the core domain block is not yet confirmed on the system domain. - DomainNonConfirmedOnConsensusChain, /// Failed to submit a cross domain message UnableToSubmitCrossDomainMessage(TrySendError), /// Invalid ChainId InvalidChainId, + /// Failed to generate MMR proof + MmrProof(sp_mmr_primitives::Error), + /// MMR Leaf missing + MmrLeafMissing, + /// Missing block header + MissingBlockHeader, + /// Missing block hash + MissingBlockHash, } impl From for Error { @@ -80,102 +87,160 @@ impl From for Error { } } -type ProofOf = Proof<::Hash, ::Hash>; +type ProofOf = Proof<::Hash, H256>; type UnProcessedBlocks = Vec<(NumberFor, ::Hash)>; -impl Relayer +fn construct_consensus_mmr_proof( + consensus_chain_client: &Arc, + block_number: NumberFor, +) -> Result<(EncodableOpaqueLeaf, sp_mmr_primitives::Proof), Error> where Block: BlockT, - Client: HeaderBackend + AuxStore + ProofProvider + ProvideRuntimeApi, - Client::Api: RelayerApi>, + Client: ProvideRuntimeApi + HeaderBackend, + Client::Api: MmrApi>, { - pub(crate) fn chain_id(client: &Arc) -> Result { - client - .runtime_api() - .chain_id(client.info().best_hash) - .map_err(|_| Error::UnableToFetchDomainId) - } - - pub(crate) fn relay_confirmation_depth( - client: &Arc, - ) -> Result, Error> { - let best_block_id = client.info().best_hash; - let api = client.runtime_api(); - api.relay_confirmation_depth(best_block_id) - .map_err(|_| Error::UnableToFetchRelayConfirmationDepth) - } - - /// Constructs the proof for the given key using the consensus chain backend. - fn construct_consensus_chain_storage_proof_for_key_at( - consensus_chain_client: &Arc, - block_hash: Block::Hash, - key: &[u8], - ) -> Result, Error> { - consensus_chain_client - .header(block_hash)? - .map(|header| (header.hash(), *header.state_root())) - .and_then(|(block_hash, _state_root)| { - let proof = consensus_chain_client - .read_proof(block_hash, &mut [key].into_iter()) - .ok()?; - // TODO: derive the correct proof here - Some(Proof::Consensus { - consensus_chain_mmr_proof: ConsensusChainMmrLeafProof { - consensus_block_hash: block_hash, - opaque_mmr_leaf: EncodableOpaqueLeaf(vec![]), - proof: MmrProof { - leaf_indices: vec![], - leaf_count: 0, - items: vec![], - }, - }, - message_proof: proof, - }) - }) - .ok_or(Error::ConstructStorageProof) - } + let api = consensus_chain_client.runtime_api(); + let best_hash = consensus_chain_client.info().best_hash; + let (mut leaves, proof) = api + .generate_proof(best_hash, vec![block_number], Some(block_number)) + .map_err(Error::ApiError)? + .map_err(Error::MmrProof)?; + debug_assert!(leaves.len() == 1, "should always be of length 1"); + let leaf = leaves.pop().ok_or(Error::MmrLeafMissing)?; + Ok((leaf, proof)) +} - fn construct_cross_chain_message_and_submit< - Submitter: Fn(CrossDomainMessage) -> Result<(), Error>, - ProofConstructor: Fn(Block::Hash, &[u8]) -> Result, Error>, - >( - block_hash: Block::Hash, - msgs: Vec, - proof_constructor: ProofConstructor, - submitter: Submitter, - ) -> Result<(), Error> { - for msg in msgs { - let proof = match proof_constructor(block_hash, &msg.storage_key) { - Ok(proof) => proof, - Err(err) => { - tracing::error!( - target: LOG_TARGET, - "Failed to construct storage proof for message: {:?} bound to domain: {:?} with error: {:?}", - (msg.channel_id, msg.nonce), - msg.dst_chain_id, - err - ); - continue; - } - }; - let msg = CrossDomainMessage::from_relayer_msg_with_proof(msg, proof); - let (dst_domain, msg_id) = (msg.dst_chain_id, (msg.channel_id, msg.nonce)); - if let Err(err) = submitter(msg) { +fn construct_cross_chain_message_and_submit( + msgs: Vec, + proof_constructor: ProofConstructor, + submitter: Submitter, +) -> Result<(), Error> +where + Submitter: Fn(CrossDomainMessage) -> Result<(), Error>, + ProofConstructor: Fn(&[u8]) -> Result, Error>, +{ + for msg in msgs { + let proof = match proof_constructor(&msg.storage_key) { + Ok(proof) => proof, + Err(err) => { tracing::error!( target: LOG_TARGET, - ?err, - "Failed to submit message: {msg_id:?} to domain: {dst_domain:?}", + "Failed to construct storage proof for message: {:?} bound to chain: {:?} with error: {:?}", + (msg.channel_id, msg.nonce), + msg.dst_chain_id, + err ); + continue; } + }; + let msg = CrossDomainMessage::from_relayer_msg_with_proof(msg, proof); + let (dst_domain, msg_id) = (msg.dst_chain_id, (msg.channel_id, msg.nonce)); + if let Err(err) = submitter(msg) { + tracing::error!( + target: LOG_TARGET, + ?err, + "Failed to submit message: {msg_id:?} to domain: {dst_domain:?}", + ); } + } - Ok(()) + Ok(()) +} + +/// Sends an Outbox message from src_domain to dst_domain. +fn gossip_outbox_message( + client: &Arc, + msg: CrossDomainMessage, + sink: &GossipMessageSink, +) -> Result<(), Error> +where + Block: BlockT, + CHash: Codec, + Client: ProvideRuntimeApi + HeaderBackend, + Client::Api: RelayerApi, CHash>, +{ + let best_hash = client.info().best_hash; + let dst_chain_id = msg.dst_chain_id; + let ext = client + .runtime_api() + .outbox_message_unsigned(best_hash, msg)? + .ok_or(Error::FailedToConstructExtrinsic)?; + + sink.unbounded_send(GossipMessage { + chain_id: dst_chain_id, + encoded_data: ext.encode(), + }) + .map_err(Error::UnableToSubmitCrossDomainMessage) +} + +/// Sends an Inbox message response from src_domain to dst_domain +/// Inbox message was earlier sent by dst_domain to src_domain and +/// this message is the response of the Inbox message execution. +fn gossip_inbox_message_response( + client: &Arc, + msg: CrossDomainMessage, + sink: &GossipMessageSink, +) -> Result<(), Error> +where + Block: BlockT, + CHash: Codec, + Client: ProvideRuntimeApi + HeaderBackend, + Client::Api: RelayerApi, CHash>, +{ + let best_hash = client.info().best_hash; + let dst_chain_id = msg.dst_chain_id; + let ext = client + .runtime_api() + .inbox_response_message_unsigned(best_hash, msg)? + .ok_or(Error::FailedToConstructExtrinsic)?; + + sink.unbounded_send(GossipMessage { + chain_id: dst_chain_id, + encoded_data: ext.encode(), + }) + .map_err(Error::UnableToSubmitCrossDomainMessage) +} + +impl Relayer +where + Block: BlockT, + Client: HeaderBackend + AuxStore + ProofProvider + ProvideRuntimeApi, +{ + /// Constructs the proof for the given key using the consensus chain backend. + fn construct_consensus_chain_xdm_proof_for_key_at( + consensus_chain_client: &Arc, + finalized_block: (NumberFor, Block::Hash), + block_hash_to_process: Block::Hash, + key: &[u8], + ) -> Result, Error> + where + Client::Api: MmrApi>, + { + let proof = consensus_chain_client + .read_proof(block_hash_to_process, &mut [key].into_iter()) + .map_err(|_| Error::ConstructStorageProof)?; + + let (mmr_leaf, mmr_proof) = + construct_consensus_mmr_proof(consensus_chain_client, finalized_block.0)?; + + Ok(Proof::Consensus { + consensus_chain_mmr_proof: ConsensusChainMmrLeafProof { + consensus_block_hash: finalized_block.1, + opaque_mmr_leaf: mmr_leaf, + proof: mmr_proof, + }, + message_proof: proof, + }) } - fn filter_messages( + fn filter_messages( client: &Arc, mut msgs: BlockMessagesWithStorageKey, - ) -> Result { + ) -> Result + where + CHash: Codec, + Client::Api: RelayerApi, CHash>, + { let api = client.runtime_api(); let best_hash = client.info().best_hash; msgs.outbox.retain(|msg| { @@ -215,12 +280,27 @@ where pub(crate) fn submit_messages_from_consensus_chain( consensus_chain_client: &Arc, - confirmed_block_hash: Block::Hash, + finalized_block: (NumberFor, Block::Hash), gossip_message_sink: &GossipMessageSink, - ) -> Result<(), Error> { + ) -> Result<(), Error> + where + Client::Api: MmrApi> + + RelayerApi, Block::Hash>, + { + // since the `finalized_block_number - 1` block MMR leaf is included in `finalized_block_number`, + // we process that block instead while using the finalized block number to generate the MMR + // proof `finalized_block_number -1` + let block_number_to_process = match finalized_block.0.checked_sub(&One::one()) { + None => return Ok(()), + Some(number) => number, + }; + let block_hash_to_process = consensus_chain_client + .hash(block_number_to_process)? + .ok_or(Error::MissingBlockHeader)?; + let api = consensus_chain_client.runtime_api(); let block_messages: BlockMessagesWithStorageKey = api - .block_messages(confirmed_block_hash) + .block_messages(block_hash_to_process) .map_err(|_| Error::FetchAssignedMessages)?; let filtered_messages = Self::filter_messages(consensus_chain_client, block_messages)?; @@ -229,88 +309,58 @@ where return Ok(()); } - Self::construct_cross_chain_message_and_submit( - confirmed_block_hash, + construct_cross_chain_message_and_submit::( filtered_messages.outbox, - |block_id, key| { - Self::construct_consensus_chain_storage_proof_for_key_at( + |key| { + Self::construct_consensus_chain_xdm_proof_for_key_at( consensus_chain_client, - block_id, + finalized_block, + block_hash_to_process, key, ) }, - |msg| Self::gossip_outbox_message(consensus_chain_client, msg, gossip_message_sink), + |msg| gossip_outbox_message(consensus_chain_client, msg, gossip_message_sink), )?; - Self::construct_cross_chain_message_and_submit( - confirmed_block_hash, + construct_cross_chain_message_and_submit::( filtered_messages.inbox_responses, - |block_id, key| { - Self::construct_consensus_chain_storage_proof_for_key_at( + |key| { + Self::construct_consensus_chain_xdm_proof_for_key_at( consensus_chain_client, - block_id, + finalized_block, + block_hash_to_process, key, ) }, - |msg| { - Self::gossip_inbox_message_response( - consensus_chain_client, - msg, - gossip_message_sink, - ) - }, + |msg| gossip_inbox_message_response(consensus_chain_client, msg, gossip_message_sink), )?; Ok(()) } - pub(crate) fn submit_messages_from_domain( + pub(crate) fn submit_messages_from_domain( + domain_id: DomainId, domain_client: &Arc, - consensus_chain_client: &Arc, - confirmed_block_hash: Block::Hash, + consensus_chain_client: &Arc, + finalized_consensus_block: (NumberFor, CBlock::Hash), + confirmed_domain_block_hash: Block::Hash, gossip_message_sink: &GossipMessageSink, - relay_confirmation_depth: NumberFor, ) -> Result<(), Error> where - CCBlock: BlockT, - Block::Hash: FullCodec, - NumberFor: FullCodec + TypeInfo, - NumberFor: Into>, - CCBlock::Hash: Into, - CCC: HeaderBackend + ProvideRuntimeApi + ProofProvider, - CCC::Api: DomainsApi + MessengerApi>, + CBlock: BlockT, + CClient: HeaderBackend + ProvideRuntimeApi + ProofProvider, + CClient::Api: DomainsApi + + MessengerApi> + + MmrApi>, + Client::Api: RelayerApi, CBlock::Hash>, { - let chain_id = Self::chain_id(domain_client)?; - let ChainId::Domain(domain_id) = chain_id else { - return Err(Error::InvalidChainId); - }; - - let domain_block_header = domain_client.expect_header(confirmed_block_hash)?; - let consensus_chain_api = consensus_chain_client.runtime_api(); - let best_consensus_chain_hash = consensus_chain_client.info().best_hash; - let best_consensus_chain_block_header = - consensus_chain_client.expect_header(best_consensus_chain_hash)?; - - // verify if the domain number is K-deep on Consensus chain - if !consensus_chain_api - .domain_best_number(best_consensus_chain_hash, domain_id)? - .map( - |best_number| match best_number.checked_sub(&relay_confirmation_depth) { - None => false, - Some(best_confirmed) => best_confirmed >= (*domain_block_header.number()), - }, - ) - .unwrap_or(false) - { - return Err(Error::DomainNonConfirmedOnConsensusChain); - } - // fetch messages to be relayed let domain_api = domain_client.runtime_api(); let block_messages: BlockMessagesWithStorageKey = domain_api - .block_messages(confirmed_block_hash) + .block_messages(confirmed_domain_block_hash) .map_err(|_| Error::FetchAssignedMessages)?; + // filter out already relayed messages let filtered_messages = Self::filter_messages(domain_client, block_messages)?; // short circuit if the there are no messages to relay @@ -318,140 +368,117 @@ where return Ok(()); } - // generate domain proof that points to the confirmed domain block on consensus chain + // Generate domain proof that points to the confirmed domain block. + // Confirmed domain block is taken from the parent of the finalized consensus block + let consensus_chain_number_to_process = + match finalized_consensus_block.0.checked_sub(&One::one()) { + None => return Ok(()), + Some(number) => number, + }; + + let consensus_block_hash = consensus_chain_client + .hash(consensus_chain_number_to_process)? + .ok_or(Error::MissingBlockHash)?; + + let consensus_chain_api = consensus_chain_client.runtime_api(); let storage_key = consensus_chain_api - .confirmed_domain_block_storage_key(best_consensus_chain_hash, domain_id)?; + .confirmed_domain_block_storage_key(consensus_block_hash, domain_id)?; let domain_proof = consensus_chain_client.read_proof( - best_consensus_chain_hash, + consensus_block_hash, &mut [storage_key.as_ref()].into_iter(), )?; - Self::construct_cross_chain_message_and_submit( - confirmed_block_hash, + construct_cross_chain_message_and_submit::( filtered_messages.outbox, - |block_hash, key| { - Self::construct_domain_storage_proof_for_key_at( - best_consensus_chain_hash, + |key| { + Self::construct_domain_chain_xdm_proof_for_key_at( + finalized_consensus_block, + consensus_chain_client, domain_client, - block_hash, + confirmed_domain_block_hash, key, - *best_consensus_chain_block_header.state_root(), domain_proof.clone(), ) }, - |msg| Self::gossip_outbox_message(domain_client, msg, gossip_message_sink), + |msg| gossip_outbox_message(domain_client, msg, gossip_message_sink), )?; - Self::construct_cross_chain_message_and_submit( - confirmed_block_hash, + construct_cross_chain_message_and_submit::( filtered_messages.inbox_responses, - |block_id, key| { - Self::construct_domain_storage_proof_for_key_at( - best_consensus_chain_hash, + |key| { + Self::construct_domain_chain_xdm_proof_for_key_at( + finalized_consensus_block, + consensus_chain_client, domain_client, - block_id, + confirmed_domain_block_hash, key, - *best_consensus_chain_block_header.state_root(), domain_proof.clone(), ) }, - |msg| Self::gossip_inbox_message_response(domain_client, msg, gossip_message_sink), + |msg| gossip_inbox_message_response(domain_client, msg, gossip_message_sink), )?; Ok(()) } /// Constructs the proof for the given key using the domain backend. - fn construct_domain_storage_proof_for_key_at( - consensus_chain_block_hash: CHash, + fn construct_domain_chain_xdm_proof_for_key_at( + consensus_chain_finalized_block: (NumberFor, CBlock::Hash), + consensus_chain_client: &Arc, domain_client: &Arc, block_hash: Block::Hash, key: &[u8], - _consensus_chain_state_root: CHash, domain_proof: StorageProof, - ) -> Result, Error> + ) -> Result, Error> where - CHash: Into, + CBlock: BlockT, + CClient: HeaderBackend + ProvideRuntimeApi + ProofProvider, + CClient::Api: DomainsApi + + MessengerApi> + + MmrApi>, { - domain_client - .header(block_hash)? - .map(|header| (*header.number(), header.hash())) - .and_then(|(_number, _hash)| { - let proof = domain_client - .read_proof(block_hash, &mut [key].into_iter()) - .ok()?; - // TODO: Derive correct domain proof - Some(Proof::Domain { - consensus_chain_mmr_proof: ConsensusChainMmrLeafProof { - consensus_block_hash: consensus_chain_block_hash.into(), - opaque_mmr_leaf: EncodableOpaqueLeaf(vec![]), - proof: MmrProof { - leaf_indices: vec![], - leaf_count: 0, - items: vec![], - }, - }, - domain_proof, - message_proof: proof, - }) - }) - .ok_or(Error::ConstructStorageProof) - } + let (mmr_leaf, mmr_proof) = construct_consensus_mmr_proof( + consensus_chain_client, + consensus_chain_finalized_block.0, + )?; - /// Sends an Outbox message from src_domain to dst_domain. - fn gossip_outbox_message( - client: &Arc, - msg: CrossDomainMessage, - sink: &GossipMessageSink, - ) -> Result<(), Error> { - let best_hash = client.info().best_hash; - let dst_chain_id = msg.dst_chain_id; - let ext = client - .runtime_api() - .outbox_message_unsigned(best_hash, msg)? - .ok_or(Error::FailedToConstructExtrinsic)?; - - sink.unbounded_send(GossipMessage { - chain_id: dst_chain_id, - encoded_data: ext.encode(), - }) - .map_err(Error::UnableToSubmitCrossDomainMessage) - } + let proof = domain_client + .read_proof(block_hash, &mut [key].into_iter()) + .map_err(|_| Error::ConstructStorageProof)?; - /// Sends an Inbox message response from src_domain to dst_domain - /// Inbox message was earlier sent by dst_domain to src_domain and - /// this message is the response of the Inbox message execution. - fn gossip_inbox_message_response( - client: &Arc, - msg: CrossDomainMessage, - sink: &GossipMessageSink, - ) -> Result<(), Error> { - let best_hash = client.info().best_hash; - let dst_chain_id = msg.dst_chain_id; - let ext = client - .runtime_api() - .inbox_response_message_unsigned(best_hash, msg)? - .ok_or(Error::FailedToConstructExtrinsic)?; - - sink.unbounded_send(GossipMessage { - chain_id: dst_chain_id, - encoded_data: ext.encode(), + Ok(Proof::Domain { + consensus_chain_mmr_proof: ConsensusChainMmrLeafProof { + consensus_block_hash: consensus_chain_finalized_block.1, + opaque_mmr_leaf: mmr_leaf, + proof: mmr_proof, + }, + domain_proof, + message_proof: proof, }) - .map_err(Error::UnableToSubmitCrossDomainMessage) } - fn relayed_blocks_at_number_key(chain_id: ChainId, number: NumberFor) -> Vec { + fn relayed_consensus_blocks_at_number_key( + chain_id: ChainId, + number: NumberFor, + ) -> Vec { ( - b"message_relayer_processed_block_of_domain", + b"message_relayer_processed_consensus_block_of_chain", chain_id, number, ) .encode() } + fn relayed_domain_blocks_at_number_key( + domain_id: DomainId, + number: NumberFor, + ) -> Vec { + (b"message_relayer_processed_domain_block", domain_id, number).encode() + } + /// Takes number as tip and finds all the unprocessed blocks including the tip. - fn fetch_unprocessed_blocks_until( + fn fetch_unprocessed_consensus_blocks_until( client: &Arc, chain_id: ChainId, best_number: NumberFor, @@ -459,7 +486,7 @@ where ) -> Result, Error> { let mut blocks_to_process = vec![]; let (mut number_to_check, mut hash_to_check) = (best_number, best_hash); - while !Self::fetch_blocks_relayed_at(client, chain_id, number_to_check) + while !Self::fetch_consensus_blocks_relayed_at(client, chain_id, number_to_check) .contains(&hash_to_check) { blocks_to_process.push((number_to_check, hash_to_check)); @@ -485,13 +512,31 @@ where Ok(blocks_to_process) } - fn fetch_blocks_relayed_at( + fn fetch_consensus_blocks_relayed_at( client: &Arc, chain_id: ChainId, number: NumberFor, ) -> Vec { + Self::fetch_blocks_relayed_at( + client, + Self::relayed_consensus_blocks_at_number_key(chain_id, number), + ) + } + + fn fetch_domains_blocks_relayed_at( + client: &Arc, + domain_id: DomainId, + number: NumberFor, + ) -> Vec { + Self::fetch_blocks_relayed_at( + client, + Self::relayed_domain_blocks_at_number_key(domain_id, number), + ) + } + + fn fetch_blocks_relayed_at(client: &Arc, key: Vec) -> Vec { client - .get_aux(&Self::relayed_blocks_at_number_key(chain_id, number)) + .get_aux(&key) .ok() .flatten() .and_then(|enc_val| Vec::::decode(&mut enc_val.as_ref()).ok()) @@ -503,26 +548,45 @@ where // and then prune the storage. // We can use Finalize event but its not triggered yet as we dont finalize. // Other option would be to use fraud proof period. - pub(crate) fn store_relayed_block( + fn store_relayed_block( client: &Arc, - chain_id: ChainId, - block_number: NumberFor, + key: Vec, block_hash: Block::Hash, ) -> Result<(), Error> { - let mut processed_blocks = Self::fetch_blocks_relayed_at(client, chain_id, block_number); + let mut processed_blocks = Self::fetch_blocks_relayed_at(client, key.clone()); if processed_blocks.contains(&block_hash) { return Ok(()); } processed_blocks.push(block_hash); client - .insert_aux( - &[( - Self::relayed_blocks_at_number_key(chain_id, block_number).as_ref(), - processed_blocks.encode().as_ref(), - )], - &[], - ) + .insert_aux(&[(key.as_ref(), processed_blocks.encode().as_ref())], &[]) .map_err(|_| Error::StoreRelayedBlockNumber) } + + fn store_relayed_consensus_block( + client: &Arc, + chain_id: ChainId, + block_number: NumberFor, + block_hash: Block::Hash, + ) -> Result<(), Error> { + Self::store_relayed_block( + client, + Self::relayed_consensus_blocks_at_number_key(chain_id, block_number), + block_hash, + ) + } + + fn store_relayed_domain_block( + client: &Arc, + domain_id: DomainId, + block_number: NumberFor, + block_hash: Block::Hash, + ) -> Result<(), Error> { + Self::store_relayed_block( + client, + Self::relayed_domain_blocks_at_number_key(domain_id, block_number), + block_hash, + ) + } } diff --git a/domains/client/relayer/src/worker.rs b/domains/client/relayer/src/worker.rs index eda26b3461..9bc7faeeab 100644 --- a/domains/client/relayer/src/worker.rs +++ b/domains/client/relayer/src/worker.rs @@ -1,19 +1,18 @@ use crate::{BlockT, Error, GossipMessageSink, HeaderBackend, HeaderT, Relayer, LOG_TARGET}; use futures::StreamExt; -use parity_scale_codec::FullCodec; use sc_client_api::{AuxStore, BlockchainEvents, ProofProvider}; use sc_state_db::PruningMode; use sp_api::{ApiError, ProvideRuntimeApi}; use sp_consensus::SyncOracle; -use sp_domains::DomainsApi; +use sp_domains::{DomainId, DomainsApi}; use sp_messenger::messages::ChainId; use sp_messenger::{MessengerApi, RelayerApi}; -use sp_runtime::scale_info::TypeInfo; -use sp_runtime::traits::{CheckedSub, NumberFor, Zero}; +use sp_mmr_primitives::MmrApi; +use sp_runtime::traits::{CheckedSub, NumberFor, One}; use std::sync::Arc; /// Starts relaying consensus chain messages to other domains. -/// If the node is in major sync, worker waits waits until the sync is finished. +/// If the node is in major sync, worker waits until the sync is finished. pub async fn relay_consensus_chain_messages( consensus_chain_client: Arc, state_pruning_mode: PruningMode, @@ -26,24 +25,31 @@ pub async fn relay_consensus_chain_messages( + AuxStore + ProofProvider + ProvideRuntimeApi, - Client::Api: RelayerApi>, + Client::Api: RelayerApi, Block::Hash> + + MmrApi>, SO: SyncOracle, { - // there is not confirmation depth for relayer on system domain - // since all the relayers will haven embed client to known the canonical chain. let result = start_relaying_messages( - NumberFor::::zero(), + ChainId::Consensus, consensus_chain_client.clone(), - |client, block_hash| { - Relayer::submit_messages_from_consensus_chain(client, block_hash, &gossip_message_sink) + |client, block_id, _| { + Relayer::submit_messages_from_consensus_chain(client, block_id, &gossip_message_sink) }, sync_oracle, - |_, relay_number| -> Result { - Ok(is_state_available( - &state_pruning_mode, - &consensus_chain_client, - relay_number, - )) + |block_number| -> Result, ApiError> { + // since a parent mmr leaf is included in its child, + // we process the finalized block's parent instead since we know parent is implicitly finalized + // so we ensure the state of the parent is available here + Ok(block_number + .checked_sub(&One::one()) + .and_then(|number_to_check| { + is_state_available( + &state_pruning_mode, + &consensus_chain_client, + number_to_check, + ) + .then_some(()) + })) }, ) .await; @@ -57,74 +63,113 @@ pub async fn relay_consensus_chain_messages( } } +type DomainExtraData = (NumberFor, ::Hash); + /// Starts relaying domain messages to other chains. -/// If the domain node is in major sync, worker waits waits until the sync is finished. -pub async fn relay_domain_messages( - consensus_chain_client: Arc, - domain_client: Arc, +/// If the domain node is in major sync, worker waits until the sync is finished. +pub async fn relay_domain_messages( + domain_id: DomainId, + consensus_chain_client: Arc, + consensus_state_pruning: PruningMode, + domain_client: Arc, domain_state_pruning: PruningMode, sync_oracle: SO, gossip_message_sink: GossipMessageSink, ) where Block: BlockT, - CCBlock: BlockT, - Block::Hash: FullCodec, - NumberFor: FullCodec + TypeInfo, - NumberFor: Into>, - CCBlock::Hash: Into, - DC: BlockchainEvents - + HeaderBackend - + AuxStore - + ProofProvider - + ProvideRuntimeApi, - DC::Api: RelayerApi>, - CCC: HeaderBackend + ProvideRuntimeApi + ProofProvider, - CCC::Api: DomainsApi + MessengerApi>, + CBlock: BlockT, + Client: HeaderBackend + AuxStore + ProofProvider + ProvideRuntimeApi, + Client::Api: RelayerApi, CBlock::Hash>, + CClient: BlockchainEvents + + HeaderBackend + + ProvideRuntimeApi + + ProofProvider + + AuxStore, + CClient::Api: DomainsApi + + MessengerApi> + + MmrApi>, SO: SyncOracle + Send, { - let relay_confirmation_depth = match Relayer::relay_confirmation_depth(&domain_client) { - Ok(depth) => depth, - Err(err) => { - tracing::error!(target: LOG_TARGET, ?err, "Failed to get confirmation depth"); - return; - } - }; - let result = start_relaying_messages( - relay_confirmation_depth, - domain_client.clone(), - |client, block_hash| { - Relayer::submit_messages_from_domain( - client, - &consensus_chain_client, - block_hash, + ChainId::Domain(domain_id), + consensus_chain_client.clone(), + |consensus_chain_client, consensus_block, (domain_block_number, domain_hash)| { + let res = Relayer::submit_messages_from_domain( + domain_id, + &domain_client, + consensus_chain_client, + consensus_block, + domain_hash, &gossip_message_sink, - relay_confirmation_depth, - ) + ); + + if res.is_ok() { + Relayer::store_relayed_domain_block( + &domain_client, + domain_id, + domain_block_number, + domain_hash, + )?; + } + + res }, sync_oracle, - |chain_id, block_number| -> Result { - let ChainId::Domain(domain_id) = chain_id else { - return Err(ApiError::Application(Box::from( - "Should always be running under a Domain".to_string(), - ))); + |consensus_block_number| + -> Result>, ApiError> { + // since a parent mmr leaf is included in its child, + // we process the finalized block's parent instead since we know parent is implicitly finalized + // so we ensure the state of the parent is available here + let consensus_block_number = match consensus_block_number.checked_sub(&One::one()) { + None => return Ok(None), + Some(number) => number }; - // short circuit if the domain state is unavailable to relay messages. - if !is_state_available(&domain_state_pruning, &domain_client, block_number) { - return Ok(false); + if !is_state_available( + &consensus_state_pruning, + &consensus_chain_client, + consensus_block_number, + ) { + return Ok(None); } + let consensus_hash_to_process = consensus_chain_client + .hash(consensus_block_number)? + .ok_or(ApiError::UnknownBlock(format!("Missing Hash for block number: {consensus_block_number:?}")))?; let api = consensus_chain_client.runtime_api(); - let at = consensus_chain_client.info().best_hash; - Ok(api - .oldest_unconfirmed_receipt_number(at, domain_id)? - // ensure block number is at least the oldest tracked number - .map(|oldest_tracked_number| block_number >= oldest_tracked_number) - .unwrap_or(false)) + let confirmed_domain_block = + api.latest_confirmed_domain_block(consensus_hash_to_process, domain_id)?; + + if let Some((domain_block_number, domain_block_hash)) = confirmed_domain_block { + // short circuit if the domain state is unavailable to relay messages. + if !is_state_available(&domain_state_pruning, &domain_client, domain_block_number) { + return Ok(None); + } + + // check if this domain block is already relayed + if Relayer::fetch_domains_blocks_relayed_at( + &domain_client, + domain_id, + domain_block_number, + ) + .contains(&domain_block_hash) + { + return Ok(None); + } + + tracing::debug!( + target: LOG_TARGET, + "Domain block: {domain_block_number:?} and {domain_block_hash:?} confirmed at Consensus block {consensus_block_number:?}" + ); + + Ok(confirmed_domain_block) + } else { + // if there is not confirmed domain block for this domain, skip + Ok(None) + } }, ) - .await; + .await; if let Err(err) = result { tracing::error!( target: LOG_TARGET, @@ -160,64 +205,51 @@ where } } -async fn start_relaying_messages( - relay_confirmation_depth: NumberFor, - client: Arc, +async fn start_relaying_messages( + chain_id: ChainId, + consensus_client: Arc, message_processor: MP, sync_oracle: SO, can_relay_message_from_block: CRM, ) -> Result<(), Error> where - Block: BlockT, - Client: BlockchainEvents - + HeaderBackend + CBlock: BlockT, + CClient: BlockchainEvents + + HeaderBackend + AuxStore - + ProofProvider - + ProvideRuntimeApi, - Client::Api: RelayerApi>, - MP: Fn(&Arc, Block::Hash) -> Result<(), Error>, + + ProofProvider + + ProvideRuntimeApi, + MP: Fn(&Arc, (NumberFor, CBlock::Hash), ExtraData) -> Result<(), Error>, SO: SyncOracle, - CRM: Fn(ChainId, NumberFor) -> Result, + CRM: Fn(NumberFor) -> Result, ApiError>, { - let chain_id = Relayer::chain_id(&client)?; tracing::info!( target: LOG_TARGET, "Starting relayer for chain: {:?}", chain_id, ); - let mut chain_block_import = client.import_notification_stream(); + let mut chain_block_finalization = consensus_client.finality_notification_stream(); // from the start block, start processing all the messages assigned - // wait for new block import of chain, + // wait for new block finalization of the chain, // then fetch new messages in the block // construct proof of each message to be relayed // submit XDM as unsigned extrinsic. - while let Some(block) = chain_block_import.next().await { + while let Some(block) = chain_block_finalization.next().await { // if the client is in major sync, wait until sync is complete if sync_oracle.is_major_syncing() { tracing::debug!(target: LOG_TARGET, "Client is in major sync. Skipping..."); continue; } - let relay_block_until = match block.header.number().checked_sub(&relay_confirmation_depth) { - None => { - // not enough confirmed blocks. - tracing::debug!( - target: LOG_TARGET, - "Not enough confirmed blocks for domain: {:?}. Skipping...", - chain_id - ); - continue; - } - Some(confirmed_block) => confirmed_block, - }; - let (number, hash) = (*block.header.number(), block.header.hash()); - let blocks_to_process: Vec<(NumberFor, Block::Hash)> = - Relayer::fetch_unprocessed_blocks_until(&client, chain_id, number, hash)? - .into_iter() - .filter(|(number, _)| *number <= relay_block_until) - .collect(); + let blocks_to_process: Vec<(NumberFor, CBlock::Hash)> = + Relayer::fetch_unprocessed_consensus_blocks_until( + &consensus_client, + chain_id, + number, + hash, + )?; for (number, hash) in blocks_to_process { tracing::debug!( @@ -228,40 +260,35 @@ where // check if the message is ready to be relayed. // if not, the node is lagging behind and/or there is no way to generate a proof. // mark this block processed and continue to next one. - if !can_relay_message_from_block(chain_id, number)? { - Relayer::store_relayed_block(&client, chain_id, number, hash)?; - tracing::debug!( - target: LOG_TARGET, - "Chain({chain_id:?}) messages in the Block ({number:?}, {hash:?}) cannot be relayed. Skipping...", - ); - } else { - match message_processor(&client, hash) { + if let Some(extra_data) = can_relay_message_from_block(number)? { + match message_processor(&consensus_client, (number, hash), extra_data) { Ok(_) => { - Relayer::store_relayed_block(&client, chain_id, number, hash)?; + Relayer::store_relayed_consensus_block( + &consensus_client, + chain_id, + number, + hash, + )?; tracing::debug!( target: LOG_TARGET, "Messages from {chain_id:?} at block({number:?}, {hash:?}) are processed." ) } Err(err) => { - match err { - Error::DomainNonConfirmedOnConsensusChain => { - tracing::debug!( - target: LOG_TARGET, - "Waiting for Domain[{chain_id:?}] block({number:?}, {hash:?}) to be confirmed on Consensus chain." - ) - } - _ => { - tracing::error!( - target: LOG_TARGET, - ?err, - "Failed to submit messages from the chain {chain_id:?} at the block ({number:?}, {hash:?})" - ); - } - } + tracing::error!( + target: LOG_TARGET, + ?err, + "Failed to submit messages from the chain {chain_id:?} at the block ({number:?}, {hash:?})" + ); break; } } + } else { + Relayer::store_relayed_consensus_block(&consensus_client, chain_id, number, hash)?; + tracing::debug!( + target: LOG_TARGET, + "Chain({chain_id:?}) messages in the Block ({number:?}, {hash:?}) cannot be relayed. Skipping...", + ); } } } diff --git a/domains/pallets/messenger/src/lib.rs b/domains/pallets/messenger/src/lib.rs index 155e02d1f8..dd76252cba 100644 --- a/domains/pallets/messenger/src/lib.rs +++ b/domains/pallets/messenger/src/lib.rs @@ -114,8 +114,8 @@ mod pallet { use sp_domains::proof_provider_and_verifier::{StorageProofVerifier, VerificationError}; use sp_messenger::endpoint::{Endpoint, EndpointHandler, EndpointRequest, Sender}; use sp_messenger::messages::{ - ChainId, CrossDomainMessage, InitiateChannelParams, Message, MessageId, MessageWeightTag, - Payload, ProtocolMessageRequest, RequestResponse, VersionedPayload, + ChainId, CrossDomainMessage, InitiateChannelParams, Message, MessageId, MessageKey, + MessageWeightTag, Payload, ProtocolMessageRequest, RequestResponse, VersionedPayload, }; use sp_messenger::{MmrProofVerifier, OnXDMRewards, StorageKeys}; use sp_mmr_primitives::EncodableOpaqueLeaf; @@ -735,8 +735,8 @@ mod pallet { // derive the key as stored on the src_chain. let key = StorageKey( T::StorageKeys::outbox_storage_key( - T::SelfChainId::get(), - (xdm.channel_id, xdm.nonce), + xdm.src_chain_id, + (T::SelfChainId::get(), xdm.channel_id, xdm.nonce), ) .ok_or(UnknownTransaction::CannotLookup)?, ); @@ -817,8 +817,8 @@ mod pallet { // derive the key as stored on the src_chain. let key = StorageKey( T::StorageKeys::inbox_responses_storage_key( - T::SelfChainId::get(), - (xdm.channel_id, xdm.nonce), + xdm.src_chain_id, + (T::SelfChainId::get(), xdm.channel_id, xdm.nonce), ) .ok_or(UnknownTransaction::CannotLookup)?, ); @@ -883,7 +883,7 @@ mod pallet { .map_err(|err| { log::error!( target: "runtime::messenger", - "Failed to verify storage proof: {:?}", + "Failed to verify storage proof for confirmed Domain block: {:?}", err ); TransactionValidityError::Invalid(InvalidTransaction::BadProof) @@ -903,7 +903,7 @@ mod pallet { .map_err(|err| { log::error!( target: "runtime::messenger", - "Failed to verify storage proof: {:?}", + "Failed to verify storage proof for message: {:?}", err ); TransactionValidityError::Invalid(InvalidTransaction::BadProof) @@ -912,14 +912,12 @@ mod pallet { Ok(msg) } - pub fn outbox_storage_key(message_id: MessageId) -> Vec { - let (channel_id, nonce) = message_id; - Outbox::::hashed_key_for((T::SelfChainId::get(), channel_id, nonce)) + pub fn outbox_storage_key(message_key: MessageKey) -> Vec { + Outbox::::hashed_key_for(message_key) } - pub fn inbox_response_storage_key(message_id: MessageId) -> Vec { - let (channel_id, nonce) = message_id; - InboxResponses::::hashed_key_for((T::SelfChainId::get(), channel_id, nonce)) + pub fn inbox_response_storage_key(message_key: MessageKey) -> Vec { + InboxResponses::::hashed_key_for(message_key) } } } diff --git a/domains/pallets/messenger/src/mock.rs b/domains/pallets/messenger/src/mock.rs index 29adda0890..3cd4989b80 100644 --- a/domains/pallets/messenger/src/mock.rs +++ b/domains/pallets/messenger/src/mock.rs @@ -169,7 +169,7 @@ macro_rules! impl_runtime { } pub const USER_ACCOUNT: AccountId = 1; - pub const USER_INITIAL_BALANCE: Balance = 1000; + pub const USER_INITIAL_BALANCE: Balance = 500000000; pub fn new_test_ext() -> TestExternalities { let mut t = frame_system::GenesisConfig::::default() diff --git a/domains/pallets/messenger/src/tests.rs b/domains/pallets/messenger/src/tests.rs index 97f100871e..1dd8e9ed40 100644 --- a/domains/pallets/messenger/src/tests.rs +++ b/domains/pallets/messenger/src/tests.rs @@ -7,7 +7,7 @@ use crate::mock::{ }; use crate::{ Channel, ChannelId, ChannelState, Channels, Error, FeeModel, Inbox, InboxResponses, Nonce, - Outbox, OutboxMessageResult, OutboxResponses, U256, + Outbox, OutboxMessageResult, OutboxResponses, Pallet, U256, }; use frame_support::{assert_err, assert_ok}; use pallet_transporter::Location; @@ -16,11 +16,12 @@ use sp_core::{Blake2Hasher, H256}; use sp_domains::proof_provider_and_verifier::{StorageProofVerifier, VerificationError}; use sp_messenger::endpoint::{Endpoint, EndpointPayload, EndpointRequest, Sender}; use sp_messenger::messages::{ - ChainId, ConsensusChainMmrLeafProof, CrossDomainMessage, InitiateChannelParams, Payload, Proof, - ProtocolMessageRequest, RequestResponse, VersionedPayload, + ChainId, ConsensusChainMmrLeafProof, CrossDomainMessage, InitiateChannelParams, + MessageWeightTag, Payload, Proof, ProtocolMessageRequest, RequestResponse, VersionedPayload, }; use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof as MmrProof}; -use sp_runtime::traits::{Convert, ValidateUnsigned}; +use sp_runtime::traits::Convert; +use sp_trie::StorageProof; fn create_channel(chain_id: ChainId, channel_id: ChannelId, fee_model: FeeModel) { let params = InitiateChannelParams { @@ -186,7 +187,6 @@ fn test_close_open_channel() { } #[test] -#[ignore] fn test_storage_proof_verification_invalid() { let mut t = new_chain_a_ext(); let chain_id = 2.into(); @@ -196,23 +196,18 @@ fn test_storage_proof_verification_invalid() { assert_ok!(Messenger::do_open_channel(chain_id, channel_id)); }); - let (_, _, storage_proof) = + let (_, storage_key, storage_proof) = crate::mock::storage_proof_of_channels::(t.as_backend(), chain_id, channel_id); - let proof = Proof::::Consensus { - consensus_chain_mmr_proof: default_consensus_proof(), - message_proof: storage_proof, - }; let res: Result, VerificationError> = StorageProofVerifier::::get_decoded_value( &H256::zero(), - proof.message_proof(), - StorageKey(vec![]), + storage_proof, + storage_key, ); assert_err!(res, VerificationError::InvalidProof); } #[test] -#[ignore] fn test_storage_proof_verification_missing_value() { let mut t = new_chain_a_ext(); let chain_id = 2.into(); @@ -222,23 +217,18 @@ fn test_storage_proof_verification_missing_value() { assert_ok!(Messenger::do_open_channel(chain_id, channel_id)); }); - let (_state_root, storage_key, storage_proof) = + let (state_root, _, storage_proof) = crate::mock::storage_proof_of_channels::(t.as_backend(), chain_id, U256::one()); - let proof = Proof::::Consensus { - consensus_chain_mmr_proof: default_consensus_proof(), - message_proof: storage_proof, - }; let res: Result, VerificationError> = StorageProofVerifier::::get_decoded_value( - &H256::zero(), - proof.message_proof(), - storage_key, + &state_root, + storage_proof, + StorageKey(vec![]), ); assert_err!(res, VerificationError::MissingValue); } #[test] -#[ignore] fn test_storage_proof_verification() { let mut t = new_chain_a_ext(); let chain_id = 2.into(); @@ -250,16 +240,12 @@ fn test_storage_proof_verification() { expected_channel = Channels::::get(chain_id, channel_id); }); - let (_state_root, storage_key, storage_proof) = + let (state_root, storage_key, storage_proof) = crate::mock::storage_proof_of_channels::(t.as_backend(), chain_id, channel_id); - let proof = Proof::::Consensus { - consensus_chain_mmr_proof: default_consensus_proof(), - message_proof: storage_proof, - }; let res: Result, VerificationError> = StorageProofVerifier::::get_decoded_value( - &H256::zero(), - proof.message_proof(), + &state_root, + storage_proof, storage_key, ); @@ -287,6 +273,9 @@ fn open_channel_between_chains( chain_b_test_ext, channel_id, Nonce::zero(), + true, + MessageWeightTag::ProtocolChannelOpen, + None, ); // check channel state be open on chain_b @@ -363,13 +352,6 @@ fn send_message_between_chains( }, ); assert_ok!(resp); - chain_a::System::assert_last_event(RuntimeEvent::Messenger( - crate::Event::::OutboxMessage { - chain_id: chain_b_id, - channel_id, - nonce: Nonce::one(), - }, - )); }); channel_relay_request_and_response( @@ -377,6 +359,9 @@ fn send_message_between_chains( chain_b_test_ext, channel_id, Nonce::one(), + false, + Default::default(), + Some(Endpoint::Id(0)), ); // check state on chain_b @@ -424,6 +409,9 @@ fn close_channel_between_chains( chain_b_test_ext, channel_id, Nonce::one(), + true, + MessageWeightTag::ProtocolChannelClose, + None, ); // check channel state be close on chain_b @@ -481,16 +469,52 @@ fn close_channel_between_chains( }) } +fn force_toggle_channel_state( + dst_chain_id: ChainId, + channel_id: ChannelId, + toggle: bool, +) { + let fee_model = FeeModel { + relay_fee: Default::default(), + }; + let init_params = InitiateChannelParams { + max_outgoing_messages: 100, + fee_model, + }; + + let channel = Pallet::::channels(dst_chain_id, channel_id).unwrap_or_else(|| { + Pallet::::do_init_channel(dst_chain_id, init_params).unwrap(); + Pallet::::channels(dst_chain_id, channel_id).unwrap() + }); + + if !toggle { + return; + } + + if channel.state == ChannelState::Initiated { + Pallet::::do_open_channel(dst_chain_id, channel_id).unwrap(); + } + + if channel.state == ChannelState::Open { + Pallet::::do_close_channel(dst_chain_id, channel_id).unwrap(); + } +} + fn channel_relay_request_and_response( chain_a_test_ext: &mut TestExternalities, chain_b_test_ext: &mut TestExternalities, channel_id: ChannelId, nonce: Nonce, + toggle_channel_state: bool, + weight_tag: MessageWeightTag, + maybe_endpoint: Option, ) { let chain_a_id = chain_a::SelfChainId::get(); let chain_b_id = chain_b::SelfChainId::get(); // relay message to chain_b + let msg = chain_a_test_ext + .execute_with(|| Outbox::::get((chain_b_id, channel_id, nonce)).unwrap()); let (_state_root, _key, message_proof) = storage_proof_of_outbox_messages::( chain_a_test_ext.as_backend(), chain_b_id, @@ -503,32 +527,28 @@ fn channel_relay_request_and_response( dst_chain_id: chain_b_id, channel_id, nonce, - proof: Proof::Consensus { + proof: Proof::Domain { consensus_chain_mmr_proof: default_consensus_proof(), + domain_proof: StorageProof::empty(), message_proof, }, - weight_tag: Default::default(), + weight_tag: maybe_endpoint + .clone() + .map(MessageWeightTag::EndpointRequest) + .unwrap_or(weight_tag.clone()), }; chain_b_test_ext.execute_with(|| { - // validate the message - let pre_check = - crate::Pallet::::pre_dispatch(&crate::Call::relay_message { - msg: xdm.clone(), - }); - assert_ok!(pre_check); + force_toggle_channel_state::( + chain_a_id, + channel_id, + toggle_channel_state, + ); + Inbox::::set(Some(msg)); // process inbox message let result = chain_b::Messenger::relay_message(chain_b::RuntimeOrigin::none(), xdm); assert_ok!(result); - chain_b::System::assert_has_event(chain_b::RuntimeEvent::Messenger(crate::Event::< - chain_b::Runtime, - >::InboxMessage { - chain_id: chain_a_id, - channel_id, - nonce, - })); - chain_b::System::assert_has_event(chain_b::RuntimeEvent::Messenger(crate::Event::< chain_b::Runtime, >::InboxMessageResponse { @@ -555,6 +575,10 @@ fn channel_relay_request_and_response( nonce, ); + let msg = chain_b_test_ext.execute_with(|| { + InboxResponses::::get((chain_a_id, channel_id, nonce)).unwrap() + }); + let xdm = CrossDomainMessage { src_chain_id: chain_b_id, dst_chain_id: chain_a_id, @@ -564,15 +588,18 @@ fn channel_relay_request_and_response( consensus_chain_mmr_proof: default_consensus_proof(), message_proof, }, - weight_tag: Default::default(), + weight_tag: maybe_endpoint + .clone() + .map(MessageWeightTag::EndpointResponse) + .unwrap_or(weight_tag), }; chain_a_test_ext.execute_with(|| { - // validate message response - let pre_check = - crate::Pallet::::pre_dispatch(&crate::Call::relay_message_response { - msg: xdm.clone(), - }); - assert_ok!(pre_check); + force_toggle_channel_state::( + chain_b_id, + channel_id, + toggle_channel_state, + ); + OutboxResponses::::set(Some(msg)); // process outbox message response let result = @@ -598,7 +625,6 @@ fn channel_relay_request_and_response( } #[test] -#[ignore] fn test_open_channel_between_chains() { let mut chain_a_test_ext = chain_a::new_test_ext(); let mut chain_b_test_ext = chain_b::new_test_ext(); @@ -612,7 +638,6 @@ fn test_open_channel_between_chains() { } #[test] -#[ignore] fn test_close_channel_between_chains() { let mut chain_a_test_ext = chain_a::new_test_ext(); let mut chain_b_test_ext = chain_b::new_test_ext(); @@ -629,7 +654,6 @@ fn test_close_channel_between_chains() { } #[test] -#[ignore] fn test_send_message_between_chains() { let mut chain_a_test_ext = chain_a::new_test_ext(); let mut chain_b_test_ext = chain_b::new_test_ext(); @@ -677,11 +701,6 @@ fn initiate_transfer_on_chain(chain_a_ext: &mut TestExternalities) { channel_id: U256::zero(), nonce: U256::one(), })); - let fee_model = chain_b::Messenger::channels(chain_b::SelfChainId::get(), U256::zero()) - .unwrap_or_default() - .fee; - let fees = fee_model.relay_fee; - assert_eq!(chain_a::Balances::free_balance(account_id), 500 - fees); assert!(chain_a::Transporter::outgoing_transfers( chain_b::SelfChainId::get(), (U256::zero(), U256::one()), @@ -707,14 +726,6 @@ fn verify_transfer_on_chain( message_id: (U256::zero(), U256::one()), }, )); - chain_a::System::assert_has_event(chain_a::RuntimeEvent::Messenger( - crate::Event::::OutboxMessageResponse { - chain_id: chain_b::SelfChainId::get(), - channel_id: U256::zero(), - nonce: U256::one(), - }, - )); - assert_eq!(chain_a::Balances::free_balance(account_id), 496); assert!(chain_a::Transporter::outgoing_transfers( chain_b::SelfChainId::get(), (U256::zero(), U256::one()), @@ -739,12 +750,11 @@ fn verify_transfer_on_chain( channel_id: U256::zero(), nonce: U256::one(), })); - assert_eq!(chain_b::Balances::free_balance(account_id), 1500); + assert_eq!(chain_b::Balances::free_balance(account_id), 500000500); }) } #[test] -#[ignore] fn test_transport_funds_between_chains() { let mut chain_a_test_ext = chain_a::new_test_ext(); let mut chain_b_test_ext = chain_b::new_test_ext(); @@ -766,6 +776,9 @@ fn test_transport_funds_between_chains() { &mut chain_b_test_ext, channel_id, Nonce::one(), + false, + Default::default(), + Some(Endpoint::Id(100)), ); // post check @@ -773,7 +786,6 @@ fn test_transport_funds_between_chains() { } #[test] -#[ignore] fn test_transport_funds_between_chains_failed_low_balance() { let mut chain_a_test_ext = chain_a::new_test_ext(); let mut chain_b_test_ext = chain_b::new_test_ext(); diff --git a/domains/primitives/messenger-host-functions/src/host_functions.rs b/domains/primitives/messenger-host-functions/src/host_functions.rs index 5f85fe140e..87845bae8d 100644 --- a/domains/primitives/messenger-host-functions/src/host_functions.rs +++ b/domains/primitives/messenger-host-functions/src/host_functions.rs @@ -55,12 +55,13 @@ where DomainBlock: BlockT, Client: HeaderBackend + ProvideRuntimeApi, Client::Api: DomainsApi, + Executor: CodeExecutor + RuntimeVersionOf, { - fn get_domain_runtime_code( + fn get_domain_runtime( &self, consensus_block_hash: Block::Hash, domain_id: DomainId, - ) -> Option> { + ) -> Option> { let runtime_api = self.consensus_client.runtime_api(); // Use the parent hash to get the actual used domain runtime code // TODO: update once we can get the actual used domain runtime code by `consensus_block_hash` @@ -69,10 +70,22 @@ where .header(consensus_block_hash) .ok() .flatten()?; - runtime_api + let domain_runtime = runtime_api .domain_runtime_code(*consensus_block_header.parent_hash(), domain_id) .ok() + .flatten()?; + + // we need the initial state here so that SelfChainId is initialised on domain + let domain_state = runtime_api + .domain_instance_data(*consensus_block_header.parent_hash(), domain_id) + .ok() .flatten() + .map(|(data, _)| data.raw_genesis.into_storage())?; + let mut domain_stateless_runtime = + StatelessRuntime::::new(self.executor.clone(), domain_runtime.into()); + + domain_stateless_runtime.set_storage(domain_state); + Some(domain_stateless_runtime) } } @@ -94,41 +107,33 @@ where .confirmed_domain_block_storage_key(best_hash, domain_id) .map(Some), StorageKeyRequest::OutboxStorageKey { - message_id, + message_key, chain_id: ChainId::Consensus, } => runtime_api - .outbox_storage_key(best_hash, message_id) + .outbox_storage_key(best_hash, message_key) .map(Some), StorageKeyRequest::OutboxStorageKey { - message_id, + message_key, chain_id: ChainId::Domain(domain_id), } => { - let runtime_code = self.get_domain_runtime_code(best_hash, domain_id)?; - let domain_stateless_runtime = StatelessRuntime::::new( - self.executor.clone(), - runtime_code.into(), - ); + let domain_stateless_runtime = self.get_domain_runtime(best_hash, domain_id)?; domain_stateless_runtime - .outbox_storage_key(message_id) + .outbox_storage_key(message_key) .map(Some) } StorageKeyRequest::InboxResponseStorageKey { - message_id, + message_key, chain_id: ChainId::Consensus, } => runtime_api - .inbox_response_storage_key(best_hash, message_id) + .inbox_response_storage_key(best_hash, message_key) .map(Some), StorageKeyRequest::InboxResponseStorageKey { - message_id, + message_key, chain_id: ChainId::Domain(domain_id), } => { - let runtime_code = self.get_domain_runtime_code(best_hash, domain_id)?; - let domain_stateless_runtime = StatelessRuntime::::new( - self.executor.clone(), - runtime_code.into(), - ); + let domain_stateless_runtime = self.get_domain_runtime(best_hash, domain_id)?; domain_stateless_runtime - .inbox_response_storage_key(message_id) + .inbox_response_storage_key(message_key) .map(Some) } } diff --git a/domains/primitives/messenger-host-functions/src/lib.rs b/domains/primitives/messenger-host-functions/src/lib.rs index eceaf11082..da9deb7e53 100644 --- a/domains/primitives/messenger-host-functions/src/lib.rs +++ b/domains/primitives/messenger-host-functions/src/lib.rs @@ -32,7 +32,7 @@ pub use runtime_interface::messenger_runtime_interface::get_storage_key; pub use runtime_interface::messenger_runtime_interface::HostFunctions; use scale_info::TypeInfo; use sp_domains::DomainId; -use sp_messenger::messages::{ChainId, MessageId}; +use sp_messenger::messages::{ChainId, MessageKey}; use sp_runtime_interface::pass_by; use sp_runtime_interface::pass_by::PassBy; @@ -43,12 +43,12 @@ pub enum StorageKeyRequest { /// Request to get Outbox storage key for given chain and message. OutboxStorageKey { chain_id: ChainId, - message_id: MessageId, + message_key: MessageKey, }, /// Request to get Inbox response storage key for given chain and message. InboxResponseStorageKey { chain_id: ChainId, - message_id: MessageId, + message_key: MessageKey, }, } diff --git a/domains/primitives/messenger/src/lib.rs b/domains/primitives/messenger/src/lib.rs index 979b6667c1..f7cc273567 100644 --- a/domains/primitives/messenger/src/lib.rs +++ b/domains/primitives/messenger/src/lib.rs @@ -23,6 +23,7 @@ pub mod messages; #[cfg(not(feature = "std"))] extern crate alloc; +use crate::messages::MessageKey; #[cfg(not(feature = "std"))] use alloc::vec::Vec; use codec::{Decode, Encode}; @@ -63,10 +64,10 @@ pub trait StorageKeys { fn confirmed_domain_block_storage_key(domain_id: DomainId) -> Option>; /// Returns the outbox storage key for given chain. - fn outbox_storage_key(chain_id: ChainId, message_id: MessageId) -> Option>; + fn outbox_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option>; /// Returns the inbox responses storage key for given chain. - fn inbox_responses_storage_key(chain_id: ChainId, message_id: MessageId) -> Option>; + fn inbox_responses_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option>; } impl StorageKeys for () { @@ -74,39 +75,37 @@ impl StorageKeys for () { None } - fn outbox_storage_key(_chain_id: ChainId, _message_id: MessageId) -> Option> { + fn outbox_storage_key(_chain_id: ChainId, _message_key: MessageKey) -> Option> { None } - fn inbox_responses_storage_key(_chain_id: ChainId, _message_id: MessageId) -> Option> { + fn inbox_responses_storage_key( + _chain_id: ChainId, + _message_key: MessageKey, + ) -> Option> { None } } sp_api::decl_runtime_apis! { /// Api useful for relayers to fetch messages and submit transactions. - pub trait RelayerApi< BlockNumber> + pub trait RelayerApi where - BlockNumber: Encode + Decode + BlockNumber: Encode + Decode, + CHash: Encode + Decode, { - /// Returns the the chain_id of the Runtime. - fn chain_id() -> ChainId; - - /// Returns the confirmation depth to relay message. - fn relay_confirmation_depth() -> BlockNumber; - /// Returns all the outbox and inbox responses to deliver. /// Storage key is used to generate the storage proof for the message. fn block_messages() -> BlockMessagesWithStorageKey; /// Constructs an outbox message to the dst_chain as an unsigned extrinsic. fn outbox_message_unsigned( - msg: CrossDomainMessage, + msg: CrossDomainMessage, ) -> Option; /// Constructs an inbox response message to the dst_chain as an unsigned extrinsic. fn inbox_response_message_unsigned( - msg: CrossDomainMessage, + msg: CrossDomainMessage, ) -> Option; /// Returns true if the outbox message is ready to be relayed to dst_chain. @@ -130,9 +129,9 @@ sp_api::decl_runtime_apis! { fn confirmed_domain_block_storage_key(domain_id: DomainId) -> Vec; /// Returns storage key for outbox for a given message_id. - fn outbox_storage_key(message_id: MessageId) -> Vec; + fn outbox_storage_key(message_key: MessageKey) -> Vec; /// Returns storage key for inbox response for a given message_id. - fn inbox_response_storage_key(message_id: MessageId) -> Vec; + fn inbox_response_storage_key(message_key: MessageKey) -> Vec; } } diff --git a/domains/primitives/messenger/src/messages.rs b/domains/primitives/messenger/src/messages.rs index 2463c213dc..b2a5f3644c 100644 --- a/domains/primitives/messenger/src/messages.rs +++ b/domains/primitives/messenger/src/messages.rs @@ -22,6 +22,9 @@ pub type Nonce = U256; /// Unique Id of a message between two chains. pub type MessageId = (ChannelId, Nonce); +/// Unique message key for Outbox and Inbox responses +pub type MessageKey = (ChainId, ChannelId, Nonce); + /// Fee model to send a request and receive a response from another chain. #[derive(Default, Debug, Encode, Decode, Clone, Copy, Eq, PartialEq, TypeInfo)] pub struct FeeModel { @@ -253,10 +256,10 @@ pub struct BlockMessagesWithStorageKey { pub inbox_responses: Vec, } -impl CrossDomainMessage { +impl CrossDomainMessage { pub fn from_relayer_msg_with_proof( r_msg: BlockMessageWithStorageKey, - proof: Proof, + proof: Proof, ) -> Self { CrossDomainMessage { src_chain_id: r_msg.src_chain_id, diff --git a/domains/runtime/evm/src/lib.rs b/domains/runtime/evm/src/lib.rs index 2952cc5198..92d9233855 100644 --- a/domains/runtime/evm/src/lib.rs +++ b/domains/runtime/evm/src/lib.rs @@ -48,7 +48,9 @@ use sp_core::crypto::KeyTypeId; use sp_core::{Get, OpaqueMetadata, H160, H256, U256}; use sp_domains::{DomainId, Transfers}; use sp_messenger::endpoint::{Endpoint, EndpointHandler as EndpointHandlerT, EndpointId}; -use sp_messenger::messages::{BlockMessagesWithStorageKey, ChainId, CrossDomainMessage, MessageId}; +use sp_messenger::messages::{ + BlockMessagesWithStorageKey, ChainId, CrossDomainMessage, MessageId, MessageKey, +}; use sp_messenger_host_functions::{get_storage_key, StorageKeyRequest}; use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof}; use sp_runtime::generic::Era; @@ -436,17 +438,17 @@ impl sp_messenger::StorageKeys for StorageKeys { get_storage_key(StorageKeyRequest::ConfirmedDomainBlockStorageKey(domain_id)) } - fn outbox_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn outbox_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::OutboxStorageKey { chain_id, - message_id, + message_key, }) } - fn inbox_responses_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn inbox_responses_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::InboxResponseStorageKey { chain_id, - message_id, + message_key, }) } } @@ -1060,24 +1062,16 @@ impl_runtime_apis! { vec![] } - fn outbox_storage_key(message_id: MessageId) -> Vec { - Messenger::outbox_storage_key(message_id) + fn outbox_storage_key(message_key: MessageKey) -> Vec { + Messenger::outbox_storage_key(message_key) } - fn inbox_response_storage_key(message_id: MessageId) -> Vec { - Messenger::inbox_response_storage_key(message_id) + fn inbox_response_storage_key(message_key: MessageKey) -> Vec { + Messenger::inbox_response_storage_key(message_key) } } - impl sp_messenger::RelayerApi for Runtime { - fn chain_id() -> ChainId { - SelfChainId::get() - } - - fn relay_confirmation_depth() -> BlockNumber { - RelayConfirmationDepth::get() - } - + impl sp_messenger::RelayerApi for Runtime { fn block_messages() -> BlockMessagesWithStorageKey { Messenger::get_block_messages() } diff --git a/domains/service/src/domain.rs b/domains/service/src/domain.rs index c6c1f9727e..d21f6693aa 100644 --- a/domains/service/src/domain.rs +++ b/domains/service/src/domain.rs @@ -11,7 +11,8 @@ use futures::channel::mpsc; use futures::Stream; use pallet_transaction_payment_rpc::TransactionPaymentRuntimeApi; use sc_client_api::{ - BlockBackend, BlockImportNotification, BlockchainEvents, ExecutorProvider, ProofProvider, + AuxStore, BlockBackend, BlockImportNotification, BlockchainEvents, ExecutorProvider, + ProofProvider, }; use sc_consensus::SharedBlockImport; use sc_domains::{ExtensionsFactory, RuntimeExecutor}; @@ -19,7 +20,7 @@ use sc_network::NetworkPeers; use sc_rpc_api::DenyUnsafe; use sc_service::{ BuildNetworkParams, Configuration as ServiceConfiguration, NetworkStarter, PartialComponents, - SpawnTasksParams, TFullBackend, TaskManager, + PruningMode, SpawnTasksParams, TFullBackend, TaskManager, }; use sc_telemetry::{Telemetry, TelemetryWorker, TelemetryWorkerHandle}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; @@ -85,7 +86,7 @@ where + TransactionPaymentRuntimeApi + DomainCoreApi + MessengerApi> - + RelayerApi>, + + RelayerApi, CBlock::Hash>, AccountId: Encode + Decode, { /// Task manager. @@ -231,6 +232,7 @@ where pub domain_message_receiver: TracingUnboundedReceiver, pub provider: Provider, pub skip_empty_bundle_production: bool, + pub consensus_state_pruning: PruningMode, pub skip_out_of_order_slot: bool, } @@ -268,11 +270,12 @@ where + ProofProvider + ProvideRuntimeApi + BlockchainEvents + + AuxStore + Send + Sync + 'static, CClient::Api: DomainsApi - + RelayerApi> + + RelayerApi, CBlock::Hash> + MessengerApi> + BundleProducerElectionApi + FraudProofApi @@ -292,7 +295,7 @@ where + TaggedTransactionQueue + AccountNonceApi + TransactionPaymentRuntimeApi - + RelayerApi>, + + RelayerApi, CBlock::Hash>, AccountId: DeserializeOwned + Encode + Decode @@ -329,6 +332,7 @@ where domain_message_receiver, provider, skip_empty_bundle_production, + consensus_state_pruning, skip_out_of_order_slot, } = domain_params; @@ -455,7 +459,9 @@ where if is_authority { let relayer_worker = domain_client_message_relayer::worker::relay_domain_messages( + domain_id, consensus_client.clone(), + consensus_state_pruning, client.clone(), domain_state_pruning, // domain relayer will use consensus chain sync oracle instead of domain sync orcle diff --git a/domains/test/runtime/evm/src/lib.rs b/domains/test/runtime/evm/src/lib.rs index ab8b44d1d9..cb47d63293 100644 --- a/domains/test/runtime/evm/src/lib.rs +++ b/domains/test/runtime/evm/src/lib.rs @@ -46,7 +46,7 @@ use sp_core::{Get, OpaqueMetadata, H160, H256, U256}; use sp_domains::{DomainId, Transfers}; use sp_messenger::endpoint::{Endpoint, EndpointHandler as EndpointHandlerT, EndpointId}; use sp_messenger::messages::{ - BlockMessagesWithStorageKey, ChainId, ChannelId, CrossDomainMessage, MessageId, + BlockMessagesWithStorageKey, ChainId, ChannelId, CrossDomainMessage, MessageId, MessageKey, }; use sp_messenger_host_functions::{get_storage_key, StorageKeyRequest}; use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof}; @@ -425,17 +425,17 @@ impl sp_messenger::StorageKeys for StorageKeys { get_storage_key(StorageKeyRequest::ConfirmedDomainBlockStorageKey(domain_id)) } - fn outbox_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn outbox_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::OutboxStorageKey { chain_id, - message_id, + message_key, }) } - fn inbox_responses_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn inbox_responses_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::InboxResponseStorageKey { chain_id, - message_id, + message_key, }) } } @@ -1028,24 +1028,16 @@ impl_runtime_apis! { vec![] } - fn outbox_storage_key(message_id: MessageId) -> Vec { - Messenger::outbox_storage_key(message_id) + fn outbox_storage_key(message_key: MessageKey) -> Vec { + Messenger::outbox_storage_key(message_key) } - fn inbox_response_storage_key(message_id: MessageId) -> Vec { - Messenger::inbox_response_storage_key(message_id) + fn inbox_response_storage_key(message_key: MessageKey) -> Vec { + Messenger::inbox_response_storage_key(message_key) } } - impl sp_messenger::RelayerApi for Runtime { - fn chain_id() -> ChainId { - SelfChainId::get() - } - - fn relay_confirmation_depth() -> BlockNumber { - RelayConfirmationDepth::get() - } - + impl sp_messenger::RelayerApi for Runtime { fn block_messages() -> BlockMessagesWithStorageKey { Messenger::get_block_messages() } diff --git a/domains/test/service/src/domain.rs b/domains/test/service/src/domain.rs index cdf44afbe7..39341f1e5b 100644 --- a/domains/test/service/src/domain.rs +++ b/domains/test/service/src/domain.rs @@ -22,7 +22,7 @@ use sc_domains::RuntimeExecutor; use sc_network::{NetworkService, NetworkStateInfo}; use sc_network_sync::SyncingService; use sc_service::config::MultiaddrWithPeerId; -use sc_service::{BasePath, Role, RpcHandlers, TFullBackend, TaskManager}; +use sc_service::{BasePath, PruningMode, Role, RpcHandlers, TFullBackend, TaskManager}; use sc_transaction_pool_api::OffchainTransactionPoolFactory; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; use serde::de::DeserializeOwned; @@ -31,10 +31,10 @@ use sp_block_builder::BlockBuilder; use sp_core::{Decode, Encode, H256}; use sp_domains::core_api::DomainCoreApi; use sp_domains::DomainId; -use sp_messenger::messages::ChainId; +use sp_messenger::messages::{ChainId, ChannelId}; use sp_messenger::{MessengerApi, RelayerApi}; use sp_offchain::OffchainWorkerApi; -use sp_runtime::traits::{Dispatchable, NumberFor}; +use sp_runtime::traits::{Block as BlockT, Dispatchable, NumberFor}; use sp_runtime::OpaqueExtrinsic; use sp_session::SessionKeys; use sp_transaction_pool::runtime_api::TaggedTransactionQueue; @@ -86,7 +86,7 @@ where + TaggedTransactionQueue + AccountNonceApi + TransactionPaymentRuntimeApi - + RelayerApi>, + + RelayerApi, ::Hash>, AccountId: Encode + Decode + FromKeyring, { /// The domain id @@ -137,7 +137,7 @@ where + AccountNonceApi + TransactionPaymentRuntimeApi + MessengerApi> - + RelayerApi> + + RelayerApi, ::Hash> + OnchainStateApi + EthereumRuntimeRPCApi, AccountId: DeserializeOwned @@ -229,6 +229,7 @@ where skip_empty_bundle_production, skip_out_of_order_slot: true, maybe_operator_id, + consensus_state_pruning: PruningMode::ArchiveCanonical, }; let domain_node = @@ -385,6 +386,14 @@ where .free_balance(self.client.info().best_hash, account_id) .expect("Fail to get account free balance") } + + /// Returns the open XDM channel for given chain + pub fn get_open_channel_for_chain(&self, chain_id: ChainId) -> Option { + self.client + .runtime_api() + .get_open_channel_for_chain(self.client.info().best_hash, chain_id) + .expect("Fail to get open channel for Chain") + } } /// A builder to create a [`DomainNode`]. diff --git a/test/subspace-test-primitives/Cargo.toml b/test/subspace-test-primitives/Cargo.toml new file mode 100644 index 0000000000..26dce7dd80 --- /dev/null +++ b/test/subspace-test-primitives/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "subspace-test-primitives" +version = "0.1.0" +authors = ["Subspace Labs "] +edition = "2021" +license = "GPL-3.0-or-later" +homepage = "https://subspace.network" +repository = "https://github.com/subspace/subspace" +include = [ + "/src", + "/Cargo.toml", +] + +[dependencies] +codec = { package = "parity-scale-codec", version = "3.4.0", default-features = false, features = ["derive"] } +sp-api = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8", default-features = false } +sp-messenger = { version = "0.1.0", default-features = false, path = "../../domains/primitives/messenger" } +subspace-runtime-primitives = { version = "0.1.0", path = "../../crates/subspace-runtime-primitives", default-features = false } + +[features] +default = ["std"] +std = [ + "sp-api/std", + "sp-messenger/std", + "subspace-runtime-primitives/std", +] diff --git a/test/subspace-test-primitives/src/lib.rs b/test/subspace-test-primitives/src/lib.rs new file mode 100644 index 0000000000..ce2e66d5b8 --- /dev/null +++ b/test/subspace-test-primitives/src/lib.rs @@ -0,0 +1,20 @@ +#![cfg_attr(not(feature = "std"), no_std)] +//! Test primitive crates that expose necessary extensions that are used in tests. + +use codec::{Decode, Encode}; +use sp_messenger::messages::{ChainId, ChannelId}; + +sp_api::decl_runtime_apis! { + /// Api for querying onchain state in the test + pub trait OnchainStateApi + where + AccountId: Encode + Decode, + Balance: Encode + Decode + { + /// Api to get the free balance of the given account + fn free_balance(account_id: AccountId) -> Balance; + + /// Returns the last open channel for a given domain. + fn get_open_channel_for_chain(dst_chain_id: ChainId) -> Option; + } +} diff --git a/test/subspace-test-runtime/Cargo.toml b/test/subspace-test-runtime/Cargo.toml index 901c2c597b..cf9d079d83 100644 --- a/test/subspace-test-runtime/Cargo.toml +++ b/test/subspace-test-runtime/Cargo.toml @@ -58,6 +58,7 @@ sp-version = { version = "22.0.0", default-features = false, git = "https://gith static_assertions = "1.1.0" subspace-core-primitives = { version = "0.1.0", default-features = false, path = "../../crates/subspace-core-primitives" } subspace-runtime-primitives = { version = "0.1.0", default-features = false, path = "../../crates/subspace-runtime-primitives" } +subspace-test-primitives = { version = "0.1.0", default-features = false, path = "../subspace-test-primitives" } # Used for the node template's RPCs frame-system-rpc-runtime-api = { version = "4.0.0-dev", default-features = false, git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } diff --git a/test/subspace-test-runtime/src/lib.rs b/test/subspace-test-runtime/src/lib.rs index d54e4ff8ca..f40dd202aa 100644 --- a/test/subspace-test-runtime/src/lib.rs +++ b/test/subspace-test-runtime/src/lib.rs @@ -59,7 +59,9 @@ use sp_domains::{ }; use sp_domains_fraud_proof::fraud_proof::FraudProof; use sp_messenger::endpoint::{Endpoint, EndpointHandler as EndpointHandlerT, EndpointId}; -use sp_messenger::messages::{BlockMessagesWithStorageKey, ChainId, CrossDomainMessage, MessageId}; +use sp_messenger::messages::{ + BlockMessagesWithStorageKey, ChainId, ChannelId, CrossDomainMessage, MessageId, MessageKey, +}; use sp_messenger_host_functions::{get_storage_key, StorageKeyRequest}; use sp_mmr_primitives::{EncodableOpaqueLeaf, Proof}; use sp_runtime::traits::{ @@ -545,17 +547,17 @@ impl sp_messenger::StorageKeys for StorageKeys { Some(Domains::confirmed_domain_block_storage_key(domain_id)) } - fn outbox_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn outbox_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::OutboxStorageKey { chain_id, - message_id, + message_key, }) } - fn inbox_responses_storage_key(chain_id: ChainId, message_id: MessageId) -> Option> { + fn inbox_responses_storage_key(chain_id: ChainId, message_key: MessageKey) -> Option> { get_storage_key(StorageKeyRequest::InboxResponseStorageKey { chain_id, - message_id, + message_key, }) } } @@ -1298,6 +1300,10 @@ impl_runtime_apis! { DOMAIN_STORAGE_FEE_MULTIPLIER * TransactionFees::transaction_byte_fee() } + fn latest_confirmed_domain_block(domain_id: DomainId) -> Option<(DomainNumber, DomainHash)>{ + Domains::latest_confirmed_domain_block(domain_id) + } + fn is_bad_er_pending_to_prune(domain_id: DomainId, receipt_hash: DomainHash) -> bool { Domains::execution_receipt(receipt_hash).map( |er| Domains::is_bad_er_pending_to_prune(domain_id, er.domain_block_number) @@ -1366,24 +1372,16 @@ impl_runtime_apis! { Domains::confirmed_domain_block_storage_key(domain_id) } - fn outbox_storage_key(message_id: MessageId) -> Vec { - Messenger::outbox_storage_key(message_id) + fn outbox_storage_key(message_key: MessageKey) -> Vec { + Messenger::outbox_storage_key(message_key) } - fn inbox_response_storage_key(message_id: MessageId) -> Vec { - Messenger::inbox_response_storage_key(message_id) + fn inbox_response_storage_key(message_key: MessageKey) -> Vec { + Messenger::inbox_response_storage_key(message_key) } } - impl sp_messenger::RelayerApi for Runtime { - fn chain_id() -> ChainId { - SelfChainId::get() - } - - fn relay_confirmation_depth() -> BlockNumber { - RelayConfirmationDepth::get() - } - + impl sp_messenger::RelayerApi::Hash> for Runtime { fn block_messages() -> BlockMessagesWithStorageKey { Messenger::get_block_messages() } @@ -1463,4 +1461,14 @@ impl_runtime_apis! { pallet_mmr::verify_leaves_proof::(root, nodes, proof) } } + + impl subspace_test_primitives::OnchainStateApi for Runtime { + fn free_balance(account_id: AccountId) -> Balance { + Balances::free_balance(account_id) + } + + fn get_open_channel_for_chain(dst_chain_id: ChainId) -> Option { + Messenger::get_open_channel_for_chain(dst_chain_id).map(|(c, _)| c) + } + } } diff --git a/test/subspace-test-service/Cargo.toml b/test/subspace-test-service/Cargo.toml index b4a003ebdc..d7868d71c9 100644 --- a/test/subspace-test-service/Cargo.toml +++ b/test/subspace-test-service/Cargo.toml @@ -18,9 +18,11 @@ targets = ["x86_64-unknown-linux-gnu"] async-trait = "0.1.77" cross-domain-message-gossip = { version = "0.1.0", path = "../../domains/client/cross-domain-message-gossip" } codec = { package = "parity-scale-codec", version = "3.2.1", features = ["derive"] } +domain-client-message-relayer = { version = "0.1.0", path = "../../domains/client/relayer" } domain-runtime-primitives = { version = "0.1.0", path = "../../domains/primitives/runtime" } futures = "0.3.29" jsonrpsee = { version = "0.16.3", features = ["server"] } +mmr-gadget = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } rand = "0.8.5" pallet-domains = { version = "0.1.0", path = "../../crates/pallet-domains" } parking_lot = "0.12.1" @@ -48,6 +50,7 @@ sp-domains-fraud-proof = { version = "0.1.0", path = "../../crates/sp-domains-fr sp-externalities = { version = "0.19.0", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-keyring = { git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-messenger = { version = "0.1.0", path = "../../domains/primitives/messenger" } +sp-messenger-host-functions = { version = "0.1.0", path = "../../domains/primitives/messenger-host-functions" } sp-mmr-primitives = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } sp-subspace-mmr = { version = "0.1.0", path = "../../crates/sp-subspace-mmr" } sp-timestamp = { version = "4.0.0-dev", git = "https://github.com/subspace/polkadot-sdk", rev = "d6b500960579d73c43fc4ef550b703acfa61c4c8" } @@ -57,6 +60,7 @@ subspace-core-primitives = { version = "0.1.0", default-features = false, path = subspace-runtime-primitives = { path = "../../crates/subspace-runtime-primitives" } subspace-service = { path = "../../crates/subspace-service" } subspace-test-client = { path = "../subspace-test-client" } +subspace-test-primitives = { version = "0.1.0", path = "../subspace-test-primitives" } subspace-test-runtime = { version = "0.1.0", path = "../subspace-test-runtime" } tokio = "1.35.1" tracing = "0.1.40" diff --git a/test/subspace-test-service/src/lib.rs b/test/subspace-test-service/src/lib.rs index d3ac2f1e9a..0c70d29eb8 100644 --- a/test/subspace-test-service/src/lib.rs +++ b/test/subspace-test-service/src/lib.rs @@ -27,7 +27,7 @@ use jsonrpsee::RpcModule; use parking_lot::Mutex; use sc_block_builder::BlockBuilderBuilder; use sc_client_api::execution_extensions::ExtensionsFactory; -use sc_client_api::{BlockBackend, ExecutorProvider}; +use sc_client_api::{Backend as BackendT, BlockBackend, ExecutorProvider, Finalizer}; use sc_consensus::block_import::{ BlockCheckParams, BlockImportParams, ForkChoiceStrategy, ImportResult, }; @@ -38,11 +38,12 @@ use sc_domains::ExtensionsFactory as DomainsExtensionFactory; use sc_network::config::{NetworkConfiguration, TransportConfig}; use sc_network::{multiaddr, NotificationService}; use sc_service::config::{ - DatabaseSource, KeystoreConfig, MultiaddrWithPeerId, WasmExecutionMethod, + DatabaseSource, KeystoreConfig, MultiaddrWithPeerId, OffchainWorkerConfig, WasmExecutionMethod, WasmtimeInstantiationStrategy, }; use sc_service::{ - BasePath, BlocksPruning, Configuration, NetworkStarter, Role, SpawnTasksParams, TaskManager, + BasePath, BlocksPruning, Configuration, NetworkStarter, PruningMode, Role, SpawnTasksParams, + TaskManager, }; use sc_transaction_pool::error::Error as PoolError; use sc_transaction_pool_api::{InPoolTransaction, TransactionPool, TransactionSource}; @@ -56,15 +57,18 @@ use sp_consensus_subspace::digests::{ extract_pre_digest, CompatibleDigestItem, PreDigest, PreDigestPotInfo, }; use sp_consensus_subspace::{FarmerPublicKey, PotExtension}; +use sp_core::offchain::storage::OffchainDb; +use sp_core::offchain::OffchainDbExt; use sp_core::traits::{CodeExecutor, SpawnEssentialNamed}; use sp_core::H256; -use sp_domains::{BundleProducerElectionApi, DomainsApi, OpaqueBundle}; +use sp_domains::{BundleProducerElectionApi, ChainId, DomainsApi, OpaqueBundle}; use sp_domains_fraud_proof::fraud_proof::FraudProof; use sp_domains_fraud_proof::{FraudProofExtension, FraudProofHostFunctionsImpl}; use sp_externalities::Extensions; use sp_inherents::{InherentData, InherentDataProvider}; use sp_keyring::Sr25519Keyring; use sp_messenger::MessengerApi; +use sp_messenger_host_functions::{MessengerExtension, MessengerHostFunctionsImpl}; use sp_mmr_primitives::MmrApi; use sp_runtime::generic::{BlockId, Digest}; use sp_runtime::traits::{ @@ -86,6 +90,7 @@ use subspace_runtime_primitives::{AccountId, Balance, Hash}; use subspace_service::transaction_pool::FullPool; use subspace_service::{FullSelectChain, RuntimeExecutor}; use subspace_test_client::{chain_spec, Backend, Client}; +use subspace_test_primitives::OnchainStateApi; use subspace_test_runtime::{RuntimeApi, RuntimeCall, UncheckedExtrinsic, SLOT_DURATION}; type FraudProofFor = @@ -166,7 +171,10 @@ pub fn node_config( prometheus_config: None, telemetry_endpoints: None, default_heap_pages: None, - offchain_worker: Default::default(), + offchain_worker: OffchainWorkerConfig { + enabled: false, + indexing_enabled: true, + }, force_authoring, disable_grandpa: false, dev_key_seed: Some(key_seed), @@ -183,21 +191,26 @@ pub fn node_config( type StorageChanges = sp_api::StorageChanges; -struct MockExtensionsFactory { +struct MockExtensionsFactory { consensus_client: Arc, + consensus_backend: Arc, executor: Arc, mock_pot_verifier: Arc, _phantom: PhantomData, } -impl MockExtensionsFactory { +impl + MockExtensionsFactory +{ fn new( consensus_client: Arc, executor: Arc, mock_pot_verifier: Arc, + consensus_backend: Arc, ) -> Self { Self { consensus_client, + consensus_backend, executor, mock_pot_verifier, _phantom: Default::default(), @@ -218,8 +231,8 @@ impl MockPotVerfier { } } -impl ExtensionsFactory - for MockExtensionsFactory +impl ExtensionsFactory + for MockExtensionsFactory where Block: BlockT, Block::Hash: From, @@ -231,6 +244,7 @@ where + MessengerApi> + MmrApi>, Executor: CodeExecutor + sc_executor::RuntimeVersionOf, + CBackend: BackendT + 'static, { fn extensions_for( &self, @@ -239,18 +253,30 @@ where ) -> Extensions { let mut exts = Extensions::new(); exts.register(FraudProofExtension::new(Arc::new( - FraudProofHostFunctionsImpl::<_, _, DomainBlock, Executor>::new( + FraudProofHostFunctionsImpl::<_, _, DomainBlock, Executor, _>::new( self.consensus_client.clone(), self.executor.clone(), - Box::new(DomainsExtensionFactory::<_, Block, DomainBlock, _>::new( - self.consensus_client.clone(), - self.executor.clone(), - )), + |client, executor| { + let extension_factory = + DomainsExtensionFactory::<_, Block, DomainBlock, _>::new(client, executor); + Box::new(extension_factory) as Box> + }, ), ))); exts.register(SubspaceMmrExtension::new(Arc::new( SubspaceMmrHostFunctionsImpl::::new(self.consensus_client.clone()), ))); + exts.register(MessengerExtension::new(Arc::new( + MessengerHostFunctionsImpl::::new( + self.consensus_client.clone(), + self.executor.clone(), + ), + ))); + + if let Some(offchain_storage) = self.consensus_backend.offchain_storage() { + let offchain_db = OffchainDb::new(offchain_storage); + exts.register(OffchainDbExt::new(offchain_db)); + } exts.register(PotExtension::new({ let client = Arc::clone(&self.consensus_client); let mock_pot_verifier = Arc::clone(&self.mock_pot_verifier); @@ -324,6 +350,9 @@ pub struct MockConsensusNode { /// Mock subspace solution used to mock the subspace `PreDigest` mock_solution: Solution, log_prefix: &'static str, + /// Ferdie key + pub key: Sr25519Keyring, + finalize_block_depth: Option>, } impl MockConsensusNode { @@ -332,6 +361,16 @@ impl MockConsensusNode { tokio_handle: tokio::runtime::Handle, key: Sr25519Keyring, base_path: BasePath, + ) -> MockConsensusNode { + Self::run_with_finalization_depth(tokio_handle, key, base_path, None) + } + + /// Run a mock consensus node with finalization depth + pub fn run_with_finalization_depth( + tokio_handle: tokio::runtime::Handle, + key: Sr25519Keyring, + base_path: BasePath, + finalize_block_depth: Option>, ) -> MockConsensusNode { let log_prefix = key.into(); @@ -363,14 +402,19 @@ impl MockConsensusNode { _, DomainBlock, sc_domains::RuntimeExecutor, + _, >::new( client.clone(), Arc::new(domain_executor), Arc::clone(&mock_pot_verifier), + backend.clone(), )); let select_chain = sc_consensus::LongestChain::new(backend.clone()); - + let state_pruning = config + .state_pruning + .clone() + .unwrap_or(PruningMode::ArchiveCanonical); let sync_target_block_number = Arc::new(AtomicU32::new(0)); let transaction_pool = subspace_service::transaction_pool::new_full( config.transaction_pool.clone(), @@ -427,6 +471,54 @@ impl MockConsensusNode { key.to_account_id(), ); + let mut gossip_builder = GossipWorkerBuilder::new(); + task_manager + .spawn_essential_handle() + .spawn_essential_blocking( + "consensus-chain-relayer", + None, + Box::pin( + domain_client_message_relayer::worker::relay_consensus_chain_messages( + client.clone(), + state_pruning.clone(), + sync_service.clone(), + gossip_builder.gossip_msg_sink(), + ), + ), + ); + + let (consensus_msg_sink, consensus_msg_receiver) = + tracing_unbounded("consensus_message_channel", 100); + + // Start cross domain message listener for Consensus chain to receive messages from domains in the network + let consensus_listener = cross_domain_message_gossip::start_cross_chain_message_listener( + ChainId::Consensus, + client.clone(), + transaction_pool.clone(), + network_service.clone(), + consensus_msg_receiver, + ); + + task_manager + .spawn_essential_handle() + .spawn_essential_blocking( + "consensus-message-listener", + None, + Box::pin(consensus_listener), + ); + + gossip_builder.push_chain_tx_pool_sink(ChainId::Consensus, consensus_msg_sink); + + task_manager.spawn_essential_handle().spawn_blocking( + "mmr-gadget", + None, + mmr_gadget::MmrGadget::start( + client.clone(), + backend.clone(), + sp_mmr_primitives::INDEXING_PREFIX.to_vec(), + ), + ); + MockConsensusNode { task_manager, client, @@ -444,9 +536,11 @@ impl MockConsensusNode { new_slot_notification_subscribers: Vec::new(), acknowledgement_sender_subscribers: Vec::new(), block_import, - xdm_gossip_worker_builder: Some(GossipWorkerBuilder::new()), + xdm_gossip_worker_builder: Some(gossip_builder), mock_solution, log_prefix, + key, + finalize_block_depth, } } @@ -715,6 +809,14 @@ impl MockConsensusNode { } }) } + + /// Get the free balance of the given account + pub fn free_balance(&self, account_id: AccountId) -> subspace_runtime_primitives::Balance { + self.client + .runtime_api() + .free_balance(self.client.info().best_hash, account_id) + .expect("Fail to get account free balance") + } } impl MockConsensusNode { @@ -796,6 +898,7 @@ impl MockConsensusNode { let (header, body) = block.deconstruct(); let header_hash = header.hash(); + let header_number = header.number; let block_import_params = { let mut import_block = BlockImportParams::new(BlockOrigin::Own, header); @@ -811,6 +914,20 @@ impl MockConsensusNode { let import_result = self.block_import.import_block(block_import_params).await?; + if let Some(finalized_block_hash) = self + .finalize_block_depth + .and_then(|depth| header_number.checked_sub(depth)) + .and_then(|block_to_finalize| { + self.client + .hash(block_to_finalize) + .expect("Block hash not found for number: {block_to_finalize:?}") + }) + { + self.client + .finalize_block(finalized_block_hash, None, true) + .unwrap(); + } + match import_result { ImportResult::Imported(_) | ImportResult::AlreadyInChain => Ok(header_hash), bad_res => Err(format!("Fail to import block due to {bad_res:?}").into()),