From d4c426afd46f43b81115911657ccc0002a361ddb Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Thu, 16 Nov 2023 12:59:06 +0100 Subject: [PATCH 01/48] [ci] Enable zombienet jobs in PRs (#2361) Since preparation for the merge queues needs more time I'm enabling zombienet jobs in PRs CI back. --- .gitlab/pipeline/publish.yml | 2 +- .gitlab/pipeline/zombienet/cumulus.yml | 2 +- .gitlab/pipeline/zombienet/polkadot.yml | 8 ++++---- .gitlab/pipeline/zombienet/substrate.yml | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index f2308c334e0c..3cc2002cc1c5 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -72,7 +72,7 @@ publish-rustdoc: IMAGE_NAME: "" # docker.io/paritypr/image_name script: # Exit if the job is not running in a merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - $BUILDAH_COMMAND build --format=docker --build-arg VCS_REF="${CI_COMMIT_SHA}" diff --git a/.gitlab/pipeline/zombienet/cumulus.yml b/.gitlab/pipeline/zombienet/cumulus.yml index c8a1df004e34..3cac67c2966e 100644 --- a/.gitlab/pipeline/zombienet/cumulus.yml +++ b/.gitlab/pipeline/zombienet/cumulus.yml @@ -4,7 +4,7 @@ .zombienet-before-script: before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - echo "Zombie-net Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${POLKADOT_IMAGE}" diff --git a/.gitlab/pipeline/zombienet/polkadot.yml b/.gitlab/pipeline/zombienet/polkadot.yml index cc9605572981..995dd9825320 100644 --- a/.gitlab/pipeline/zombienet/polkadot.yml +++ b/.gitlab/pipeline/zombienet/polkadot.yml @@ -5,7 +5,7 @@ .zombienet-polkadot-common: before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - export BUILD_RELEASE_VERSION="$(cat ./artifacts/BUILD_RELEASE_VERSION)" # from build-linux-stable job - export DEBUG=zombie,zombie::network-node - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} @@ -120,7 +120,7 @@ zombienet-polkadot-smoke-0001-parachains-smoke-test: - .zombienet-polkadot-common before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} - echo "Zombienet Tests Config" @@ -139,7 +139,7 @@ zombienet-polkadot-smoke-0002-parachains-parachains-upgrade-smoke: - .zombienet-polkadot-common before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - export ZOMBIENET_INTEGRATION_TEST_IMAGE="${POLKADOT_IMAGE}":${PIPELINE_IMAGE_TAG} - export CUMULUS_IMAGE="docker.io/paritypr/polkadot-parachain-debug:${DOCKER_IMAGES_VERSION}" - echo "Zombienet Tests Config" @@ -183,7 +183,7 @@ zombienet-polkadot-misc-0002-upgrade-node: artifacts: true before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - export ZOMBIENET_INTEGRATION_TEST_IMAGE="docker.io/parity/polkadot:latest" - echo "Overrided poladot image ${ZOMBIENET_INTEGRATION_TEST_IMAGE}" - export COL_IMAGE="${COLANDER_IMAGE}":${PIPELINE_IMAGE_TAG} diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index e627575a31ad..6334e7db9a32 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -5,7 +5,7 @@ .zombienet-substrate-common: before_script: # Exit if the job is not merge queue - - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi - echo "Zombienet Tests Config" - echo "${ZOMBIENET_IMAGE}" - echo "${GH_DIR}" From 02e8061bbadf51bce2f8c419c8645555532c7489 Mon Sep 17 00:00:00 2001 From: Adrian Catangiu Date: Thu, 16 Nov 2023 17:36:37 +0200 Subject: [PATCH 02/48] westend: remove SessionKeys migration already applied on-chain (#2363) Westend now successfully updated to `spec: 103000`, we **have to remove** the session keys migration before the next release as it doesn't gracefully handle reapplying it. --- polkadot/runtime/westend/src/lib.rs | 48 ----------------------------- 1 file changed, 48 deletions(-) diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 2e8394b0ee4c..4d537832ae39 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -425,17 +425,6 @@ parameter_types! { pub const Offset: BlockNumber = 0; } -impl_opaque_keys! { - pub struct OldSessionKeys { - pub grandpa: Grandpa, - pub babe: Babe, - pub im_online: ImOnline, - pub para_validator: Initializer, - pub para_assignment: ParaSessionInfo, - pub authority_discovery: AuthorityDiscovery, - } -} - impl_opaque_keys! { pub struct SessionKeys { pub grandpa: Grandpa, @@ -448,32 +437,6 @@ impl_opaque_keys! { } } -// remove this when removing `OldSessionKeys` -fn transform_session_keys(v: AccountId, old: OldSessionKeys) -> SessionKeys { - SessionKeys { - grandpa: old.grandpa, - babe: old.babe, - im_online: old.im_online, - para_validator: old.para_validator, - para_assignment: old.para_assignment, - authority_discovery: old.authority_discovery, - beefy: { - // From Session::upgrade_keys(): - // - // Care should be taken that the raw versions of the - // added keys are unique for every `ValidatorId, KeyTypeId` combination. - // This is an invariant that the session pallet typically maintains internally. - // - // So, produce a dummy value that's unique for the `ValidatorId, KeyTypeId` combination. - let mut id: BeefyId = sp_application_crypto::ecdsa::Public::from_raw([0u8; 33]).into(); - let id_raw: &mut [u8] = id.as_mut(); - id_raw[1..33].copy_from_slice(v.as_ref()); - id_raw[0..4].copy_from_slice(b"beef"); - id - }, - } -} - impl pallet_session::Config for Runtime { type RuntimeEvent = RuntimeEvent; type ValidatorId = AccountId; @@ -1560,16 +1523,6 @@ pub type Migrations = migrations::Unreleased; pub mod migrations { use super::*; - /// Upgrade Session keys to include BEEFY key. - /// When this is removed, should also remove `OldSessionKeys`. - pub struct UpgradeSessionKeys; - impl frame_support::traits::OnRuntimeUpgrade for UpgradeSessionKeys { - fn on_runtime_upgrade() -> Weight { - Session::upgrade_keys::(transform_session_keys); - Perbill::from_percent(50) * BlockWeights::get().max_block - } - } - /// Unreleased migrations. Add new ones here: pub type Unreleased = ( pallet_im_online::migration::v1::Migration, @@ -1578,7 +1531,6 @@ pub mod migrations { assigned_slots::migration::v1::MigrateToV1, parachains_scheduler::migration::v1::MigrateToV1, parachains_configuration::migration::v8::MigrateToV8, - UpgradeSessionKeys, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::MigrateToV1, pallet_nomination_pools::migration::versioned_migrations::V5toV6, From 5e98803f032a7c0ac2d8fae6b0cfc76417917e2a Mon Sep 17 00:00:00 2001 From: ordian Date: Thu, 16 Nov 2023 21:13:04 +0100 Subject: [PATCH 03/48] implementers-guide: update github link (#2368) cc @joepetrowski --- polkadot/roadmap/implementers-guide/book.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/roadmap/implementers-guide/book.toml b/polkadot/roadmap/implementers-guide/book.toml index 1e6680f6f4b7..f677c0d59c04 100644 --- a/polkadot/roadmap/implementers-guide/book.toml +++ b/polkadot/roadmap/implementers-guide/book.toml @@ -17,6 +17,6 @@ renderer = ["html"] additional-css = ["last-changed.css"] additional-js = ["mermaid.min.js", "mermaid-init.js"] # Repository URL used in the last-changed link. -git-repository-url = "https://github.com/paritytech/polkadot" +git-repository-url = "https://github.com/paritytech/polkadot-sdk" [output.linkcheck] From 4ac2db8095e55a68f9b3d843d4e4689759fcc9d5 Mon Sep 17 00:00:00 2001 From: joe petrowski <25483142+joepetrowski@users.noreply.github.com> Date: Fri, 17 Nov 2023 00:02:56 +0100 Subject: [PATCH 04/48] Fix Typo: `PalletXcmExtrinsicsBenchmark` (#2354) Missed in https://github.com/paritytech/polkadot-sdk/pull/1672 --- .../parachains/runtimes/assets/asset-hub-kusama/src/lib.rs | 6 +++--- .../runtimes/assets/asset-hub-polkadot/src/lib.rs | 6 +++--- .../parachains/runtimes/assets/asset-hub-rococo/src/lib.rs | 6 +++--- .../parachains/runtimes/assets/asset-hub-westend/src/lib.rs | 6 +++--- .../runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs | 6 +++--- .../runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs | 6 +++--- .../runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs | 6 +++--- .../runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs | 6 +++--- .../runtimes/collectives/collectives-polkadot/src/lib.rs | 6 +++--- .../runtimes/contracts/contracts-rococo/src/lib.rs | 6 +++--- polkadot/runtime/rococo/src/lib.rs | 6 +++--- polkadot/runtime/westend/src/lib.rs | 6 +++--- 12 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs index e4ed77884bf3..af0116d7014a 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-kusama/src/lib.rs @@ -962,7 +962,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [cumulus_pallet_dmp_queue, DmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -1200,7 +1200,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -1244,7 +1244,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs index 6f853b6f56ed..1b7ef10f4857 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-polkadot/src/lib.rs @@ -868,7 +868,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [cumulus_pallet_dmp_queue, DmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -1082,7 +1082,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -1125,7 +1125,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs index 4492971566b1..4b4ae61a3e8d 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs @@ -996,7 +996,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [pallet_xcm_bridge_hub_router, ToWestend] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -1234,7 +1234,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; use pallet_xcm_bridge_hub_router::benchmarking::Pallet as XcmBridgeHubRouterBench; // This is defined once again in dispatch_benchmark, because list_benchmarks! @@ -1286,7 +1286,7 @@ impl_runtime_apis! { Config as XcmBridgeHubRouterConfig, }; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs index cd17b9d86f74..d52edfe479ce 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/lib.rs @@ -1085,7 +1085,7 @@ mod benches { [cumulus_pallet_dmp_queue, DmpQueue] [pallet_xcm_bridge_hub_router, ToRococo] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -1369,7 +1369,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; use pallet_xcm_bridge_hub_router::benchmarking::Pallet as XcmBridgeHubRouterBench; // This is defined once again in dispatch_benchmark, because list_benchmarks! @@ -1416,7 +1416,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs index b37507000842..d2db0340790e 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-kusama/src/lib.rs @@ -490,7 +490,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [cumulus_pallet_dmp_queue, DmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -670,7 +670,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -706,7 +706,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs index 841bb4ee8611..02f05a8bb877 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-polkadot/src/lib.rs @@ -491,7 +491,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [cumulus_pallet_dmp_queue, DmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -671,7 +671,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -707,7 +707,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index b8fc2fffc889..5a44ccbb75a2 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -554,7 +554,7 @@ mod benches { [cumulus_pallet_xcmp_queue, XcmpQueue] [cumulus_pallet_dmp_queue, DmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -784,7 +784,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -826,7 +826,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs index 9e8fd84e7125..d1d2b4a41595 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs @@ -544,7 +544,7 @@ mod benches { [pallet_collator_selection, CollatorSelection] [cumulus_pallet_xcmp_queue, XcmpQueue] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -772,7 +772,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; // This is defined once again in dispatch_benchmark, because list_benchmarks! // and add_benchmarks! are macros exported by define_benchmarks! macros and those types @@ -814,7 +814,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs index 206f46140606..c3d671c90855 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-polkadot/src/lib.rs @@ -751,7 +751,7 @@ mod benches { [cumulus_pallet_dmp_queue, DmpQueue] [pallet_alliance, Alliance] [pallet_collective, AllianceMotion] - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] [pallet_preimage, Preimage] [pallet_scheduler, Scheduler] [pallet_referenda, FellowshipReferenda] @@ -939,7 +939,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; let mut list = Vec::::new(); list_benchmarks!(list, extra); @@ -969,7 +969,7 @@ impl_runtime_apis! { use cumulus_pallet_session_benchmarking::Pallet as SessionBench; impl cumulus_pallet_session_benchmarking::Config for Runtime {} - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs index 4c66e780ba9f..5b828bad0c7f 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs @@ -433,7 +433,7 @@ mod benches { [pallet_timestamp, Timestamp] [pallet_collator_selection, CollatorSelection] [pallet_contracts, Contracts] - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] ); } @@ -678,7 +678,7 @@ impl_runtime_apis! { use frame_support::traits::StorageInfoTrait; use frame_system_benchmarking::Pallet as SystemBench; use cumulus_pallet_session_benchmarking::Pallet as SessionBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; let mut list = Vec::::new(); list_benchmarks!(list, extra); @@ -709,7 +709,7 @@ impl_runtime_apis! { impl cumulus_pallet_session_benchmarking::Config for Runtime {} use xcm::latest::prelude::*; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; impl pallet_xcm::benchmarking::Config for Runtime { fn reachable_dest() -> Option { Some(Parent.into()) diff --git a/polkadot/runtime/rococo/src/lib.rs b/polkadot/runtime/rococo/src/lib.rs index 277c9981dab8..675e0a20b2b0 100644 --- a/polkadot/runtime/rococo/src/lib.rs +++ b/polkadot/runtime/rococo/src/lib.rs @@ -1617,7 +1617,7 @@ mod benches { [pallet_asset_rate, AssetRate] [pallet_whitelist, Whitelist] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] [pallet_xcm_benchmarks::fungible, pallet_xcm_benchmarks::fungible::Pallet::] [pallet_xcm_benchmarks::generic, pallet_xcm_benchmarks::generic::Pallet::] ); @@ -2096,7 +2096,7 @@ sp_api::impl_runtime_apis! { use frame_system_benchmarking::Pallet as SystemBench; use frame_benchmarking::baseline::Pallet as Baseline; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; let mut list = Vec::::new(); list_benchmarks!(list, extra); @@ -2115,7 +2115,7 @@ sp_api::impl_runtime_apis! { use frame_benchmarking::{Benchmarking, BenchmarkBatch, BenchmarkError}; use frame_system_benchmarking::Pallet as SystemBench; use frame_benchmarking::baseline::Pallet as Baseline; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; use sp_storage::TrackedStorageKey; use xcm::latest::prelude::*; use xcm_config::{ diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 4d537832ae39..16a0b112a0b0 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1606,7 +1606,7 @@ mod benches { [pallet_whitelist, Whitelist] [pallet_asset_rate, AssetRate] // XCM - [pallet_xcm, PalletXcmExtrinsiscsBenchmark::] + [pallet_xcm, PalletXcmExtrinsicsBenchmark::] // NOTE: Make sure you point to the individual modules below. [pallet_xcm_benchmarks::fungible, XcmBalances] [pallet_xcm_benchmarks::generic, XcmGeneric] @@ -2129,7 +2129,7 @@ sp_api::impl_runtime_apis! { use pallet_session_benchmarking::Pallet as SessionBench; use pallet_offences_benchmarking::Pallet as OffencesBench; use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; use frame_system_benchmarking::Pallet as SystemBench; use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; @@ -2157,7 +2157,7 @@ sp_api::impl_runtime_apis! { use pallet_session_benchmarking::Pallet as SessionBench; use pallet_offences_benchmarking::Pallet as OffencesBench; use pallet_election_provider_support_benchmarking::Pallet as ElectionProviderBench; - use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsiscsBenchmark; + use pallet_xcm::benchmarking::Pallet as PalletXcmExtrinsicsBenchmark; use frame_system_benchmarking::Pallet as SystemBench; use pallet_nomination_pools_benchmarking::Pallet as NominationPoolsBench; From 596088a273eff13fbefe6fa20a5fef6507b329cb Mon Sep 17 00:00:00 2001 From: Bruno Galvao Date: Fri, 17 Nov 2023 01:31:31 -0500 Subject: [PATCH 05/48] add pallet nomination-pools versioned migration to kitchensink (#2167) The versioned migrations are already there in pallet nomination-pools: https://github.com/paritytech/polkadot-sdk/blob/f6ee4781f633f0f89598f7b230595afe401da8dc/substrate/frame/nomination-pools/src/migration.rs#L27-L48 Just updating the kitchensink runtime to point to them. This is also nice because it points the dev to an example of how to use `VersionedMigration`. --- polkadot/runtime/westend/src/lib.rs | 4 ++-- substrate/bin/node/runtime/src/lib.rs | 5 +++-- substrate/frame/nomination-pools/src/migration.rs | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/polkadot/runtime/westend/src/lib.rs b/polkadot/runtime/westend/src/lib.rs index 16a0b112a0b0..29183fdfe00b 100644 --- a/polkadot/runtime/westend/src/lib.rs +++ b/polkadot/runtime/westend/src/lib.rs @@ -1533,9 +1533,9 @@ pub mod migrations { parachains_configuration::migration::v8::MigrateToV8, parachains_configuration::migration::v9::MigrateToV9, paras_registrar::migration::MigrateToV1, - pallet_nomination_pools::migration::versioned_migrations::V5toV6, + pallet_nomination_pools::migration::versioned::V5toV6, pallet_referenda::migration::v1::MigrateV0ToV1, - pallet_nomination_pools::migration::versioned_migrations::V6ToV7, + pallet_nomination_pools::migration::versioned::V6ToV7, pallet_grandpa::migrations::MigrateV4ToV5, parachains_configuration::migration::v10::MigrateToV10, ); diff --git a/substrate/bin/node/runtime/src/lib.rs b/substrate/bin/node/runtime/src/lib.rs index 90946b713116..d7beb29becf4 100644 --- a/substrate/bin/node/runtime/src/lib.rs +++ b/substrate/bin/node/runtime/src/lib.rs @@ -2194,9 +2194,10 @@ pub type Executive = frame_executive::Executive< >; // All migrations executed on runtime upgrade as a nested tuple of types implementing -// `OnRuntimeUpgrade`. +// `OnRuntimeUpgrade`. Note: These are examples and do not need to be run directly +// after the genesis block. type Migrations = ( - pallet_nomination_pools::migration::v2::MigrateToV2, + pallet_nomination_pools::migration::versioned::V6ToV7, pallet_alliance::migration::Migration, pallet_contracts::Migration, ); diff --git a/substrate/frame/nomination-pools/src/migration.rs b/substrate/frame/nomination-pools/src/migration.rs index eef2a976f1a2..3d68fee1dca4 100644 --- a/substrate/frame/nomination-pools/src/migration.rs +++ b/substrate/frame/nomination-pools/src/migration.rs @@ -24,7 +24,7 @@ use sp_std::{collections::btree_map::BTreeMap, vec::Vec}; use sp_runtime::TryRuntimeError; /// Exports for versioned migration `type`s for this pallet. -pub mod versioned_migrations { +pub mod versioned { use super::*; /// Migration V6 to V7 wrapped in a [`frame_support::migrations::VersionedMigration`], ensuring From b85c64aa5fa945027c7a2bfe1788e09309febc7a Mon Sep 17 00:00:00 2001 From: cuteolaf Date: Thu, 16 Nov 2023 23:20:09 -0800 Subject: [PATCH 06/48] fix typo (#2377) --- substrate/frame/proxy/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/substrate/frame/proxy/README.md b/substrate/frame/proxy/README.md index bfe26d9aefbc..c52a881c5909 100644 --- a/substrate/frame/proxy/README.md +++ b/substrate/frame/proxy/README.md @@ -2,7 +2,7 @@ A module allowing accounts to give permission to other accounts to dispatch types of calls from their signed origin. -The accounts to which permission is delegated may be requied to announce the action that they +The accounts to which permission is delegated may be required to announce the action that they wish to execute some duration prior to execution happens. In this case, the target account may reject the announcement and in doing so, veto the execution. From 20723ea80e0f442487764a9bcde1d4c989958e56 Mon Sep 17 00:00:00 2001 From: Javier Viola Date: Fri, 17 Nov 2023 08:21:42 -0300 Subject: [PATCH 07/48] bump zombienet version `v1.3.80` (#2376) New release includes logic to move all jobs to `spot instances`. Thx! --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 1dc483004f24..b485ca3317ed 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.79" + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.80" DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}" default: From 2e001de9346257321a8e2447d22e86d7c71a917a Mon Sep 17 00:00:00 2001 From: Ankan <10196091+Ank4n@users.noreply.github.com> Date: Fri, 17 Nov 2023 13:48:31 +0100 Subject: [PATCH 08/48] [NPoS] Check if staker is exposed in paged exposure storage entries (#2369) Addresses a bug caused by https://github.com/paritytech/polkadot-sdk/pull/1189. The changes are still not released yet, so would like to push the fix soon so it can go together with the release of the above PR. `fast_unstake` checks if a staker is exposed in an era. However, this fn is still returning whether the staker is exposed based on the old storage item. This PR fixes that by looking in both old and new exposure storages. Also adds some integrity tests for paged exposures. --- substrate/frame/staking/src/pallet/impls.rs | 74 ++++++++++++++++++++- substrate/frame/staking/src/tests.rs | 16 +++++ 2 files changed, 89 insertions(+), 1 deletion(-) diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index 9c36c94b87b4..40f30735258f 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -794,7 +794,7 @@ impl Pallet { stash: T::AccountId, exposure: Exposure>, ) { - >::insert(¤t_era, &stash, &exposure); + EraInfo::::set_exposure(current_era, &stash, exposure); } #[cfg(feature = "runtime-benchmarks")] @@ -1745,9 +1745,16 @@ impl StakingInterface for Pallet { } fn is_exposed_in_era(who: &Self::AccountId, era: &EraIndex) -> bool { + // look in the non paged exposures + // FIXME: Can be cleaned up once non paged exposures are cleared (https://github.com/paritytech/polkadot-sdk/issues/433) ErasStakers::::iter_prefix(era).any(|(validator, exposures)| { validator == *who || exposures.others.iter().any(|i| i.who == *who) }) + || + // look in the paged exposures + ErasStakersPaged::::iter_prefix((era,)).any(|((validator, _), exposure_page)| { + validator == *who || exposure_page.others.iter().any(|i| i.who == *who) + }) } fn status( who: &Self::AccountId, @@ -1812,6 +1819,7 @@ impl Pallet { Self::check_nominators()?; Self::check_exposures()?; + Self::check_paged_exposures()?; Self::check_ledgers()?; Self::check_count() } @@ -1860,6 +1868,70 @@ impl Pallet { .collect::>() } + fn check_paged_exposures() -> Result<(), TryRuntimeError> { + use sp_staking::PagedExposureMetadata; + use sp_std::collections::btree_map::BTreeMap; + + // Sanity check for the paged exposure of the active era. + let mut exposures: BTreeMap>> = + BTreeMap::new(); + let era = Self::active_era().unwrap().index; + let accumulator_default = PagedExposureMetadata { + total: Zero::zero(), + own: Zero::zero(), + nominator_count: 0, + page_count: 0, + }; + + ErasStakersPaged::::iter_prefix((era,)) + .map(|((validator, _page), expo)| { + ensure!( + expo.page_total == + expo.others.iter().map(|e| e.value).fold(Zero::zero(), |acc, x| acc + x), + "wrong total exposure for the page.", + ); + + let metadata = exposures.get(&validator).unwrap_or(&accumulator_default); + exposures.insert( + validator, + PagedExposureMetadata { + total: metadata.total + expo.page_total, + own: metadata.own, + nominator_count: metadata.nominator_count + expo.others.len() as u32, + page_count: metadata.page_count + 1, + }, + ); + + Ok(()) + }) + .collect::>()?; + + exposures + .iter() + .map(|(validator, metadata)| { + let actual_overview = ErasStakersOverview::::get(era, validator); + + ensure!(actual_overview.is_some(), "No overview found for a paged exposure"); + let actual_overview = actual_overview.unwrap(); + + ensure!( + actual_overview.total == metadata.total + actual_overview.own, + "Exposure metadata does not have correct total exposed stake." + ); + ensure!( + actual_overview.nominator_count == metadata.nominator_count, + "Exposure metadata does not have correct count of nominators." + ); + ensure!( + actual_overview.page_count == metadata.page_count, + "Exposure metadata does not have correct count of pages." + ); + + Ok(()) + }) + .collect::>() + } + fn check_nominators() -> Result<(), TryRuntimeError> { // a check per nominator to ensure their entire stake is correctly distributed. Will only // kick-in if the nomination was submitted before the current era. diff --git a/substrate/frame/staking/src/tests.rs b/substrate/frame/staking/src/tests.rs index ee6f67adf14c..bac2530b19bb 100644 --- a/substrate/frame/staking/src/tests.rs +++ b/substrate/frame/staking/src/tests.rs @@ -6637,6 +6637,14 @@ fn test_validator_exposure_is_backward_compatible_with_non_paged_rewards_payout( ); assert_eq!(EraInfo::::get_page_count(1, &11), 2); + // validator is exposed + assert!(::is_exposed_in_era(&11, &1)); + // nominators are exposed + for i in 10..15 { + let who: AccountId = 1000 + i; + assert!(::is_exposed_in_era(&who, &1)); + } + // case 2: exposure exist in ErasStakers and ErasStakersClipped (legacy). // delete paged storage and add exposure to clipped storage >::remove((1, 11, 0)); @@ -6672,6 +6680,14 @@ fn test_validator_exposure_is_backward_compatible_with_non_paged_rewards_payout( assert_eq!(actual_exposure_full.own, 1000); assert_eq!(actual_exposure_full.total, total_exposure); + // validator is exposed + assert!(::is_exposed_in_era(&11, &1)); + // nominators are exposed + for i in 10..15 { + let who: AccountId = 1000 + i; + assert!(::is_exposed_in_era(&who, &1)); + } + // for pages other than 0, clipped storage returns empty exposure assert_eq!(EraInfo::::get_paged_exposure(1, &11, 1), None); // page size is 1 for clipped storage From 3ab2bc9ff3c6ecf14694f01e3c7e8f394219585a Mon Sep 17 00:00:00 2001 From: Serban Iorga Date: Fri, 17 Nov 2023 15:24:20 +0200 Subject: [PATCH 09/48] Beefy: small fixes (#2378) Related to #2285 - save the state of the BEEFY gadget after processing a finality proof. We need this in order to avoid skipping blocks. - avoid reprocessing the old state when not necessary --- .../client/consensus/beefy/src/worker.rs | 36 ++++++++++++------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs index 309d8c5135be..966c74103657 100644 --- a/substrate/client/consensus/beefy/src/worker.rs +++ b/substrate/client/consensus/beefy/src/worker.rs @@ -456,6 +456,7 @@ where .filter(|genesis| *genesis == self.persisted_state.pallet_genesis) .ok_or(Error::ConsensusReset)?; + let mut new_session_added = false; if *header.number() > self.best_grandpa_block() { // update best GRANDPA finalized block we have seen self.persisted_state.set_best_grandpa(header.clone()); @@ -475,9 +476,15 @@ where { if let Some(new_validator_set) = find_authorities_change::(&header) { self.init_session_at(new_validator_set, *header.number()); + new_session_added = true; } } + if new_session_added { + crate::aux_schema::write_voter_state(&*self.backend, &self.persisted_state) + .map_err(|e| Error::Backend(e.to_string()))?; + } + // Update gossip validator votes filter. if let Err(e) = self .persisted_state @@ -848,15 +855,10 @@ where .fuse(), ); + self.process_new_state(); let error = loop { - // Act on changed 'state'. - self.process_new_state(); - // Mutable reference used to drive the gossip engine. let mut gossip_engine = &mut self.comms.gossip_engine; - // Use temp val and report after async section, - // to avoid having to Mutex-wrap `gossip_engine`. - let mut gossip_report: Option = None; // Wait for, and handle external events. // The branches below only change 'state', actual voting happens afterwards, @@ -884,10 +886,15 @@ where if let Err(err) = self.triage_incoming_justif(justif) { debug!(target: LOG_TARGET, "🥩 {}", err); } - gossip_report = Some(peer_report); + self.comms.gossip_engine.report(peer_report.who, peer_report.cost_benefit); + }, + ResponseInfo::PeerReport(peer_report) => { + self.comms.gossip_engine.report(peer_report.who, peer_report.cost_benefit); + continue; + }, + ResponseInfo::Pending => { + continue; }, - ResponseInfo::PeerReport(peer_report) => gossip_report = Some(peer_report), - ResponseInfo::Pending => (), } }, justif = block_import_justif.next() => { @@ -924,12 +931,15 @@ where }, // Process peer reports. report = self.comms.gossip_report_stream.next() => { - gossip_report = report; + if let Some(PeerReport { who, cost_benefit }) = report { + self.comms.gossip_engine.report(who, cost_benefit); + } + continue; }, } - if let Some(PeerReport { who, cost_benefit }) = gossip_report { - self.comms.gossip_engine.report(who, cost_benefit); - } + + // Act on changed 'state'. + self.process_new_state(); }; // return error _and_ `comms` that can be reused From 5007e2dd5cb5514fe11ad1bdb21ea9c7fb0ae5ca Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Fri, 17 Nov 2023 14:43:37 +0100 Subject: [PATCH 10/48] crypto: `lazy_static` removed, light parser for address URI added (#2250) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The `lazy_static` package does not work well in `no-std`: it requires `spin_no_std` feature, which also will propagate into `std` if enabled. This is not what we want. This PR provides simple address uri parser which allows to get rid of _regex_ which was used to parse the address uri, what in turns allows to remove lazy_static. Three regular expressions (`SS58_REGEX`,`SECRET_PHRASE_REGEX`,`JUNCTION_REGEX`) were replaced with the parser which unifies all of them. The new parser does not support Unicode, it is ASCII only. Related to: #2044 --------- Co-authored-by: Bastian Köcher Co-authored-by: Koute Co-authored-by: command-bot <> --- Cargo.lock | 45 +- Cargo.toml | 1 + ...age_ensure_span_are_ok_on_wrong_gen.stderr | 12 +- ...re_span_are_ok_on_wrong_gen_unnamed.stderr | 12 +- substrate/primitives/core/Cargo.toml | 6 +- substrate/primitives/core/fuzz/Cargo.toml | 20 + .../fuzz/fuzz_targets/fuzz_address_uri.rs | 53 +++ substrate/primitives/core/src/address_uri.rs | 432 ++++++++++++++++++ substrate/primitives/core/src/crypto.rs | 85 ++-- substrate/primitives/core/src/lib.rs | 2 + 10 files changed, 601 insertions(+), 67 deletions(-) create mode 100644 substrate/primitives/core/fuzz/Cargo.toml create mode 100644 substrate/primitives/core/fuzz/fuzz_targets/fuzz_address_uri.rs create mode 100644 substrate/primitives/core/src/address_uri.rs diff --git a/Cargo.lock b/Cargo.lock index 7854042ddbcf..64da4b8996ca 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7408,6 +7408,17 @@ dependencies = [ "rle-decode-fast", ] +[[package]] +name = "libfuzzer-sys" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a96cfd5557eb82f2b83fed4955246c988d331975a002961b07c81584d107e7f7" +dependencies = [ + "arbitrary", + "cc", + "once_cell", +] + [[package]] name = "libloading" version = "0.7.4" @@ -8182,9 +8193,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "f665ee40bc4a3c5590afb1e9677db74a508659dfd71e126420da8274909a0167" [[package]] name = "memfd" @@ -14297,14 +14308,14 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.10.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "380b951a9c5e80ddfd6136919eef32310721aa4aacd4889a8d39124b026ab343" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.3.6", - "regex-syntax 0.7.4", + "regex-automata 0.4.3", + "regex-syntax 0.8.2", ] [[package]] @@ -14321,10 +14332,16 @@ name = "regex-automata" version = "0.3.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" + +[[package]] +name = "regex-automata" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.7.4", + "regex-syntax 0.8.2", ] [[package]] @@ -14335,9 +14352,9 @@ checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" [[package]] name = "remote-ext-tests-bags-list" @@ -17535,6 +17552,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "sp-core-fuzz" +version = "0.0.0" +dependencies = [ + "lazy_static", + "libfuzzer-sys", + "regex", + "sp-core", +] + [[package]] name = "sp-core-hashing" version = "9.0.0" diff --git a/Cargo.toml b/Cargo.toml index ed252e07053f..57079aa4d03d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -405,6 +405,7 @@ members = [ "substrate/primitives/consensus/sassafras", "substrate/primitives/consensus/slots", "substrate/primitives/core", + "substrate/primitives/core/fuzz", "substrate/primitives/core/hashing", "substrate/primitives/core/hashing/proc-macro", "substrate/primitives/crypto/ec-utils", diff --git a/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr b/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr index 7375bcd2f16a..b5d108275249 100644 --- a/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen.stderr @@ -6,8 +6,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -44,8 +44,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others @@ -81,8 +81,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -119,8 +119,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others @@ -137,8 +137,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -177,8 +177,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others diff --git a/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr b/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr index 3a0a25712aaf..b58902590b85 100644 --- a/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr +++ b/substrate/frame/support/test/tests/pallet_ui/storage_ensure_span_are_ok_on_wrong_gen_unnamed.stderr @@ -6,8 +6,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -44,8 +44,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others @@ -81,8 +81,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -119,8 +119,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others @@ -137,8 +137,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeDecode` is not satisfied | = help: the following other types implement trait `WrapperTypeDecode`: Box - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc = note: required for `Bar` to implement `Decode` = note: required for `Bar` to implement `FullCodec` @@ -177,8 +177,8 @@ error[E0277]: the trait bound `Bar: WrapperTypeEncode` is not satisfied bytes::bytes::Bytes Cow<'a, T> parity_scale_codec::Ref<'a, T, U> - frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Rc + frame_support::sp_runtime::sp_application_crypto::sp_core::Bytes Arc Vec and $N others diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 79df81e62c66..9ecce0a22f5f 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -26,10 +26,8 @@ bs58 = { version = "0.5.0", default-features = false, optional = true } rand = { version = "0.8.5", features = ["small_rng"], optional = true } substrate-bip39 = { version = "0.4.4", optional = true } bip39 = { version = "2.0.0", default-features = false } -regex = { version = "1.6.0", optional = true } zeroize = { version = "1.4.3", default-features = false } secrecy = { version = "0.8.0", default-features = false } -lazy_static = { version = "1.4.0", default-features = false, optional = true } parking_lot = { version = "0.12.1", optional = true } ss58-registry = { version = "1.34.0", default-features = false } sp-std = { path = "../std", default-features = false} @@ -63,6 +61,8 @@ bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "cbc342e", [dev-dependencies] criterion = "0.4.0" serde_json = "1.0.108" +lazy_static = "1.4.0" +regex = "1.6.0" sp-core-hashing-proc-macro = { path = "hashing/proc-macro" } [[bench]] @@ -92,7 +92,6 @@ std = [ "hash256-std-hasher/std", "impl-serde/std", "itertools", - "lazy_static", "libsecp256k1/std", "log/std", "merlin/std", @@ -102,7 +101,6 @@ std = [ "primitive-types/serde", "primitive-types/std", "rand", - "regex", "scale-info/std", "schnorrkel/std", "secp256k1/global-context", diff --git a/substrate/primitives/core/fuzz/Cargo.toml b/substrate/primitives/core/fuzz/Cargo.toml new file mode 100644 index 000000000000..9a094b07d4a1 --- /dev/null +++ b/substrate/primitives/core/fuzz/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "sp-core-fuzz" +version = "0.0.0" +publish = false + +[package.metadata] +cargo-fuzz = true + +[dependencies] +lazy_static = "1.4.0" +libfuzzer-sys = "0.4" +regex = "1.10.2" + +sp-core = { path = ".." } + +[[bin]] +name = "fuzz_address_uri" +path = "fuzz_targets/fuzz_address_uri.rs" +test = false +doc = false diff --git a/substrate/primitives/core/fuzz/fuzz_targets/fuzz_address_uri.rs b/substrate/primitives/core/fuzz/fuzz_targets/fuzz_address_uri.rs new file mode 100644 index 000000000000..e2d9e2fc8b08 --- /dev/null +++ b/substrate/primitives/core/fuzz/fuzz_targets/fuzz_address_uri.rs @@ -0,0 +1,53 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#![no_main] + +extern crate libfuzzer_sys; +extern crate regex; +extern crate sp_core; + +use libfuzzer_sys::fuzz_target; +use regex::Regex; +use sp_core::crypto::AddressUri; + +lazy_static::lazy_static! { + static ref SECRET_PHRASE_REGEX: Regex = Regex::new(r"^(?P[a-zA-Z0-9 ]+)?(?P(//?[^/]+)*)(///(?P.*))?$") + .expect("constructed from known-good static value; qed"); +} + +fuzz_target!(|input: &str| { + let regex_result = SECRET_PHRASE_REGEX.captures(input); + let manual_result = AddressUri::parse(input); + assert_eq!(regex_result.is_some(), manual_result.is_ok()); + if manual_result.is_err() { + let _ = format!("{}", manual_result.as_ref().err().unwrap()); + } + if let (Some(regex_result), Ok(manual_result)) = (regex_result, manual_result) { + assert_eq!(regex_result.name("phrase").map(|p| p.as_str()), manual_result.phrase); + + let manual_paths = manual_result + .paths + .iter() + .map(|s| "/".to_string() + s) + .collect::>() + .join(""); + + assert_eq!(regex_result.name("path").unwrap().as_str().to_string(), manual_paths); + assert_eq!(regex_result.name("password").map(|pass| pass.as_str()), manual_result.pass); + } +}); diff --git a/substrate/primitives/core/src/address_uri.rs b/substrate/primitives/core/src/address_uri.rs new file mode 100644 index 000000000000..862747c9a4b6 --- /dev/null +++ b/substrate/primitives/core/src/address_uri.rs @@ -0,0 +1,432 @@ +// This file is part of Substrate. + +// Copyright (C) Parity Technologies (UK) Ltd. +// SPDX-License-Identifier: Apache-2.0 + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//! Little util for parsing an address URI. Replaces regular expressions. + +#[cfg(all(not(feature = "std"), any(feature = "serde", feature = "full_crypto")))] +use sp_std::{ + alloc::string::{String, ToString}, + vec::Vec, +}; + +/// A container for results of parsing the address uri string. +/// +/// Intended to be equivalent of: +/// `Regex::new(r"^(?P[a-zA-Z0-9 ]+)?(?P(//?[^/]+)*)(///(?P.*))?$")` +/// which also handles soft and hard derivation paths: +/// `Regex::new(r"/(/?[^/]+)")` +/// +/// Example: +/// ``` +/// use sp_core::crypto::AddressUri; +/// let manual_result = AddressUri::parse("hello world/s//h///pass"); +/// assert_eq!( +/// manual_result.unwrap(), +/// AddressUri { phrase: Some("hello world"), paths: vec!["s", "/h"], pass: Some("pass") } +/// ); +/// ``` +#[derive(Debug, PartialEq)] +pub struct AddressUri<'a> { + /// Phrase, hexadecimal string, or ss58-compatible string. + pub phrase: Option<&'a str>, + /// Key derivation paths, ordered as in input string, + pub paths: Vec<&'a str>, + /// Password. + pub pass: Option<&'a str>, +} + +/// Errors that are possible during parsing the address URI. +#[allow(missing_docs)] +#[cfg_attr(feature = "std", derive(thiserror::Error))] +#[derive(Debug, PartialEq, Eq, Clone)] +pub enum Error { + #[cfg_attr(feature = "std", error("Invalid character in phrase:\n{0}"))] + InvalidCharacterInPhrase(InvalidCharacterInfo), + #[cfg_attr(feature = "std", error("Invalid character in password:\n{0}"))] + InvalidCharacterInPass(InvalidCharacterInfo), + #[cfg_attr(feature = "std", error("Missing character in hard path:\n{0}"))] + MissingCharacterInHardPath(InvalidCharacterInfo), + #[cfg_attr(feature = "std", error("Missing character in soft path:\n{0}"))] + MissingCharacterInSoftPath(InvalidCharacterInfo), +} + +impl Error { + /// Creates an instance of `Error::InvalidCharacterInPhrase` using given parameters. + pub fn in_phrase(input: &str, pos: usize) -> Self { + Self::InvalidCharacterInPhrase(InvalidCharacterInfo::new(input, pos)) + } + /// Creates an instance of `Error::InvalidCharacterInPass` using given parameters. + pub fn in_pass(input: &str, pos: usize) -> Self { + Self::InvalidCharacterInPass(InvalidCharacterInfo::new(input, pos)) + } + /// Creates an instance of `Error::MissingCharacterInHardPath` using given parameters. + pub fn in_hard_path(input: &str, pos: usize) -> Self { + Self::MissingCharacterInHardPath(InvalidCharacterInfo::new(input, pos)) + } + /// Creates an instance of `Error::MissingCharacterInSoftPath` using given parameters. + pub fn in_soft_path(input: &str, pos: usize) -> Self { + Self::MissingCharacterInSoftPath(InvalidCharacterInfo::new(input, pos)) + } +} + +/// Complementary error information. +/// +/// Strucutre contains complementary information about parsing address URI string. +/// String contains a copy of an original URI string, 0-based integer indicates position of invalid +/// character. +#[derive(Debug, PartialEq, Eq, Clone)] +pub struct InvalidCharacterInfo(String, usize); + +impl InvalidCharacterInfo { + fn new(info: &str, pos: usize) -> Self { + Self(info.to_string(), pos) + } +} + +impl sp_std::fmt::Display for InvalidCharacterInfo { + fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { + let (s, pos) = escape_string(&self.0, self.1); + write!(f, "{s}\n{i}^", i = sp_std::iter::repeat(" ").take(pos).collect::()) + } +} + +/// Escapes the control characters in given string, and recomputes the position if some characters +/// were actually escaped. +fn escape_string(input: &str, pos: usize) -> (String, usize) { + let mut out = String::with_capacity(2 * input.len()); + let mut out_pos = 0; + input + .chars() + .enumerate() + .map(|(i, c)| { + let esc = |c| (i, Some('\\'), c, 2); + match c { + '\t' => esc('t'), + '\n' => esc('n'), + '\r' => esc('r'), + '\x07' => esc('a'), + '\x08' => esc('b'), + '\x0b' => esc('v'), + '\x0c' => esc('f'), + _ => (i, None, c, 1), + } + }) + .for_each(|(i, maybe_escape, c, increment)| { + maybe_escape.map(|e| out.push(e)); + out.push(c); + if i < pos { + out_pos += increment; + } + }); + (out, out_pos) +} + +fn extract_prefix<'a>(input: &mut &'a str, is_allowed: &dyn Fn(char) -> bool) -> Option<&'a str> { + let output = input.trim_start_matches(is_allowed); + let prefix_len = input.len() - output.len(); + let prefix = if prefix_len > 0 { Some(&input[..prefix_len]) } else { None }; + *input = output; + prefix +} + +fn strip_prefix(input: &mut &str, prefix: &str) -> bool { + if let Some(stripped_input) = input.strip_prefix(prefix) { + *input = stripped_input; + true + } else { + false + } +} + +impl<'a> AddressUri<'a> { + /// Parses the given string. + pub fn parse(mut input: &'a str) -> Result { + let initial_input = input; + let initial_input_len = input.len(); + let phrase = extract_prefix(&mut input, &|ch: char| { + ch.is_ascii_digit() || ch.is_ascii_alphabetic() || ch == ' ' + }); + + let mut pass = None; + let mut paths = Vec::new(); + while !input.is_empty() { + let unstripped_input = input; + if strip_prefix(&mut input, "///") { + pass = Some(extract_prefix(&mut input, &|ch: char| ch != '\n').unwrap_or("")); + } else if strip_prefix(&mut input, "//") { + let path = extract_prefix(&mut input, &|ch: char| ch != '/') + .ok_or(Error::in_hard_path(initial_input, initial_input_len - input.len()))?; + assert!(path.len() > 0); + // hard path shall contain leading '/', so take it from unstripped input. + paths.push(&unstripped_input[1..path.len() + 2]); + } else if strip_prefix(&mut input, "/") { + paths.push( + extract_prefix(&mut input, &|ch: char| ch != '/').ok_or( + Error::in_soft_path(initial_input, initial_input_len - input.len()), + )?, + ); + } else { + return Err(if pass.is_some() { + Error::in_pass(initial_input, initial_input_len - input.len()) + } else { + Error::in_phrase(initial_input, initial_input_len - input.len()) + }); + } + } + + Ok(Self { phrase, paths, pass }) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use regex::Regex; + + lazy_static::lazy_static! { + static ref SECRET_PHRASE_REGEX: Regex = Regex::new(r"^(?P[a-zA-Z0-9 ]+)?(?P(//?[^/]+)*)(///(?P.*))?$") + .expect("constructed from known-good static value; qed"); + } + + fn check_with_regex(input: &str) { + let regex_result = SECRET_PHRASE_REGEX.captures(input); + let manual_result = AddressUri::parse(input); + assert_eq!(regex_result.is_some(), manual_result.is_ok()); + if let (Some(regex_result), Ok(manual_result)) = (regex_result, manual_result) { + assert_eq!( + regex_result.name("phrase").map(|phrase| phrase.as_str()), + manual_result.phrase + ); + + let manual_paths = manual_result + .paths + .iter() + .map(|s| "/".to_string() + s) + .collect::>() + .join(""); + + assert_eq!(regex_result.name("path").unwrap().as_str().to_string(), manual_paths); + assert_eq!( + regex_result.name("password").map(|phrase| phrase.as_str()), + manual_result.pass + ); + } + } + + fn check(input: &str, result: Result) { + let manual_result = AddressUri::parse(input); + assert_eq!(manual_result, result); + check_with_regex(input); + } + + #[test] + fn test00() { + check("///", Ok(AddressUri { phrase: None, pass: Some(""), paths: vec![] })); + } + + #[test] + fn test01() { + check("////////", Ok(AddressUri { phrase: None, pass: Some("/////"), paths: vec![] })) + } + + #[test] + fn test02() { + check( + "sdasd///asda", + Ok(AddressUri { phrase: Some("sdasd"), pass: Some("asda"), paths: vec![] }), + ); + } + + #[test] + fn test03() { + check( + "sdasd//asda", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["/asda"] }), + ); + } + + #[test] + fn test04() { + check("sdasd//a", Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["/a"] })); + } + + #[test] + fn test05() { + let input = "sdasd//"; + check(input, Err(Error::in_hard_path(input, 7))); + } + + #[test] + fn test06() { + check( + "sdasd/xx//asda", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["xx", "/asda"] }), + ); + } + + #[test] + fn test07() { + check( + "sdasd/xx//a/b//c///pass", + Ok(AddressUri { + phrase: Some("sdasd"), + pass: Some("pass"), + paths: vec!["xx", "/a", "b", "/c"], + }), + ); + } + + #[test] + fn test08() { + check( + "sdasd/xx//a", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["xx", "/a"] }), + ); + } + + #[test] + fn test09() { + let input = "sdasd/xx//"; + check(input, Err(Error::in_hard_path(input, 10))); + } + + #[test] + fn test10() { + check( + "sdasd/asda", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["asda"] }), + ); + } + + #[test] + fn test11() { + check( + "sdasd/asda//x", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["asda", "/x"] }), + ); + } + + #[test] + fn test12() { + check("sdasd/a", Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["a"] })); + } + + #[test] + fn test13() { + let input = "sdasd/"; + check(input, Err(Error::in_soft_path(input, 6))); + } + + #[test] + fn test14() { + check("sdasd", Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec![] })); + } + + #[test] + fn test15() { + let input = "sdasd."; + check(input, Err(Error::in_phrase(input, 5))); + } + + #[test] + fn test16() { + let input = "sd.asd/asd.a"; + check(input, Err(Error::in_phrase(input, 2))); + } + + #[test] + fn test17() { + let input = "sd.asd//asd.a"; + check(input, Err(Error::in_phrase(input, 2))); + } + + #[test] + fn test18() { + check( + "sdasd/asd.a", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["asd.a"] }), + ); + } + + #[test] + fn test19() { + check( + "sdasd//asd.a", + Ok(AddressUri { phrase: Some("sdasd"), pass: None, paths: vec!["/asd.a"] }), + ); + } + + #[test] + fn test20() { + let input = "///\n"; + check(input, Err(Error::in_pass(input, 3))); + } + + #[test] + fn test21() { + let input = "///a\n"; + check(input, Err(Error::in_pass(input, 4))); + } + + #[test] + fn test22() { + let input = "sd asd///asd.a\n"; + check(input, Err(Error::in_pass(input, 14))); + } + + #[test] + fn test_invalid_char_info_1() { + let expected = "01234\n^"; + let f = format!("{}", InvalidCharacterInfo::new("01234", 0)); + assert_eq!(expected, f); + } + + #[test] + fn test_invalid_char_info_2() { + let expected = "01\n ^"; + let f = format!("{}", InvalidCharacterInfo::new("01", 1)); + assert_eq!(expected, f); + } + + #[test] + fn test_invalid_char_info_3() { + let expected = "01234\n ^"; + let f = format!("{}", InvalidCharacterInfo::new("01234", 2)); + assert_eq!(expected, f); + } + + #[test] + fn test_invalid_char_info_4() { + let expected = "012\\n456\n ^"; + let f = format!("{}", InvalidCharacterInfo::new("012\n456", 3)); + assert_eq!(expected, f); + } + + #[test] + fn test_invalid_char_info_5() { + let expected = "012\\n456\n ^"; + let f = format!("{}", InvalidCharacterInfo::new("012\n456", 5)); + assert_eq!(expected, f); + } + + #[test] + fn test_invalid_char_info_6() { + let expected = "012\\f456\\t89\n ^"; + let f = format!("{}", InvalidCharacterInfo::new("012\x0c456\t89", 9)); + assert_eq!(expected, f); + } +} diff --git a/substrate/primitives/core/src/crypto.rs b/substrate/primitives/core/src/crypto.rs index d369de5a1c01..c9719e344d3e 100644 --- a/substrate/primitives/core/src/crypto.rs +++ b/substrate/primitives/core/src/crypto.rs @@ -25,8 +25,6 @@ use codec::{Decode, Encode, MaxEncodedLen}; use itertools::Itertools; #[cfg(feature = "std")] use rand::{rngs::OsRng, RngCore}; -#[cfg(feature = "std")] -use regex::Regex; use scale_info::TypeInfo; #[cfg(feature = "std")] pub use secrecy::{ExposeSecret, SecretString}; @@ -43,6 +41,11 @@ pub use ss58_registry::{from_known_address_format, Ss58AddressFormat, Ss58Addres /// Trait to zeroize a memory buffer. pub use zeroize::Zeroize; +#[cfg(feature = "std")] +pub use crate::address_uri::AddressUri; +#[cfg(any(feature = "std", feature = "full_crypto"))] +pub use crate::address_uri::Error as AddressUriError; + /// The root phrase for our publicly known keys. pub const DEV_PHRASE: &str = "bottom drive obey lake curtain smoke basket hold race lonely fit walk"; @@ -82,8 +85,8 @@ impl> UncheckedInto for S { #[cfg(feature = "full_crypto")] pub enum SecretStringError { /// The overall format was invalid (e.g. the seed phrase contained symbols). - #[cfg_attr(feature = "std", error("Invalid format"))] - InvalidFormat, + #[cfg_attr(feature = "std", error("Invalid format {0}"))] + InvalidFormat(AddressUriError), /// The seed phrase provided is not a valid BIP39 phrase. #[cfg_attr(feature = "std", error("Invalid phrase"))] InvalidPhrase, @@ -101,6 +104,13 @@ pub enum SecretStringError { InvalidPath, } +#[cfg(any(feature = "std", feature = "full_crypto"))] +impl From for SecretStringError { + fn from(e: AddressUriError) -> Self { + Self::InvalidFormat(e) + } +} + /// An error when deriving a key. #[cfg_attr(feature = "std", derive(thiserror::Error))] #[derive(Debug, Clone, PartialEq, Eq)] @@ -208,7 +218,7 @@ impl> From for DeriveJunction { /// An error type for SS58 decoding. #[cfg_attr(feature = "std", derive(thiserror::Error))] #[cfg_attr(not(feature = "std"), derive(Debug))] -#[derive(Clone, Copy, Eq, PartialEq)] +#[derive(Clone, Eq, PartialEq)] #[allow(missing_docs)] #[cfg(any(feature = "full_crypto", feature = "serde"))] pub enum PublicError { @@ -235,6 +245,11 @@ pub enum PublicError { InvalidPath, #[cfg_attr(feature = "std", error("Disallowed SS58 Address Format for this datatype."))] FormatNotAllowed, + #[cfg_attr(feature = "std", error("Password not allowed."))] + PasswordNotAllowed, + #[cfg(feature = "std")] + #[cfg_attr(feature = "std", error("Incorrect URI syntax {0}."))] + MalformedUri(#[from] AddressUriError), } #[cfg(feature = "std")] @@ -414,47 +429,40 @@ pub fn set_default_ss58_version(new_default: Ss58AddressFormat) { DEFAULT_VERSION.store(new_default.into(), core::sync::atomic::Ordering::Relaxed); } -#[cfg(feature = "std")] -lazy_static::lazy_static! { - static ref SS58_REGEX: Regex = Regex::new(r"^(?P[\w\d ]+)?(?P(//?[^/]+)*)$") - .expect("constructed from known-good static value; qed"); - static ref SECRET_PHRASE_REGEX: Regex = Regex::new(r"^(?P[\d\w ]+)?(?P(//?[^/]+)*)(///(?P.*))?$") - .expect("constructed from known-good static value; qed"); - static ref JUNCTION_REGEX: Regex = Regex::new(r"/(/?[^/]+)") - .expect("constructed from known-good static value; qed"); -} - #[cfg(feature = "std")] impl + AsRef<[u8]> + Public + Derive> Ss58Codec for T { fn from_string(s: &str) -> Result { - let cap = SS58_REGEX.captures(s).ok_or(PublicError::InvalidFormat)?; - let s = cap.name("ss58").map(|r| r.as_str()).unwrap_or(DEV_ADDRESS); + let cap = AddressUri::parse(s)?; + if cap.pass.is_some() { + return Err(PublicError::PasswordNotAllowed); + } + let s = cap.phrase.unwrap_or(DEV_ADDRESS); let addr = if let Some(stripped) = s.strip_prefix("0x") { let d = array_bytes::hex2bytes(stripped).map_err(|_| PublicError::InvalidFormat)?; Self::from_slice(&d).map_err(|()| PublicError::BadLength)? } else { Self::from_ss58check(s)? }; - if cap["path"].is_empty() { + if cap.paths.is_empty() { Ok(addr) } else { - let path = - JUNCTION_REGEX.captures_iter(&cap["path"]).map(|f| DeriveJunction::from(&f[1])); - addr.derive(path).ok_or(PublicError::InvalidPath) + addr.derive(cap.paths.iter().map(DeriveJunction::from)) + .ok_or(PublicError::InvalidPath) } } fn from_string_with_version(s: &str) -> Result<(Self, Ss58AddressFormat), PublicError> { - let cap = SS58_REGEX.captures(s).ok_or(PublicError::InvalidFormat)?; - let (addr, v) = Self::from_ss58check_with_version( - cap.name("ss58").map(|r| r.as_str()).unwrap_or(DEV_ADDRESS), - )?; - if cap["path"].is_empty() { + let cap = AddressUri::parse(s)?; + if cap.pass.is_some() { + return Err(PublicError::PasswordNotAllowed); + } + let (addr, v) = Self::from_ss58check_with_version(cap.phrase.unwrap_or(DEV_ADDRESS))?; + if cap.paths.is_empty() { Ok((addr, v)) } else { - let path = - JUNCTION_REGEX.captures_iter(&cap["path"]).map(|f| DeriveJunction::from(&f[1])); - addr.derive(path).ok_or(PublicError::InvalidPath).map(|a| (a, v)) + addr.derive(cap.paths.iter().map(DeriveJunction::from)) + .ok_or(PublicError::InvalidPath) + .map(|a| (a, v)) } } } @@ -817,22 +825,15 @@ impl sp_std::str::FromStr for SecretUri { type Err = SecretStringError; fn from_str(s: &str) -> Result { - let cap = SECRET_PHRASE_REGEX.captures(s).ok_or(SecretStringError::InvalidFormat)?; - - let junctions = JUNCTION_REGEX - .captures_iter(&cap["path"]) - .map(|f| DeriveJunction::from(&f[1])) - .collect::>(); - - let phrase = cap.name("phrase").map(|r| r.as_str()).unwrap_or(DEV_PHRASE); - let password = cap.name("password"); + let cap = AddressUri::parse(s)?; + let phrase = cap.phrase.unwrap_or(DEV_PHRASE); Ok(Self { phrase: SecretString::from_str(phrase).expect("Returns infallible error; qed"), - password: password.map(|v| { - SecretString::from_str(v.as_str()).expect("Returns infallible error; qed") - }), - junctions, + password: cap + .pass + .map(|v| SecretString::from_str(v).expect("Returns infallible error; qed")), + junctions: cap.paths.iter().map(DeriveJunction::from).collect::>(), }) } } diff --git a/substrate/primitives/core/src/lib.rs b/substrate/primitives/core/src/lib.rs index ec0641c54668..4873d1a21127 100644 --- a/substrate/primitives/core/src/lib.rs +++ b/substrate/primitives/core/src/lib.rs @@ -55,6 +55,8 @@ pub mod crypto; pub mod hexdisplay; pub use paste; +#[cfg(any(feature = "full_crypto", feature = "std"))] +mod address_uri; #[cfg(feature = "bandersnatch-experimental")] pub mod bandersnatch; #[cfg(feature = "bls-experimental")] From 490fb66537c60e1a391350b3f0a786efb7c17373 Mon Sep 17 00:00:00 2001 From: Adrian Catangiu Date: Fri, 17 Nov 2023 16:40:49 +0200 Subject: [PATCH 11/48] [trivial] asset-hubs runtimes: fix incorrect doc-comments (#2384) Fix some incorrect doc-comments --- .../runtimes/assets/asset-hub-rococo/src/xcm_config.rs | 4 ++-- .../runtimes/assets/asset-hub-westend/src/xcm_config.rs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs index b85cb76642fb..6e04924f0380 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs @@ -109,7 +109,7 @@ pub type CurrencyTransactor = CurrencyAdapter< (), >; -/// `AssetId`/`Balance` converter for `PoolAssets`. +/// `AssetId`/`Balance` converter for `TrustBackedAssets`. pub type TrustBackedAssetsConvertedConcreteId = assets_common::TrustBackedAssetsConvertedConcreteId; @@ -130,7 +130,7 @@ pub type FungiblesTransactor = FungiblesAdapter< CheckingAccount, >; -/// `AssetId/Balance` converter for `TrustBackedAssets` +/// `AssetId`/`Balance` converter for `ForeignAssets`. pub type ForeignAssetsConvertedConcreteId = assets_common::ForeignAssetsConvertedConcreteId< ( // Ignore `TrustBackedAssets` explicitly diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs index 17312c0f46ef..6942559671b2 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs @@ -109,7 +109,7 @@ pub type CurrencyTransactor = CurrencyAdapter< (), >; -/// `AssetId/Balance` converter for `TrustBackedAssets` +/// `AssetId`/`Balance` converter for `TrustBackedAssets`. pub type TrustBackedAssetsConvertedConcreteId = assets_common::TrustBackedAssetsConvertedConcreteId; @@ -130,7 +130,7 @@ pub type FungiblesTransactor = FungiblesAdapter< CheckingAccount, >; -/// `AssetId/Balance` converter for `TrustBackedAssets` +/// `AssetId`/`Balance` converter for `ForeignAssets`. pub type ForeignAssetsConvertedConcreteId = assets_common::ForeignAssetsConvertedConcreteId< ( // Ignore `TrustBackedAssets` explicitly From 079b14f624bef9dd901f93c29424cba0ebbd325d Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Fri, 17 Nov 2023 18:01:08 +0200 Subject: [PATCH 12/48] Do not panic if the `fdlimit` call to increase the file descriptor limit fails (#2155) # Description Sometimes changing file descriptor limits is not allowed, but there is no need to crash the node if/when this happens. Since `fdlimit`'s author decided to use panics instead of returning `Result`, we need to catch it. # Checklist - [x] My PR includes a detailed description as outlined in the "Description" section above - [ ] My PR follows the [labeling requirements](CONTRIBUTING.md#Process) of this project (at minimum one label for `T` required) - [ ] I have made corresponding changes to the documentation (if applicable) - [ ] I have added tests that prove my fix is effective or that my feature works (if applicable) --------- Co-authored-by: Koute --- Cargo.lock | 13 +++++++------ substrate/client/cli/Cargo.toml | 2 +- substrate/client/cli/src/config.rs | 23 +++++++++++++++++------ substrate/client/service/test/Cargo.toml | 2 +- substrate/client/service/test/src/lib.rs | 2 +- 5 files changed, 27 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 64da4b8996ca..da266eb5d472 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5342,11 +5342,12 @@ dependencies = [ [[package]] name = "fdlimit" -version = "0.2.1" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c4c9e43643f5a3be4ca5b67d26b98031ff9db6806c3440ae32e02e3ceac3f1b" +checksum = "e182f7dbc2ef73d9ef67351c5fbbea084729c48362d3ce9dd44c28e32e277fe5" dependencies = [ "libc", + "thiserror", ] [[package]] @@ -19321,9 +19322,9 @@ checksum = "222a222a5bfe1bba4a77b45ec488a741b3cb8872e5e499451fd7d0129c9c7c3d" [[package]] name = "thiserror" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d6d7a740b8a666a7e828dd00da9c0dc290dff53154ea77ac109281de90589b7" +checksum = "f9a7210f5c9a7156bb50aa36aed4c95afb51df0df00713949448cf9e97d382d2" dependencies = [ "thiserror-impl", ] @@ -19350,9 +19351,9 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "1.0.48" +version = "1.0.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49922ecae66cc8a249b77e68d1d0623c1b2c514f0060c27cdc68bd62a1219d35" +checksum = "266b2e40bc00e5a6c09c3584011e08b06f123c00362c92b975ba9843aaaa14b8" dependencies = [ "proc-macro2", "quote", diff --git a/substrate/client/cli/Cargo.toml b/substrate/client/cli/Cargo.toml index c415527c372c..c4464c5f787d 100644 --- a/substrate/client/cli/Cargo.toml +++ b/substrate/client/cli/Cargo.toml @@ -16,7 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] array-bytes = "6.1" chrono = "0.4.27" clap = { version = "4.4.6", features = ["derive", "string", "wrap_help"] } -fdlimit = "0.2.1" +fdlimit = "0.3.0" futures = "0.3.21" itertools = "0.10.3" libp2p-identity = { version = "0.1.3", features = ["peerid", "ed25519"]} diff --git a/substrate/client/cli/src/config.rs b/substrate/client/cli/src/config.rs index 4d218da6aa89..b842df5a690a 100644 --- a/substrate/client/cli/src/config.rs +++ b/substrate/client/cli/src/config.rs @@ -605,14 +605,25 @@ pub trait CliConfiguration: Sized { logger.init()?; - if let Some(new_limit) = fdlimit::raise_fd_limit() { - if new_limit < RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT { + match fdlimit::raise_fd_limit() { + Ok(fdlimit::Outcome::LimitRaised { to, .. }) => + if to < RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT { + warn!( + "Low open file descriptor limit configured for the process. \ + Current value: {:?}, recommended value: {:?}.", + to, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT, + ); + }, + Ok(fdlimit::Outcome::Unsupported) => { + // Unsupported platform (non-Linux) + }, + Err(error) => { warn!( - "Low open file descriptor limit configured for the process. \ - Current value: {:?}, recommended value: {:?}.", - new_limit, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT, + "Failed to configure file descriptor limit for the process: \ + {}, recommended value: {:?}.", + error, RECOMMENDED_OPEN_FILE_DESCRIPTOR_LIMIT, ); - } + }, } Ok(()) diff --git a/substrate/client/service/test/Cargo.toml b/substrate/client/service/test/Cargo.toml index 670312e4161a..c6091f97d637 100644 --- a/substrate/client/service/test/Cargo.toml +++ b/substrate/client/service/test/Cargo.toml @@ -14,7 +14,7 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] async-channel = "1.8.0" array-bytes = "6.1" -fdlimit = "0.2.1" +fdlimit = "0.3.0" futures = "0.3.21" log = "0.4.17" parity-scale-codec = "3.6.1" diff --git a/substrate/client/service/test/src/lib.rs b/substrate/client/service/test/src/lib.rs index 9700c7643c48..456df73459a3 100644 --- a/substrate/client/service/test/src/lib.rs +++ b/substrate/client/service/test/src/lib.rs @@ -285,7 +285,7 @@ where base_port: u16, ) -> TestNet { sp_tracing::try_init_simple(); - fdlimit::raise_fd_limit(); + fdlimit::raise_fd_limit().unwrap(); let runtime = Runtime::new().expect("Error creating tokio runtime"); let mut net = TestNet { runtime, From 0385902cd05523155832b155fe7ce4d1eaf7b779 Mon Sep 17 00:00:00 2001 From: Branislav Kontur Date: Fri, 17 Nov 2023 17:06:45 +0100 Subject: [PATCH 13/48] Relax `force_default_xcm_version` for testnet system parachains (#2385) This PR fixes two things: - relax `force_default_xcm_version` for testnet system parachains (e.g. BridgeHubWestend has now 2 and there is no way to change it to 3, so we need to call `force_xcm_version(3)` for every parachain that it is connected to, because we send XCMv3 messages) - add `Storage` item to `PolkadotXcm` pallet definition (now we cannot see storage items for `pallet_xcm` in PJS) ## TODO - [ ] when merged open PR to `polkadot-fellows/runtimes` repo --- .../assets/asset-hub-rococo/src/xcm_config.rs | 16 +++++++++------- .../assets/asset-hub-westend/src/xcm_config.rs | 16 +++++++++------- .../bridge-hubs/bridge-hub-rococo/src/lib.rs | 2 +- .../bridge-hub-rococo/src/xcm_config.rs | 16 +++++++++------- .../bridge-hubs/bridge-hub-westend/src/lib.rs | 2 +- .../bridge-hub-westend/src/xcm_config.rs | 16 +++++++++------- .../collectives-westend/src/xcm_config.rs | 6 ++++-- 7 files changed, 42 insertions(+), 32 deletions(-) diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs index 6e04924f0380..4da0a2500a54 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/xcm_config.rs @@ -275,13 +275,15 @@ impl Contains for SafeCallFilter { matches!( call, - RuntimeCall::PolkadotXcm(pallet_xcm::Call::force_xcm_version { .. }) | - RuntimeCall::System( - frame_system::Call::set_heap_pages { .. } | - frame_system::Call::set_code { .. } | - frame_system::Call::set_code_without_checks { .. } | - frame_system::Call::kill_prefix { .. }, - ) | RuntimeCall::ParachainSystem(..) | + RuntimeCall::PolkadotXcm( + pallet_xcm::Call::force_xcm_version { .. } | + pallet_xcm::Call::force_default_xcm_version { .. } + ) | RuntimeCall::System( + frame_system::Call::set_heap_pages { .. } | + frame_system::Call::set_code { .. } | + frame_system::Call::set_code_without_checks { .. } | + frame_system::Call::kill_prefix { .. }, + ) | RuntimeCall::ParachainSystem(..) | RuntimeCall::Timestamp(..) | RuntimeCall::Balances(..) | RuntimeCall::CollatorSelection( diff --git a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs index 6942559671b2..4760e087e24f 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-westend/src/xcm_config.rs @@ -272,13 +272,15 @@ impl Contains for SafeCallFilter { matches!( call, - RuntimeCall::PolkadotXcm(pallet_xcm::Call::force_xcm_version { .. }) | - RuntimeCall::System( - frame_system::Call::set_heap_pages { .. } | - frame_system::Call::set_code { .. } | - frame_system::Call::set_code_without_checks { .. } | - frame_system::Call::kill_prefix { .. }, - ) | RuntimeCall::ParachainSystem(..) | + RuntimeCall::PolkadotXcm( + pallet_xcm::Call::force_xcm_version { .. } | + pallet_xcm::Call::force_default_xcm_version { .. } + ) | RuntimeCall::System( + frame_system::Call::set_heap_pages { .. } | + frame_system::Call::set_code { .. } | + frame_system::Call::set_code_without_checks { .. } | + frame_system::Call::kill_prefix { .. }, + ) | RuntimeCall::ParachainSystem(..) | RuntimeCall::Timestamp(..) | RuntimeCall::Balances(..) | RuntimeCall::CollatorSelection( diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs index 5a44ccbb75a2..8e138822696e 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/lib.rs @@ -498,7 +498,7 @@ construct_runtime!( // XCM helpers. XcmpQueue: cumulus_pallet_xcmp_queue::{Pallet, Call, Storage, Event} = 30, - PolkadotXcm: pallet_xcm::{Pallet, Call, Event, Origin, Config} = 31, + PolkadotXcm: pallet_xcm::{Pallet, Call, Storage, Event, Origin, Config} = 31, CumulusXcm: cumulus_pallet_xcm::{Pallet, Event, Origin} = 32, DmpQueue: cumulus_pallet_dmp_queue::{Pallet, Call, Storage, Event} = 33, diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/xcm_config.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/xcm_config.rs index 1436c5b96a33..de7b5315c883 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-rococo/src/xcm_config.rs @@ -161,13 +161,15 @@ impl Contains for SafeCallFilter { matches!( call, - RuntimeCall::PolkadotXcm(pallet_xcm::Call::force_xcm_version { .. }) | - RuntimeCall::System( - frame_system::Call::set_heap_pages { .. } | - frame_system::Call::set_code { .. } | - frame_system::Call::set_code_without_checks { .. } | - frame_system::Call::kill_prefix { .. }, - ) | RuntimeCall::ParachainSystem(..) | + RuntimeCall::PolkadotXcm( + pallet_xcm::Call::force_xcm_version { .. } | + pallet_xcm::Call::force_default_xcm_version { .. } + ) | RuntimeCall::System( + frame_system::Call::set_heap_pages { .. } | + frame_system::Call::set_code { .. } | + frame_system::Call::set_code_without_checks { .. } | + frame_system::Call::kill_prefix { .. }, + ) | RuntimeCall::ParachainSystem(..) | RuntimeCall::Timestamp(..) | RuntimeCall::Balances(..) | RuntimeCall::CollatorSelection( diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs index d1d2b4a41595..9c97728058f7 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/lib.rs @@ -497,7 +497,7 @@ construct_runtime!( // XCM helpers. XcmpQueue: cumulus_pallet_xcmp_queue::{Pallet, Call, Storage, Event} = 30, - PolkadotXcm: pallet_xcm::{Pallet, Call, Event, Origin, Config} = 31, + PolkadotXcm: pallet_xcm::{Pallet, Call, Storage, Event, Origin, Config} = 31, CumulusXcm: cumulus_pallet_xcm::{Pallet, Event, Origin} = 32, DmpQueue: cumulus_pallet_dmp_queue::{Pallet, Call, Storage, Event} = 33, diff --git a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/xcm_config.rs b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/xcm_config.rs index 7084882c41f9..c89ee91c5e44 100644 --- a/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/bridge-hubs/bridge-hub-westend/src/xcm_config.rs @@ -150,13 +150,15 @@ impl Contains for SafeCallFilter { matches!( call, - RuntimeCall::PolkadotXcm(pallet_xcm::Call::force_xcm_version { .. }) | - RuntimeCall::System( - frame_system::Call::set_heap_pages { .. } | - frame_system::Call::set_code { .. } | - frame_system::Call::set_code_without_checks { .. } | - frame_system::Call::kill_prefix { .. }, - ) | RuntimeCall::ParachainSystem(..) | + RuntimeCall::PolkadotXcm( + pallet_xcm::Call::force_xcm_version { .. } | + pallet_xcm::Call::force_default_xcm_version { .. } + ) | RuntimeCall::System( + frame_system::Call::set_heap_pages { .. } | + frame_system::Call::set_code { .. } | + frame_system::Call::set_code_without_checks { .. } | + frame_system::Call::kill_prefix { .. }, + ) | RuntimeCall::ParachainSystem(..) | RuntimeCall::Timestamp(..) | RuntimeCall::Balances(..) | RuntimeCall::CollatorSelection( diff --git a/cumulus/parachains/runtimes/collectives/collectives-westend/src/xcm_config.rs b/cumulus/parachains/runtimes/collectives/collectives-westend/src/xcm_config.rs index d58995827fa4..cefc099c96f9 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-westend/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/collectives/collectives-westend/src/xcm_config.rs @@ -178,8 +178,10 @@ impl Contains for SafeCallFilter { pallet_collator_selection::Call::add_invulnerable { .. } | pallet_collator_selection::Call::remove_invulnerable { .. }, ) | RuntimeCall::Session(pallet_session::Call::purge_keys { .. }) | - RuntimeCall::PolkadotXcm(pallet_xcm::Call::force_xcm_version { .. }) | - RuntimeCall::XcmpQueue(..) | + RuntimeCall::PolkadotXcm( + pallet_xcm::Call::force_xcm_version { .. } | + pallet_xcm::Call::force_default_xcm_version { .. } + ) | RuntimeCall::XcmpQueue(..) | RuntimeCall::MessageQueue(..) | RuntimeCall::Alliance( // `init_members` accepts unbounded vecs as arguments, From 9e34163aca9d58d7351451833d9cd214b266e7f0 Mon Sep 17 00:00:00 2001 From: Sebastian Kunert Date: Fri, 17 Nov 2023 17:10:46 +0100 Subject: [PATCH 14/48] Make collator RPC mode non-experimental (#2381) The `--relay-chain-rpc-urls` CLI flag has been available for a while now. We have collators with this running and parachain teams are also using it. It should be fine now to remove the experimental status. --- cumulus/client/cli/src/lib.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/cumulus/client/cli/src/lib.rs b/cumulus/client/cli/src/lib.rs index 1b18ed064373..a2238b73b2b5 100644 --- a/cumulus/client/cli/src/lib.rs +++ b/cumulus/client/cli/src/lib.rs @@ -296,7 +296,14 @@ pub struct RunCmd { #[arg(long, conflicts_with = "validator")] pub collator: bool, - /// EXPERIMENTAL: Specify an URL to a relay chain full node to communicate with. + /// Creates a less resource-hungry node that retrieves relay chain data from an RPC endpoint. + /// + /// The provided URLs should point to RPC endpoints of the relay chain. + /// This node connects to the remote nodes following the order they were specified in. If the + /// connection fails, it attempts to connect to the next endpoint in the list. + /// + /// Note: This option doesn't stop the node from connecting to the relay chain network but + /// reduces bandwidth use. #[arg( long, value_parser = validate_relay_chain_url, From 82912acb33a9030c0ef3bf590a34fca09b72dc5f Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Fri, 17 Nov 2023 20:14:45 +0400 Subject: [PATCH 15/48] Fix migrations and add CI check for new system chains (#2336) Westend Collectives migration CI check can be fixed once we have https://github.com/paritytech/try-runtime-cli/pull/58, will open another PR once it is available. - [x] Remove deprecated `DmpQueue` pallet from Rococo Contracts, the migration is complete - [x] Fix Asset Hub Rococo storage versions - [x] Add migration check CI for Asset Hub Rococo and Westend Bridge Hub --- .gitlab/pipeline/check.yml | 25 ++++++++ .../assets/asset-hub-rococo/src/lib.rs | 59 ++++++++++++++++++- .../contracts/contracts-rococo/src/lib.rs | 1 - .../contracts-rococo/src/xcm_config.rs | 6 -- 4 files changed, 82 insertions(+), 9 deletions(-) diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 4071fdf9758b..429491fb1742 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -151,6 +151,31 @@ check-runtime-migration-asset-hub-westend: PACKAGE: "asset-hub-westend-runtime" WASM: "asset_hub_westend_runtime.compact.compressed.wasm" URI: "wss://westend-asset-hub-rpc.polkadot.io:443" + +check-runtime-migration-asset-hub-rococo: + stage: check + extends: + - .docker-env + - .test-pr-refs + - .check-runtime-migration + variables: + NETWORK: "asset-hub-rococo" + PACKAGE: "asset-hub-rococo-runtime" + WASM: "asset_hub_rococo_runtime.compact.compressed.wasm" + URI: "wss://rococo-asset-hub-rpc.polkadot.io:443" + +# Check runtime migrations for Parity managed bridge hub chains +check-runtime-migration-bridge-hub-westend: + stage: check + extends: + - .docker-env + - .test-pr-refs + - .check-runtime-migration + variables: + NETWORK: "bridge-hub-westend" + PACKAGE: "bridge-hub-westend-runtime" + WASM: "bridge_hub_westend_runtime.compact.compressed.wasm" + URI: "wss://westend-bridge-hub-rpc.polkadot.io:443" check-runtime-migration-bridge-hub-rococo: stage: check diff --git a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs index 4b4ae61a3e8d..b274f45877b3 100644 --- a/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/assets/asset-hub-rococo/src/lib.rs @@ -39,7 +39,9 @@ use sp_api::impl_runtime_apis; use sp_core::{crypto::KeyTypeId, OpaqueMetadata}; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, - traits::{AccountIdConversion, AccountIdLookup, BlakeTwo256, Block as BlockT, Verify}, + traits::{ + AccountIdConversion, AccountIdLookup, BlakeTwo256, Block as BlockT, Saturating, Verify, + }, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, Permill, }; @@ -959,7 +961,60 @@ pub type SignedExtra = ( pub type UncheckedExtrinsic = generic::UncheckedExtrinsic; /// Migrations to apply on runtime upgrade. -pub type Migrations = (pallet_collator_selection::migration::v1::MigrateToV1,); +pub type Migrations = + (pallet_collator_selection::migration::v1::MigrateToV1, InitStorageVersions); + +/// Migration to initialize storage versions for pallets added after genesis. +/// +/// This is now done automatically (see ), +/// but some pallets had made it in and had storage set in them for this parachain before it was +/// merged. +pub struct InitStorageVersions; + +impl frame_support::traits::OnRuntimeUpgrade for InitStorageVersions { + fn on_runtime_upgrade() -> Weight { + use frame_support::traits::{GetStorageVersion, StorageVersion}; + + let mut writes = 0; + + if PolkadotXcm::on_chain_storage_version() == StorageVersion::new(0) { + PolkadotXcm::current_storage_version().put::(); + writes.saturating_inc(); + } + + if Multisig::on_chain_storage_version() == StorageVersion::new(0) { + Multisig::current_storage_version().put::(); + writes.saturating_inc(); + } + + if Assets::on_chain_storage_version() == StorageVersion::new(0) { + Assets::current_storage_version().put::(); + writes.saturating_inc(); + } + + if Uniques::on_chain_storage_version() == StorageVersion::new(0) { + Uniques::current_storage_version().put::(); + writes.saturating_inc(); + } + + if Nfts::on_chain_storage_version() == StorageVersion::new(0) { + Nfts::current_storage_version().put::(); + writes.saturating_inc(); + } + + if ForeignAssets::on_chain_storage_version() == StorageVersion::new(0) { + ForeignAssets::current_storage_version().put::(); + writes.saturating_inc(); + } + + if PoolAssets::on_chain_storage_version() == StorageVersion::new(0) { + PoolAssets::current_storage_version().put::(); + writes.saturating_inc(); + } + + ::DbWeight::get().reads_writes(7, writes) + } +} /// Executive: handles dispatch to the various modules. pub type Executive = frame_executive::Executive< diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs index 5b828bad0c7f..9d6a53c5ed34 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/lib.rs @@ -405,7 +405,6 @@ construct_runtime!( XcmpQueue: cumulus_pallet_xcmp_queue::{Pallet, Call, Storage, Event} = 30, PolkadotXcm: pallet_xcm::{Pallet, Call, Storage, Event, Origin, Config} = 31, CumulusXcm: cumulus_pallet_xcm::{Pallet, Event, Origin} = 32, - DmpQueue: cumulus_pallet_dmp_queue::{Pallet, Call, Storage, Event} = 33, MessageQueue: pallet_message_queue::{Pallet, Call, Storage, Event} = 34, // Smart Contracts. diff --git a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/xcm_config.rs b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/xcm_config.rs index faee1c68fe6c..2ac93aed3f8d 100644 --- a/cumulus/parachains/runtimes/contracts/contracts-rococo/src/xcm_config.rs +++ b/cumulus/parachains/runtimes/contracts/contracts-rococo/src/xcm_config.rs @@ -304,9 +304,3 @@ impl cumulus_pallet_xcmp_queue::Config for Runtime { parameter_types! { pub const RelayOrigin: AggregateMessageOrigin = AggregateMessageOrigin::Parent; } - -impl cumulus_pallet_dmp_queue::Config for Runtime { - type WeightInfo = cumulus_pallet_dmp_queue::weights::SubstrateWeight; - type RuntimeEvent = crate::RuntimeEvent; - type DmpSink = frame_support::traits::EnqueueWithOrigin; -} From 1d1c3719487c8b9f9b770e5c33e8e8b237bd8824 Mon Sep 17 00:00:00 2001 From: Davide Galassi Date: Fri, 17 Nov 2023 23:48:34 +0100 Subject: [PATCH 16/48] Bump bandersnatch VRF revision (#2389) Closes https://github.com/paritytech/polkadot-sdk/issues/2327 cc @burdges --- Cargo.lock | 40 ++++++++++++++-------------- substrate/primitives/core/Cargo.toml | 2 +- 2 files changed, 21 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index da266eb5d472..ed9997101fbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,7 +544,7 @@ dependencies = [ [[package]] name = "ark-secret-scalar" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=cbc342e#cbc342e95d3cbcd3c5ba8d45af7200eb58e63502" +source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" dependencies = [ "ark-ec", "ark-ff", @@ -593,7 +593,7 @@ dependencies = [ [[package]] name = "ark-transcript" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=cbc342e#cbc342e95d3cbcd3c5ba8d45af7200eb58e63502" +source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" dependencies = [ "ark-ff", "ark-serialize", @@ -1369,8 +1369,8 @@ dependencies = [ [[package]] name = "bandersnatch_vrfs" -version = "0.0.3" -source = "git+https://github.com/w3f/ring-vrf?rev=cbc342e#cbc342e95d3cbcd3c5ba8d45af7200eb58e63502" +version = "0.0.4" +source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" dependencies = [ "ark-bls12-381", "ark-ec", @@ -1493,8 +1493,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" dependencies = [ "bitcoin_hashes", - "rand 0.8.5", - "rand_core 0.6.4", + "rand 0.7.3", + "rand_core 0.5.1", "serde", "unicode-normalization", ] @@ -3065,7 +3065,7 @@ dependencies = [ [[package]] name = "common" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof#edd1e90b847e560bf60fc2e8712235ccfa11a9a9" +source = "git+https://github.com/burdges/ring-proof?branch=patch-1#05a756076cb20f981a52afea3a620168de49f95f" dependencies = [ "ark-ec", "ark-ff", @@ -4814,7 +4814,7 @@ checksum = "86e3bdc80eee6e16b2b6b0f87fbc98c04bee3455e35174c0de1a125d0688c632" [[package]] name = "dleq_vrf" version = "0.0.2" -source = "git+https://github.com/w3f/ring-vrf?rev=cbc342e#cbc342e95d3cbcd3c5ba8d45af7200eb58e63502" +source = "git+https://github.com/w3f/ring-vrf?rev=3ddc205#3ddc2051066c4b3f0eadd0ba5700df12500d9754" dependencies = [ "ark-ec", "ark-ff", @@ -5389,7 +5389,7 @@ dependencies = [ [[package]] name = "fflonk" version = "0.1.0" -source = "git+https://github.com/w3f/fflonk#26a5045b24e169cffc1f9328ca83d71061145c40" +source = "git+https://github.com/w3f/fflonk#1beb0585e1c8488956fac7f05da061f9b41e8948" dependencies = [ "ark-ec", "ark-ff", @@ -14445,7 +14445,7 @@ dependencies = [ [[package]] name = "ring" version = "0.1.0" -source = "git+https://github.com/w3f/ring-proof#edd1e90b847e560bf60fc2e8712235ccfa11a9a9" +source = "git+https://github.com/burdges/ring-proof?branch=patch-1#05a756076cb20f981a52afea3a620168de49f95f" dependencies = [ "ark-ec", "ark-ff", @@ -17607,7 +17607,7 @@ dependencies = [ [[package]] name = "sp-crypto-ec-utils" version = "0.4.1" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "ark-bls12-377", "ark-bls12-377-ext", @@ -17645,7 +17645,7 @@ dependencies = [ [[package]] name = "sp-debug-derive" version = "8.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "proc-macro2", "quote", @@ -17665,7 +17665,7 @@ dependencies = [ [[package]] name = "sp-externalities" version = "0.19.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "environmental", "parity-scale-codec", @@ -17897,7 +17897,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface" version = "17.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "bytes", "impl-trait-for-tuples", @@ -17926,7 +17926,7 @@ dependencies = [ [[package]] name = "sp-runtime-interface-proc-macro" version = "11.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "Inflector", "proc-macro-crate", @@ -18054,7 +18054,7 @@ version = "8.0.0" [[package]] name = "sp-std" version = "8.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" [[package]] name = "sp-storage" @@ -18071,7 +18071,7 @@ dependencies = [ [[package]] name = "sp-storage" version = "13.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "impl-serde", "parity-scale-codec", @@ -18120,7 +18120,7 @@ dependencies = [ [[package]] name = "sp-tracing" version = "10.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "parity-scale-codec", "sp-std 8.0.0 (git+https://github.com/paritytech/polkadot-sdk)", @@ -18221,7 +18221,7 @@ dependencies = [ [[package]] name = "sp-wasm-interface" version = "14.0.0" -source = "git+https://github.com/paritytech/polkadot-sdk#fe9435db2fda7c9e2f4e29521564c72cac38f59b" +source = "git+https://github.com/paritytech/polkadot-sdk#82912acb33a9030c0ef3bf590a34fca09b72dc5f" dependencies = [ "anyhow", "impl-trait-for-tuples", @@ -20008,7 +20008,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.7", - "rand 0.8.5", + "rand 0.7.3", "static_assertions", ] diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 9ecce0a22f5f..25478bed2d95 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -56,7 +56,7 @@ sp-runtime-interface = { path = "../runtime-interface", default-features = false # bls crypto w3f-bls = { version = "0.1.3", default-features = false, optional = true} # bandersnatch crypto -bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "cbc342e", default-features = false, optional = true } +bandersnatch_vrfs = { git = "https://github.com/w3f/ring-vrf", rev = "3ddc205", default-features = false, optional = true } [dev-dependencies] criterion = "0.4.0" From 794ee98049f546d39923b7f34ffcb7e719ae349d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Sat, 18 Nov 2023 13:58:16 +0200 Subject: [PATCH 17/48] Bump secp256k1 from 0.24.3 to 0.28.0 (#2357) Bumps [secp256k1](https://github.com/rust-bitcoin/rust-secp256k1) from 0.24.3 to 0.28.0.
Changelog

Sourced from secp256k1's changelog.

0.28.0 - 2023-10-23

  • Add bindings to the ElligatorSwift implementation #627
  • Depend on recent release of bitcoin_hashes v0.13.0 #621
  • Add a verify function to PublicKey #618
  • Add serialize function for schnorr::Signature #607
  • Bump MSRV to 1.48 #595
  • Remove implementations of PartialEq, Eq, PartialOrd, Ord, and Hash from the impl_array_newtype macro. Users will now need to derive these traits if they are wanted.

0.27.0 - 2023-03-15

0.26.0 - 2202-12-19

  • Update libsecp25k1 to v0.2.0

0.25.0 - 2022-12-07

0.24.1 - 2022-10-25

0.24.0 - 2022-07-20

0.23.4 - 2022-07-14

0.23.3 - 2022-06-29

0.23.2 - 2022-06-27

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=secp256k1&package-manager=cargo&previous-version=0.24.3&new-version=0.28.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Liam Aharon --- Cargo.lock | 8 ++++---- substrate/primitives/core/Cargo.toml | 2 +- substrate/primitives/core/src/ecdsa.rs | 13 ++++--------- substrate/primitives/io/Cargo.toml | 2 +- substrate/primitives/io/src/lib.rs | 4 ++-- 5 files changed, 12 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index ed9997101fbc..ac64e65ee0ec 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -16602,18 +16602,18 @@ dependencies = [ [[package]] name = "secp256k1" -version = "0.24.3" +version = "0.28.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b1629c9c557ef9b293568b338dddfc8208c98a18c59d722a9d53f859d9c9b62" +checksum = "2acea373acb8c21ecb5a23741452acd2593ed44ee3d343e72baaa143bc89d0d5" dependencies = [ "secp256k1-sys", ] [[package]] name = "secp256k1-sys" -version = "0.6.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83080e2c2fc1006e625be82e5d1eb6a43b7fd9578b617fcc55814daf286bba4b" +checksum = "09e67c467c38fd24bd5499dc9a18183b31575c12ee549197e3e20d57aa4fe3b7" dependencies = [ "cc", ] diff --git a/substrate/primitives/core/Cargo.toml b/substrate/primitives/core/Cargo.toml index 25478bed2d95..34485c72ab03 100644 --- a/substrate/primitives/core/Cargo.toml +++ b/substrate/primitives/core/Cargo.toml @@ -49,7 +49,7 @@ blake2 = { version = "0.10.4", default-features = false, optional = true } libsecp256k1 = { version = "0.7", default-features = false, features = ["static-context"], optional = true } schnorrkel = { version = "0.9.1", features = ["preaudit_deprecated", "u64_backend"], default-features = false } merlin = { version = "2.0", default-features = false } -secp256k1 = { version = "0.24.0", default-features = false, features = ["recovery", "alloc"], optional = true } +secp256k1 = { version = "0.28.0", default-features = false, features = ["recovery", "alloc"], optional = true } sp-core-hashing = { path = "hashing", default-features = false, optional = true } sp-runtime-interface = { path = "../runtime-interface", default-features = false} diff --git a/substrate/primitives/core/src/ecdsa.rs b/substrate/primitives/core/src/ecdsa.rs index 603fa515a30e..471714582a6b 100644 --- a/substrate/primitives/core/src/ecdsa.rs +++ b/substrate/primitives/core/src/ecdsa.rs @@ -336,7 +336,7 @@ impl Signature { pub fn recover_prehashed(&self, message: &[u8; 32]) -> Option { let rid = RecoveryId::from_i32(self.0[64] as i32).ok()?; let sig = RecoverableSignature::from_compact(&self.0[..64], rid).ok()?; - let message = Message::from_slice(message).expect("Message is 32 bytes; qed"); + let message = Message::from_digest_slice(message).expect("Message is 32 bytes; qed"); #[cfg(feature = "std")] let context = SECP256K1; @@ -458,7 +458,7 @@ impl Pair { /// Sign a pre-hashed message pub fn sign_prehashed(&self, message: &[u8; 32]) -> Signature { - let message = Message::from_slice(message).expect("Message is 32 bytes; qed"); + let message = Message::from_digest_slice(message).expect("Message is 32 bytes; qed"); #[cfg(feature = "std")] let context = SECP256K1; @@ -508,12 +508,7 @@ impl Pair { #[cfg(feature = "full_crypto")] impl Drop for Pair { fn drop(&mut self) { - let ptr = self.secret.as_mut_ptr(); - for off in 0..self.secret.len() { - unsafe { - core::ptr::write_volatile(ptr.add(off), 0); - } - } + self.secret.non_secure_erase() } } @@ -760,7 +755,7 @@ mod test { let msg = [0u8; 32]; let sig1 = pair.sign_prehashed(&msg); let sig2: Signature = { - let message = Message::from_slice(&msg).unwrap(); + let message = Message::from_digest_slice(&msg).unwrap(); SECP256K1.sign_ecdsa_recoverable(&message, &pair.secret).into() }; assert_eq!(sig1, sig2); diff --git a/substrate/primitives/io/Cargo.toml b/substrate/primitives/io/Cargo.toml index 445104b736e0..59df8895bb7f 100644 --- a/substrate/primitives/io/Cargo.toml +++ b/substrate/primitives/io/Cargo.toml @@ -28,7 +28,7 @@ sp-trie = { path = "../trie", default-features = false, optional = true} sp-externalities = { path = "../externalities", default-features = false} sp-tracing = { path = "../tracing", default-features = false} log = { version = "0.4.17", optional = true } -secp256k1 = { version = "0.24.0", features = ["recovery", "global-context"], optional = true } +secp256k1 = { version = "0.28.0", features = ["recovery", "global-context"], optional = true } tracing = { version = "0.1.29", default-features = false } tracing-core = { version = "0.1.28", default-features = false} diff --git a/substrate/primitives/io/src/lib.rs b/substrate/primitives/io/src/lib.rs index c4182d6ab3a0..a300152ee66d 100644 --- a/substrate/primitives/io/src/lib.rs +++ b/substrate/primitives/io/src/lib.rs @@ -1139,7 +1139,7 @@ pub trait Crypto { .map_err(|_| EcdsaVerifyError::BadV)?; let sig = RecoverableSignature::from_compact(&sig[..64], rid) .map_err(|_| EcdsaVerifyError::BadRS)?; - let msg = Message::from_slice(msg).expect("Message is 32 bytes; qed"); + let msg = Message::from_digest_slice(msg).expect("Message is 32 bytes; qed"); let pubkey = SECP256K1 .recover_ecdsa(&msg, &sig) .map_err(|_| EcdsaVerifyError::BadSignature)?; @@ -1185,7 +1185,7 @@ pub trait Crypto { .map_err(|_| EcdsaVerifyError::BadV)?; let sig = RecoverableSignature::from_compact(&sig[..64], rid) .map_err(|_| EcdsaVerifyError::BadRS)?; - let msg = Message::from_slice(msg).expect("Message is 32 bytes; qed"); + let msg = Message::from_digest_slice(msg).expect("Message is 32 bytes; qed"); let pubkey = SECP256K1 .recover_ecdsa(&msg, &sig) .map_err(|_| EcdsaVerifyError::BadSignature)?; From b5858936e13a8adc800655478c965daf54dbdc71 Mon Sep 17 00:00:00 2001 From: Julian Eager Date: Mon, 20 Nov 2023 02:04:22 +0800 Subject: [PATCH 18/48] Preserve artifact cache unless stale (#1918) Co-authored-by: Marcin S --- Cargo.lock | 8 +- .../node/core/candidate-validation/src/lib.rs | 6 +- .../core/candidate-validation/src/tests.rs | 12 +- polkadot/node/core/pvf/Cargo.toml | 1 + .../benches/host_prepare_rococo_runtime.rs | 2 +- polkadot/node/core/pvf/common/Cargo.toml | 3 + polkadot/node/core/pvf/common/build.rs | 19 + polkadot/node/core/pvf/common/src/error.rs | 18 +- polkadot/node/core/pvf/common/src/lib.rs | 2 + polkadot/node/core/pvf/common/src/prepare.rs | 19 + polkadot/node/core/pvf/common/src/pvf.rs | 2 +- .../node/core/pvf/prepare-worker/Cargo.toml | 1 + .../node/core/pvf/prepare-worker/src/lib.rs | 37 +- polkadot/node/core/pvf/src/artifacts.rs | 383 +++++++++++++----- polkadot/node/core/pvf/src/host.rs | 125 +++--- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/src/prepare/pool.rs | 10 +- polkadot/node/core/pvf/src/prepare/queue.rs | 28 +- .../node/core/pvf/src/prepare/worker_intf.rs | 61 +-- polkadot/node/core/pvf/src/worker_intf.rs | 4 +- polkadot/node/core/pvf/tests/it/main.rs | 5 +- .../utils/build-script-utils/src/version.rs | 31 ++ 22 files changed, 535 insertions(+), 244 deletions(-) create mode 100644 polkadot/node/core/pvf/common/build.rs diff --git a/Cargo.lock b/Cargo.lock index ac64e65ee0ec..b2d655e6f41c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1585,16 +1585,15 @@ dependencies = [ [[package]] name = "blake3" -version = "1.4.1" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "199c42ab6972d92c9f8995f086273d25c42fc0f7b2a1fcefba465c1352d25ba5" +checksum = "0231f06152bf547e9c2b5194f247cd97aacf6dcd8b15d8e5ec0663f64580da87" dependencies = [ "arrayref", "arrayvec 0.7.4", "cc", "cfg-if", "constant_time_eq 0.3.0", - "digest 0.10.7", ] [[package]] @@ -12568,6 +12567,7 @@ version = "1.0.0" dependencies = [ "always-assert", "assert_matches", + "blake3", "cfg-if", "criterion 0.4.0", "futures", @@ -12646,6 +12646,7 @@ dependencies = [ "sp-externalities 0.19.0", "sp-io", "sp-tracing 10.0.0", + "substrate-build-script-utils", "tempfile", "thiserror", "tracing-gum", @@ -12670,6 +12671,7 @@ dependencies = [ name = "polkadot-node-core-pvf-prepare-worker" version = "1.0.0" dependencies = [ + "blake3", "cfg-if", "criterion 0.4.0", "libc", diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index a3d6f0473136..96ab07d3b824 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -25,7 +25,7 @@ use polkadot_node_core_pvf::{ InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PrepareError, - PrepareJobKind, PrepareStats, PvfPrepData, ValidationError, ValidationHost, + PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, }; use polkadot_node_primitives::{ BlockData, InvalidCandidate, PoV, ValidationResult, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT, @@ -794,7 +794,7 @@ trait ValidationBackend { validation_result } - async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result; + async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<(), PrepareError>; } #[async_trait] @@ -824,7 +824,7 @@ impl ValidationBackend for ValidationHost { })? } - async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result { + async fn precheck_pvf(&mut self, pvf: PvfPrepData) -> Result<(), PrepareError> { let (tx, rx) = oneshot::channel(); if let Err(err) = self.precheck_pvf(pvf, tx).await { // Return an IO error if there was an error communicating with the host. diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index cab823e1e637..801c581938a6 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -377,7 +377,7 @@ impl ValidationBackend for MockValidateCandidateBackend { result } - async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result { + async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<(), PrepareError> { unreachable!() } } @@ -1014,11 +1014,11 @@ fn pov_decompression_failure_is_invalid() { } struct MockPreCheckBackend { - result: Result, + result: Result<(), PrepareError>, } impl MockPreCheckBackend { - fn with_hardcoded_result(result: Result) -> Self { + fn with_hardcoded_result(result: Result<(), PrepareError>) -> Self { Self { result } } } @@ -1034,7 +1034,7 @@ impl ValidationBackend for MockPreCheckBackend { unreachable!() } - async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result { + async fn precheck_pvf(&mut self, _pvf: PvfPrepData) -> Result<(), PrepareError> { self.result.clone() } } @@ -1051,7 +1051,7 @@ fn precheck_works() { let (check_fut, check_result) = precheck_pvf( ctx.sender(), - MockPreCheckBackend::with_hardcoded_result(Ok(PrepareStats::default())), + MockPreCheckBackend::with_hardcoded_result(Ok(())), relay_parent, validation_code_hash, ) @@ -1113,7 +1113,7 @@ fn precheck_invalid_pvf_blob_compression() { let (check_fut, check_result) = precheck_pvf( ctx.sender(), - MockPreCheckBackend::with_hardcoded_result(Ok(PrepareStats::default())), + MockPreCheckBackend::with_hardcoded_result(Ok(())), relay_parent, validation_code_hash, ) diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index 3e72ca9e5326..27da484fe4f1 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true [dependencies] always-assert = "0.1" +blake3 = "1.5" cfg-if = "1.0" futures = "0.3.21" futures-timer = "3.0.2" diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs index 378374a10b39..c02a0b595da3 100644 --- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs +++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs @@ -56,7 +56,7 @@ impl TestHost { &self, code: &[u8], executor_params: ExecutorParams, - ) -> Result { + ) -> Result<(), PrepareError> { let (result_tx, result_rx) = futures::channel::oneshot::channel(); let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024) diff --git a/polkadot/node/core/pvf/common/Cargo.toml b/polkadot/node/core/pvf/common/Cargo.toml index e3fda06963e3..bfe1be9156fc 100644 --- a/polkadot/node/core/pvf/common/Cargo.toml +++ b/polkadot/node/core/pvf/common/Cargo.toml @@ -36,6 +36,9 @@ seccompiler = "0.4.0" assert_matches = "1.4.0" tempfile = "3.3.0" +[build-dependencies] +substrate-build-script-utils = { path = "../../../../../substrate/utils/build-script-utils" } + [features] # This feature is used to export test code to other crates without putting it in the production build. test-utils = [] diff --git a/polkadot/node/core/pvf/common/build.rs b/polkadot/node/core/pvf/common/build.rs new file mode 100644 index 000000000000..5531ad411da8 --- /dev/null +++ b/polkadot/node/core/pvf/common/build.rs @@ -0,0 +1,19 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + substrate_build_script_utils::generate_wasmtime_version(); +} diff --git a/polkadot/node/core/pvf/common/src/error.rs b/polkadot/node/core/pvf/common/src/error.rs index 34475c481f73..6bf05ece78ef 100644 --- a/polkadot/node/core/pvf/common/src/error.rs +++ b/polkadot/node/core/pvf/common/src/error.rs @@ -14,16 +14,24 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use crate::prepare::PrepareStats; +use crate::prepare::{PrepareSuccess, PrepareWorkerSuccess}; use parity_scale_codec::{Decode, Encode}; use std::fmt; -/// Result of PVF preparation performed by the validation host. Contains stats about the preparation -/// if successful -pub type PrepareResult = Result; +/// Result of PVF preparation from a worker, with checksum of the compiled PVF and stats of the +/// preparation if successful. +pub type PrepareWorkerResult = Result; + +/// Result of PVF preparation propagated all the way back to the host, with path to the concluded +/// artifact and stats of the preparation if successful. +pub type PrepareResult = Result; + +/// Result of prechecking PVF performed by the validation host. Contains stats about the preparation +/// if successful. +pub type PrecheckResult = Result<(), PrepareError>; /// An error that occurred during the prepare part of the PVF pipeline. -// Codec indexes are intended to stabilize pre-encoded payloads (see `OOM_PAYLOAD` below) +// Codec indexes are intended to stabilize pre-encoded payloads (see `OOM_PAYLOAD`) #[derive(Debug, Clone, Encode, Decode)] pub enum PrepareError { /// During the prevalidation stage of preparation an issue was found with the PVF. diff --git a/polkadot/node/core/pvf/common/src/lib.rs b/polkadot/node/core/pvf/common/src/lib.rs index e2211b97d87b..282d2f7c41d0 100644 --- a/polkadot/node/core/pvf/common/src/lib.rs +++ b/polkadot/node/core/pvf/common/src/lib.rs @@ -31,6 +31,8 @@ pub use sp_tracing; const LOG_TARGET: &str = "parachain::pvf-common"; +pub const RUNTIME_VERSION: &str = env!("SUBSTRATE_WASMTIME_VERSION"); + use std::{ io::{self, Read, Write}, mem, diff --git a/polkadot/node/core/pvf/common/src/prepare.rs b/polkadot/node/core/pvf/common/src/prepare.rs index 4436ebe4861e..28ab682ec136 100644 --- a/polkadot/node/core/pvf/common/src/prepare.rs +++ b/polkadot/node/core/pvf/common/src/prepare.rs @@ -15,6 +15,25 @@ // along with Polkadot. If not, see . use parity_scale_codec::{Decode, Encode}; +use std::path::PathBuf; + +/// Result from prepare worker if successful. +#[derive(Debug, Clone, Default, Encode, Decode)] +pub struct PrepareWorkerSuccess { + /// Checksum of the compiled PVF. + pub checksum: String, + /// Stats of the current preparation run. + pub stats: PrepareStats, +} + +/// Result of PVF preparation if successful. +#[derive(Debug, Clone, Default)] +pub struct PrepareSuccess { + /// Canonical path to the compiled artifact. + pub path: PathBuf, + /// Stats of the current preparation run. + pub stats: PrepareStats, +} /// Preparation statistics, including the CPU time and memory taken. #[derive(Debug, Clone, Default, Encode, Decode)] diff --git a/polkadot/node/core/pvf/common/src/pvf.rs b/polkadot/node/core/pvf/common/src/pvf.rs index 0cc86434c195..2d8f6430187b 100644 --- a/polkadot/node/core/pvf/common/src/pvf.rs +++ b/polkadot/node/core/pvf/common/src/pvf.rs @@ -115,7 +115,7 @@ impl fmt::Debug for PvfPrepData { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!( f, - "Pvf {{ code, code_hash: {:?}, executor_params: {:?}, prep_timeout: {:?} }}", + "Pvf {{ code: [...], code_hash: {:?}, executor_params: {:?}, prep_timeout: {:?} }}", self.code_hash, self.executor_params, self.prep_timeout ) } diff --git a/polkadot/node/core/pvf/prepare-worker/Cargo.toml b/polkadot/node/core/pvf/prepare-worker/Cargo.toml index 1cd221533f48..005f2e935117 100644 --- a/polkadot/node/core/pvf/prepare-worker/Cargo.toml +++ b/polkadot/node/core/pvf/prepare-worker/Cargo.toml @@ -7,6 +7,7 @@ edition.workspace = true license.workspace = true [dependencies] +blake3 = "1.5" cfg-if = "1.0" gum = { package = "tracing-gum", path = "../../../gum" } libc = "0.2.139" diff --git a/polkadot/node/core/pvf/prepare-worker/src/lib.rs b/polkadot/node/core/pvf/prepare-worker/src/lib.rs index 151b54efc2d1..34e6a78c26ae 100644 --- a/polkadot/node/core/pvf/prepare-worker/src/lib.rs +++ b/polkadot/node/core/pvf/prepare-worker/src/lib.rs @@ -40,10 +40,10 @@ use nix::{ use os_pipe::{self, PipeReader, PipeWriter}; use parity_scale_codec::{Decode, Encode}; use polkadot_node_core_pvf_common::{ - error::{PrepareError, PrepareResult}, + error::{PrepareError, PrepareWorkerResult}, executor_intf::create_runtime_from_artifact_bytes, framed_recv_blocking, framed_send_blocking, - prepare::{MemoryStats, PrepareJobKind, PrepareStats}, + prepare::{MemoryStats, PrepareJobKind, PrepareStats, PrepareWorkerSuccess}, pvf::PvfPrepData, worker::{ cpu_time_monitor_loop, run_worker, stringify_panic_payload, @@ -106,7 +106,7 @@ fn recv_request(stream: &mut UnixStream) -> io::Result { } /// Send a worker response. -fn send_response(stream: &mut UnixStream, result: PrepareResult) -> io::Result<()> { +fn send_response(stream: &mut UnixStream, result: PrepareWorkerResult) -> io::Result<()> { framed_send_blocking(stream, &result.encode()) } @@ -186,8 +186,8 @@ fn end_memory_tracking() -> isize { /// /// 7. If compilation succeeded, write the compiled artifact into a temporary file. /// -/// 8. Send the result of preparation back to the host. If any error occurred in the above steps, we -/// send that in the `PrepareResult`. +/// 8. Send the result of preparation back to the host, including the checksum of the artifact. If +/// any error occurred in the above steps, we send that in the `PrepareWorkerResult`. pub fn worker_entrypoint( socket_path: PathBuf, worker_dir_path: PathBuf, @@ -439,11 +439,11 @@ fn handle_child_process( Err(err) => Err(err), Ok(ok) => { cfg_if::cfg_if! { - if #[cfg(target_os = "linux")] { - let (artifact, max_rss) = ok; - } else { - let artifact = ok; - } + if #[cfg(target_os = "linux")] { + let (artifact, max_rss) = ok; + } else { + let artifact = ok; + } } // Stop the memory stats worker and get its observed memory stats. @@ -511,7 +511,7 @@ fn handle_parent_process( worker_pid: u32, usage_before: Usage, timeout: Duration, -) -> Result { +) -> Result { // Read from the child. Don't decode unless the process exited normally, which we check later. let mut received_data = Vec::new(); pipe_read @@ -554,7 +554,7 @@ fn handle_parent_process( match result { Err(err) => Err(err), - Ok(response) => { + Ok(JobResponse { artifact, memory_stats }) => { // The exit status should have been zero if no error occurred. if exit_status != 0 { return Err(PrepareError::JobError(format!( @@ -577,13 +577,14 @@ fn handle_parent_process( temp_artifact_dest.display(), ); // Write to the temp file created by the host. - if let Err(err) = fs::write(&temp_artifact_dest, &response.artifact) { + if let Err(err) = fs::write(&temp_artifact_dest, &artifact) { return Err(PrepareError::IoErr(err.to_string())) }; - Ok(PrepareStats { - memory_stats: response.memory_stats, - cpu_time_elapsed: cpu_tv, + let checksum = blake3::hash(&artifact.as_ref()).to_hex().to_string(); + Ok(PrepareWorkerSuccess { + checksum, + stats: PrepareStats { memory_stats, cpu_time_elapsed: cpu_tv }, }) }, } @@ -657,13 +658,13 @@ fn error_from_errno(context: &'static str, errno: Errno) -> PrepareError { type JobResult = Result; -/// Pre-encoded length-prefixed `Result::Err(PrepareError::OutOfMemory)` +/// Pre-encoded length-prefixed `JobResult::Err(PrepareError::OutOfMemory)` const OOM_PAYLOAD: &[u8] = b"\x02\x00\x00\x00\x00\x00\x00\x00\x01\x08"; #[test] fn pre_encoded_payloads() { // NOTE: This must match the type of `response` in `send_child_response`. - let oom_unencoded: JobResult = Result::Err(PrepareError::OutOfMemory); + let oom_unencoded: JobResult = JobResult::Err(PrepareError::OutOfMemory); let oom_encoded = oom_unencoded.encode(); // The payload is prefixed with its length in `framed_send`. let mut oom_payload = oom_encoded.len().to_le_bytes().to_vec(); diff --git a/polkadot/node/core/pvf/src/artifacts.rs b/polkadot/node/core/pvf/src/artifacts.rs index dd83f76494ed..53085eade3cb 100644 --- a/polkadot/node/core/pvf/src/artifacts.rs +++ b/polkadot/node/core/pvf/src/artifacts.rs @@ -16,10 +16,10 @@ //! PVF artifacts (final compiled code blobs). //! -//! # Lifecycle of an artifact +//! # Lifecycle of an artifact //! -//! 1. During node start-up, the artifacts cache is cleaned up. This means that all local artifacts -//! stored on-disk are cleared, and we start with an empty [`Artifacts`] table. +//! 1. During node start-up, we will check the cached artifacts, if any. The stale and corrupted +//! ones are pruned. The valid ones are registered in the [`Artifacts`] table. //! //! 2. In order to be executed, a PVF should be prepared first. This means that artifacts should //! have an [`ArtifactState::Prepared`] entry for that artifact in the table. If not, the @@ -55,18 +55,29 @@ //! older by a predefined parameter. This process is run very rarely (say, once a day). Once the //! artifact is expired it is removed from disk eagerly atomically. -use crate::host::PrepareResultSender; +use crate::{host::PrecheckResultSender, LOG_TARGET}; use always_assert::always; -use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats, pvf::PvfPrepData}; +use polkadot_core_primitives::Hash; +use polkadot_node_core_pvf_common::{ + error::PrepareError, prepare::PrepareStats, pvf::PvfPrepData, RUNTIME_VERSION, +}; use polkadot_node_primitives::NODE_VERSION; use polkadot_parachain_primitives::primitives::ValidationCodeHash; use polkadot_primitives::ExecutorParamsHash; use std::{ collections::HashMap, path::{Path, PathBuf}, + str::FromStr as _, time::{Duration, SystemTime}, }; +const RUNTIME_PREFIX: &str = "wasmtime_v"; +const NODE_PREFIX: &str = "polkadot_v"; + +fn artifact_prefix() -> String { + format!("{}{}_{}{}", RUNTIME_PREFIX, RUNTIME_VERSION, NODE_PREFIX, NODE_VERSION) +} + /// Identifier of an artifact. Encodes a code hash of the PVF and a hash of executor parameter set. #[derive(Clone, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)] pub struct ArtifactId { @@ -75,9 +86,6 @@ pub struct ArtifactId { } impl ArtifactId { - const PREFIX: &'static str = "wasmtime_"; - const NODE_VERSION_PREFIX: &'static str = "polkadot_v"; - /// Creates a new artifact ID with the given hash. pub fn new(code_hash: ValidationCodeHash, executor_params_hash: ExecutorParamsHash) -> Self { Self { code_hash, executor_params_hash } @@ -88,38 +96,34 @@ impl ArtifactId { Self::new(pvf.code_hash(), pvf.executor_params().hash()) } - /// Tries to recover the artifact id from the given file name. - #[cfg(test)] - pub fn from_file_name(file_name: &str) -> Option { - use polkadot_core_primitives::Hash; - use std::str::FromStr as _; - - let file_name = - file_name.strip_prefix(Self::PREFIX)?.strip_prefix(Self::NODE_VERSION_PREFIX)?; - - // [ node version | code hash | param hash ] - let parts: Vec<&str> = file_name.split('_').collect(); - let (_node_ver, code_hash_str, executor_params_hash_str) = (parts[0], parts[1], parts[2]); - - let code_hash = Hash::from_str(code_hash_str).ok()?.into(); - let executor_params_hash = - ExecutorParamsHash::from_hash(Hash::from_str(executor_params_hash_str).ok()?); - - Some(Self { code_hash, executor_params_hash }) - } - - /// Returns the expected path to this artifact given the root of the cache. - pub fn path(&self, cache_path: &Path) -> PathBuf { + /// Returns the canonical path to the concluded artifact. + pub(crate) fn path(&self, cache_path: &Path, checksum: &str) -> PathBuf { let file_name = format!( - "{}{}{}_{:#x}_{:#x}", - Self::PREFIX, - Self::NODE_VERSION_PREFIX, - NODE_VERSION, + "{}_{:#x}_{:#x}_0x{}", + artifact_prefix(), self.code_hash, - self.executor_params_hash + self.executor_params_hash, + checksum ); cache_path.join(file_name) } + + /// Tries to recover the artifact id from the given file name. + /// Return `None` if the given file name is invalid. + /// VALID_NAME := _ _ _ + fn from_file_name(file_name: &str) -> Option { + let file_name = file_name.strip_prefix(&artifact_prefix())?.strip_prefix('_')?; + let parts: Vec<&str> = file_name.split('_').collect(); + + if let [code_hash, param_hash, _checksum] = parts[..] { + let code_hash = Hash::from_str(code_hash).ok()?.into(); + let executor_params_hash = + ExecutorParamsHash::from_hash(Hash::from_str(param_hash).ok()?); + return Some(Self { code_hash, executor_params_hash }) + } + + None + } } /// A bundle of the artifact ID and the path. @@ -136,8 +140,8 @@ pub struct ArtifactPathId { } impl ArtifactPathId { - pub(crate) fn new(artifact_id: ArtifactId, cache_path: &Path) -> Self { - Self { path: artifact_id.path(cache_path), id: artifact_id } + pub(crate) fn new(artifact_id: ArtifactId, path: &Path) -> Self { + Self { id: artifact_id, path: path.to_owned() } } } @@ -148,6 +152,8 @@ pub enum ArtifactState { /// That means that the artifact should be accessible through the path obtained by the artifact /// id (unless, it was removed externally). Prepared { + /// The path of the compiled artifact. + path: PathBuf, /// The time when the artifact was last needed. /// /// This is updated when we get the heads up for this artifact or when we just discover @@ -159,7 +165,7 @@ pub enum ArtifactState { /// A task to prepare this artifact is scheduled. Preparing { /// List of result senders that are waiting for a response. - waiting_for_response: Vec, + waiting_for_response: Vec, /// The number of times this artifact has failed to prepare. num_failures: u32, }, @@ -177,32 +183,148 @@ pub enum ArtifactState { /// A container of all known artifact ids and their states. pub struct Artifacts { - artifacts: HashMap, + inner: HashMap, } impl Artifacts { - /// Initialize a blank cache at the given path. This will clear everything present at the - /// given path, to be populated over time. - /// - /// The recognized artifacts will be filled in the table and unrecognized will be removed. - pub async fn new(cache_path: &Path) -> Self { - // First delete the entire cache. This includes artifacts and any leftover worker dirs (see - // [`WorkerDir`]). Nodes are long-running so this should populate shortly. - let _ = tokio::fs::remove_dir_all(cache_path).await; + #[cfg(test)] + pub(crate) fn empty() -> Self { + Self { inner: HashMap::new() } + } + + #[cfg(test)] + pub(crate) fn len(&self) -> usize { + self.inner.len() + } + + /// Create an empty table and populate it with valid artifacts as [`ArtifactState::Prepared`], + /// if any. The existing caches will be checked by their file name to determine whether they are + /// valid, e.g., matching the current node version. The ones deemed invalid will be pruned. + pub async fn new_and_prune(cache_path: &Path) -> Self { + let mut artifacts = Self { inner: HashMap::new() }; + artifacts.insert_and_prune(cache_path).await; + artifacts + } + + async fn insert_and_prune(&mut self, cache_path: &Path) { + async fn is_corrupted(path: &Path) -> bool { + let checksum = match tokio::fs::read(path).await { + Ok(bytes) => blake3::hash(&bytes), + Err(err) => { + // just remove the file if we cannot read it + gum::warn!( + target: LOG_TARGET, + ?err, + "unable to read artifact {:?} when checking integrity, removing...", + path, + ); + return true + }, + }; + + if let Some(file_name) = path.file_name() { + if let Some(file_name) = file_name.to_str() { + return !file_name.ends_with(checksum.to_hex().as_str()) + } + } + true + } + + // Insert the entry into the artifacts table if it is valid. + // Otherwise, prune it. + async fn insert_or_prune( + artifacts: &mut Artifacts, + entry: &tokio::fs::DirEntry, + cache_path: &Path, + ) { + let file_type = entry.file_type().await; + let file_name = entry.file_name(); + + match file_type { + Ok(file_type) => + if !file_type.is_file() { + return + }, + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?err, + "unable to get file type for {:?}", + file_name, + ); + return + }, + } + + if let Some(file_name) = file_name.to_str() { + let id = ArtifactId::from_file_name(file_name); + let path = cache_path.join(file_name); + + if id.is_none() || is_corrupted(&path).await { + gum::warn!( + target: LOG_TARGET, + "discarding invalid artifact {:?}", + &path, + ); + let _ = tokio::fs::remove_file(&path).await; + return + } + + if let Some(id) = id { + gum::debug!( + target: LOG_TARGET, + "reusing existing {:?} for node version v{}", + &path, + NODE_VERSION, + ); + artifacts.insert_prepared(id, path, SystemTime::now(), Default::default()); + } + } else { + gum::warn!( + target: LOG_TARGET, + "non-Unicode file name {:?} found in {:?}", + file_name, + cache_path, + ); + } + } + // Make sure that the cache path directory and all its parents are created. let _ = tokio::fs::create_dir_all(cache_path).await; - Self { artifacts: HashMap::new() } - } + let mut dir = match tokio::fs::read_dir(cache_path).await { + Ok(dir) => dir, + Err(err) => { + gum::error!( + target: LOG_TARGET, + ?err, + "failed to read dir {:?}", + cache_path, + ); + return + }, + }; - #[cfg(test)] - pub(crate) fn empty() -> Self { - Self { artifacts: HashMap::new() } + loop { + match dir.next_entry().await { + Ok(Some(entry)) => insert_or_prune(self, &entry, cache_path).await, + Ok(None) => break, + Err(err) => { + gum::warn!( + target: LOG_TARGET, + ?err, + "error processing artifacts in {:?}", + cache_path, + ); + break + }, + } + } } /// Returns the state of the given artifact by its ID. pub fn artifact_state_mut(&mut self, artifact_id: &ArtifactId) -> Option<&mut ArtifactState> { - self.artifacts.get_mut(artifact_id) + self.inner.get_mut(artifact_id) } /// Inform the table about the artifact with the given ID. The state will be set to "preparing". @@ -212,53 +334,52 @@ impl Artifacts { pub fn insert_preparing( &mut self, artifact_id: ArtifactId, - waiting_for_response: Vec, + waiting_for_response: Vec, ) { // See the precondition. always!(self - .artifacts + .inner .insert(artifact_id, ArtifactState::Preparing { waiting_for_response, num_failures: 0 }) .is_none()); } /// Insert an artifact with the given ID as "prepared". /// - /// This function must be used only for brand-new artifacts and should never be used for - /// replacing existing ones. - #[cfg(test)] - pub fn insert_prepared( + /// This function should only be used to build the artifact table at startup with valid + /// artifact caches. + pub(crate) fn insert_prepared( &mut self, artifact_id: ArtifactId, + path: PathBuf, last_time_needed: SystemTime, prepare_stats: PrepareStats, ) { // See the precondition. always!(self - .artifacts - .insert(artifact_id, ArtifactState::Prepared { last_time_needed, prepare_stats }) + .inner + .insert(artifact_id, ArtifactState::Prepared { path, last_time_needed, prepare_stats }) .is_none()); } - /// Remove and retrieve the artifacts from the table that are older than the supplied - /// Time-To-Live. - pub fn prune(&mut self, artifact_ttl: Duration) -> Vec { + /// Remove artifacts older than the given TTL and return id and path of the removed ones. + pub fn prune(&mut self, artifact_ttl: Duration) -> Vec<(ArtifactId, PathBuf)> { let now = SystemTime::now(); let mut to_remove = vec![]; - for (k, v) in self.artifacts.iter() { - if let ArtifactState::Prepared { last_time_needed, .. } = *v { + for (k, v) in self.inner.iter() { + if let ArtifactState::Prepared { last_time_needed, ref path, .. } = *v { if now .duration_since(last_time_needed) .map(|age| age > artifact_ttl) .unwrap_or(false) { - to_remove.push(k.clone()); + to_remove.push((k.clone(), path.clone())); } } } for artifact in &to_remove { - self.artifacts.remove(artifact); + self.inner.remove(&artifact.0); } to_remove @@ -267,13 +388,72 @@ impl Artifacts { #[cfg(test)] mod tests { - use super::{ArtifactId, Artifacts, NODE_VERSION}; + use super::{artifact_prefix as prefix, ArtifactId, Artifacts, NODE_VERSION, RUNTIME_VERSION}; use polkadot_primitives::ExecutorParamsHash; + use rand::Rng; use sp_core::H256; - use std::{path::Path, str::FromStr}; + use std::{ + fs, + io::Write, + path::{Path, PathBuf}, + str::FromStr, + }; + + fn rand_hash(len: usize) -> String { + let mut rng = rand::thread_rng(); + let hex: Vec<_> = "0123456789abcdef".chars().collect(); + (0..len).map(|_| hex[rng.gen_range(0..hex.len())]).collect() + } + + fn file_name(code_hash: &str, param_hash: &str, checksum: &str) -> String { + format!("{}_0x{}_0x{}_0x{}", prefix(), code_hash, param_hash, checksum) + } - fn file_name(code_hash: &str, param_hash: &str) -> String { - format!("wasmtime_polkadot_v{}_0x{}_0x{}", NODE_VERSION, code_hash, param_hash) + fn create_artifact( + dir: impl AsRef, + prefix: &str, + code_hash: impl AsRef, + params_hash: impl AsRef, + ) -> (PathBuf, String) { + fn artifact_path_without_checksum( + dir: impl AsRef, + prefix: &str, + code_hash: impl AsRef, + params_hash: impl AsRef, + ) -> PathBuf { + let mut path = dir.as_ref().to_path_buf(); + let file_name = + format!("{}_0x{}_0x{}", prefix, code_hash.as_ref(), params_hash.as_ref(),); + path.push(file_name); + path + } + + let (code_hash, params_hash) = (code_hash.as_ref(), params_hash.as_ref()); + let path = artifact_path_without_checksum(dir, prefix, code_hash, params_hash); + let mut file = fs::File::create(&path).unwrap(); + + let content = format!("{}{}", code_hash, params_hash).into_bytes(); + file.write_all(&content).unwrap(); + let checksum = blake3::hash(&content).to_hex().to_string(); + + (path, checksum) + } + + fn create_rand_artifact(dir: impl AsRef, prefix: &str) -> (PathBuf, String) { + create_artifact(dir, prefix, rand_hash(64), rand_hash(64)) + } + + fn concluded_path(path: impl AsRef, checksum: &str) -> PathBuf { + let path = path.as_ref(); + let mut file_name = path.file_name().unwrap().to_os_string(); + file_name.push("_0x"); + file_name.push(checksum); + path.with_file_name(file_name) + } + + #[test] + fn artifact_prefix() { + assert_eq!(prefix(), format!("wasmtime_v{}_polkadot_v{}", RUNTIME_VERSION, NODE_VERSION)); } #[test] @@ -284,6 +464,7 @@ mod tests { let file_name = file_name( "0022800000000000000000000000000000000000000000000000000000000000", "0033900000000000000000000000000000000000000000000000000000000000", + "00000000000000000000000000000000", ); assert_eq!( @@ -305,40 +486,54 @@ mod tests { let dir = Path::new("/test"); let code_hash = "1234567890123456789012345678901234567890123456789012345678901234"; let params_hash = "4321098765432109876543210987654321098765432109876543210987654321"; - let file_name = file_name(code_hash, params_hash); + let checksum = "34567890123456789012345678901234"; + let file_name = file_name(code_hash, params_hash, checksum); let code_hash = H256::from_str(code_hash).unwrap(); let params_hash = H256::from_str(params_hash).unwrap(); + let path = ArtifactId::new(code_hash.into(), ExecutorParamsHash::from_hash(params_hash)) + .path(dir, checksum); - assert_eq!( - ArtifactId::new(code_hash.into(), ExecutorParamsHash::from_hash(params_hash)) - .path(dir) - .to_str(), - Some(format!("/test/{}", file_name).as_str()), - ); + assert_eq!(path.to_str().unwrap(), format!("/test/{}", file_name)); } #[tokio::test] - async fn artifacts_removes_cache_on_startup() { - let fake_cache_path = crate::worker_intf::tmppath("test-cache").await.unwrap(); - let fake_artifact_path = { - let mut p = fake_cache_path.clone(); - p.push("wasmtime_0x1234567890123456789012345678901234567890123456789012345678901234"); - p - }; + async fn remove_stale_cache_on_startup() { + let cache_dir = crate::worker_intf::tmppath("test-cache").await.unwrap(); + fs::create_dir_all(&cache_dir).unwrap(); + + // invalid prefix + create_rand_artifact(&cache_dir, ""); + create_rand_artifact(&cache_dir, "wasmtime_polkadot_v"); + create_rand_artifact(&cache_dir, "wasmtime_v8.0.0_polkadot_v1.0.0"); + + let prefix = prefix(); + + // no checksum + create_rand_artifact(&cache_dir, &prefix); + + // invalid hashes + let (path, checksum) = create_artifact(&cache_dir, &prefix, "000", "000001"); + let new_path = concluded_path(&path, &checksum); + fs::rename(&path, &new_path).unwrap(); - // create a tmp cache with 1 artifact. + // checksum tampered + let (path, checksum) = create_rand_artifact(&cache_dir, &prefix); + let new_path = concluded_path(&path, checksum.chars().rev().collect::().as_str()); + fs::rename(&path, &new_path).unwrap(); - std::fs::create_dir_all(&fake_cache_path).unwrap(); - std::fs::File::create(fake_artifact_path).unwrap(); + // valid + let (path, checksum) = create_rand_artifact(&cache_dir, &prefix); + let new_path = concluded_path(&path, &checksum); + fs::rename(&path, &new_path).unwrap(); - // this should remove it and re-create. + assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 7); - let p = &fake_cache_path; - Artifacts::new(p).await; + let artifacts = Artifacts::new_and_prune(&cache_dir).await; - assert_eq!(std::fs::read_dir(&fake_cache_path).unwrap().count(), 0); + assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 1); + assert_eq!(artifacts.len(), 1); - std::fs::remove_dir_all(fake_cache_path).unwrap(); + fs::remove_dir_all(cache_dir).unwrap(); } } diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 5919b9ba32c9..f67934e4171c 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -32,14 +32,15 @@ use futures::{ Future, FutureExt, SinkExt, StreamExt, }; use polkadot_node_core_pvf_common::{ - error::{PrepareError, PrepareResult}, + error::{PrecheckResult, PrepareError}, + prepare::PrepareSuccess, pvf::PvfPrepData, }; use polkadot_node_subsystem::SubsystemResult; use polkadot_parachain_primitives::primitives::ValidationResult; use std::{ collections::HashMap, - path::{Path, PathBuf}, + path::PathBuf, time::{Duration, SystemTime}, }; @@ -63,7 +64,7 @@ pub const EXECUTE_BINARY_NAME: &str = "polkadot-execute-worker"; pub(crate) type ResultSender = oneshot::Sender>; /// Transmission end used for sending the PVF preparation result. -pub(crate) type PrepareResultSender = oneshot::Sender; +pub(crate) type PrecheckResultSender = oneshot::Sender; /// A handle to the async process serving the validation host requests. #[derive(Clone)] @@ -83,7 +84,7 @@ impl ValidationHost { pub async fn precheck_pvf( &mut self, pvf: PvfPrepData, - result_tx: PrepareResultSender, + result_tx: PrecheckResultSender, ) -> Result<(), String> { self.to_host_tx .send(ToHost::PrecheckPvf { pvf, result_tx }) @@ -133,7 +134,7 @@ impl ValidationHost { } enum ToHost { - PrecheckPvf { pvf: PvfPrepData, result_tx: PrepareResultSender }, + PrecheckPvf { pvf: PvfPrepData, result_tx: PrecheckResultSender }, ExecutePvf(ExecutePvfInputs), HeadsUp { active_pvfs: Vec }, } @@ -249,10 +250,9 @@ pub async fn start( let run_sweeper = sweeper_task(to_sweeper_rx); let run_host = async move { - let artifacts = Artifacts::new(&config.cache_path).await; + let artifacts = Artifacts::new_and_prune(&config.cache_path).await; run(Inner { - cache_path: config.cache_path, cleanup_pulse_interval: Duration::from_secs(3600), artifact_ttl: Duration::from_secs(3600 * 24), artifacts, @@ -296,7 +296,6 @@ impl AwaitingPrepare { } struct Inner { - cache_path: PathBuf, cleanup_pulse_interval: Duration, artifact_ttl: Duration, artifacts: Artifacts, @@ -317,7 +316,6 @@ struct Fatal; async fn run( Inner { - cache_path, cleanup_pulse_interval, artifact_ttl, mut artifacts, @@ -361,7 +359,6 @@ async fn run( // will notice it. break_if_fatal!(handle_cleanup_pulse( - &cache_path, &mut to_sweeper_tx, &mut artifacts, artifact_ttl, @@ -380,7 +377,6 @@ async fn run( // If the artifact failed before, it could be re-scheduled for preparation here if // the preparation failure cooldown has elapsed. break_if_fatal!(handle_to_host( - &cache_path, &mut artifacts, &mut to_prepare_queue_tx, &mut to_execute_queue_tx, @@ -402,7 +398,6 @@ async fn run( // We could be eager in terms of reporting and plumb the result from the preparation // worker but we don't for the sake of simplicity. break_if_fatal!(handle_prepare_done( - &cache_path, &mut artifacts, &mut to_execute_queue_tx, &mut awaiting_prepare, @@ -414,7 +409,6 @@ async fn run( } async fn handle_to_host( - cache_path: &Path, artifacts: &mut Artifacts, prepare_queue: &mut mpsc::Sender, execute_queue: &mut mpsc::Sender, @@ -426,15 +420,8 @@ async fn handle_to_host( handle_precheck_pvf(artifacts, prepare_queue, pvf, result_tx).await?; }, ToHost::ExecutePvf(inputs) => { - handle_execute_pvf( - cache_path, - artifacts, - prepare_queue, - execute_queue, - awaiting_prepare, - inputs, - ) - .await?; + handle_execute_pvf(artifacts, prepare_queue, execute_queue, awaiting_prepare, inputs) + .await?; }, ToHost::HeadsUp { active_pvfs } => handle_heads_up(artifacts, prepare_queue, active_pvfs).await?, @@ -454,21 +441,21 @@ async fn handle_precheck_pvf( artifacts: &mut Artifacts, prepare_queue: &mut mpsc::Sender, pvf: PvfPrepData, - result_sender: PrepareResultSender, + result_sender: PrecheckResultSender, ) -> Result<(), Fatal> { let artifact_id = ArtifactId::from_pvf_prep_data(&pvf); if let Some(state) = artifacts.artifact_state_mut(&artifact_id) { match state { - ArtifactState::Prepared { last_time_needed, prepare_stats } => { + ArtifactState::Prepared { last_time_needed, .. } => { *last_time_needed = SystemTime::now(); - let _ = result_sender.send(Ok(prepare_stats.clone())); + let _ = result_sender.send(Ok(())); }, ArtifactState::Preparing { waiting_for_response, num_failures: _ } => waiting_for_response.push(result_sender), ArtifactState::FailedToProcess { error, .. } => { // Do not retry an artifact that previously failed preparation. - let _ = result_sender.send(PrepareResult::Err(error.clone())); + let _ = result_sender.send(PrecheckResult::Err(error.clone())); }, } } else { @@ -491,7 +478,6 @@ async fn handle_precheck_pvf( /// When preparing for execution, we use a more lenient timeout ([`LENIENT_PREPARATION_TIMEOUT`]) /// than when prechecking. async fn handle_execute_pvf( - cache_path: &Path, artifacts: &mut Artifacts, prepare_queue: &mut mpsc::Sender, execute_queue: &mut mpsc::Sender, @@ -504,8 +490,8 @@ async fn handle_execute_pvf( if let Some(state) = artifacts.artifact_state_mut(&artifact_id) { match state { - ArtifactState::Prepared { last_time_needed, .. } => { - let file_metadata = std::fs::metadata(artifact_id.path(cache_path)); + ArtifactState::Prepared { ref path, last_time_needed, .. } => { + let file_metadata = std::fs::metadata(path); if file_metadata.is_ok() { *last_time_needed = SystemTime::now(); @@ -514,7 +500,7 @@ async fn handle_execute_pvf( send_execute( execute_queue, execute::ToQueue::Enqueue { - artifact: ArtifactPathId::new(artifact_id, cache_path), + artifact: ArtifactPathId::new(artifact_id, path), pending_execution_request: PendingExecutionRequest { exec_timeout, params, @@ -677,7 +663,6 @@ async fn handle_heads_up( } async fn handle_prepare_done( - cache_path: &Path, artifacts: &mut Artifacts, execute_queue: &mut mpsc::Sender, awaiting_prepare: &mut AwaitingPrepare, @@ -718,7 +703,8 @@ async fn handle_prepare_done( state { for result_sender in waiting_for_response.drain(..) { - let _ = result_sender.send(result.clone()); + let result = result.clone().map(|_| ()); + let _ = result_sender.send(result); } num_failures } else { @@ -738,16 +724,18 @@ async fn handle_prepare_done( continue } - // Don't send failed artifacts to the execution's queue. - if let Err(ref error) = result { - let _ = result_tx.send(Err(ValidationError::from(error.clone()))); - continue - } + let path = match &result { + Ok(success) => success.path.clone(), + Err(error) => { + let _ = result_tx.send(Err(ValidationError::from(error.clone()))); + continue + }, + }; send_execute( execute_queue, execute::ToQueue::Enqueue { - artifact: ArtifactPathId::new(artifact_id.clone(), cache_path), + artifact: ArtifactPathId::new(artifact_id.clone(), &path), pending_execution_request: PendingExecutionRequest { exec_timeout, params, @@ -760,8 +748,8 @@ async fn handle_prepare_done( } *state = match result { - Ok(prepare_stats) => - ArtifactState::Prepared { last_time_needed: SystemTime::now(), prepare_stats }, + Ok(PrepareSuccess { path, stats: prepare_stats }) => + ArtifactState::Prepared { path, last_time_needed: SystemTime::now(), prepare_stats }, Err(error) => { let last_time_failed = SystemTime::now(); let num_failures = *num_failures + 1; @@ -814,7 +802,6 @@ async fn enqueue_prepare_for_execute( } async fn handle_cleanup_pulse( - cache_path: &Path, sweeper_tx: &mut mpsc::Sender, artifacts: &mut Artifacts, artifact_ttl: Duration, @@ -825,14 +812,13 @@ async fn handle_cleanup_pulse( "PVF pruning: {} artifacts reached their end of life", to_remove.len(), ); - for artifact_id in to_remove { + for (artifact_id, path) in to_remove { gum::debug!( target: LOG_TARGET, validation_code_hash = ?artifact_id.code_hash, "pruning artifact", ); - let artifact_path = artifact_id.path(cache_path); - sweeper_tx.send(artifact_path).await.map_err(|_| Fatal)?; + sweeper_tx.send(path).await.map_err(|_| Fatal)?; } Ok(()) @@ -890,7 +876,11 @@ pub(crate) mod tests { use crate::InvalidCandidate; use assert_matches::assert_matches; use futures::future::BoxFuture; - use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats}; + use polkadot_node_core_pvf_common::{ + error::PrepareError, + prepare::{PrepareStats, PrepareSuccess}, + }; + use sp_core::hexdisplay::AsBytesRef; const TEST_EXECUTION_TIMEOUT: Duration = Duration::from_secs(3); pub(crate) const TEST_PREPARATION_TIMEOUT: Duration = Duration::from_secs(30); @@ -910,12 +900,16 @@ pub(crate) mod tests { } /// Creates a new PVF which artifact id can be uniquely identified by the given number. - fn artifact_id(descriminator: u32) -> ArtifactId { - ArtifactId::from_pvf_prep_data(&PvfPrepData::from_discriminator(descriminator)) + fn artifact_id(discriminator: u32) -> ArtifactId { + ArtifactId::from_pvf_prep_data(&PvfPrepData::from_discriminator(discriminator)) } - fn artifact_path(descriminator: u32) -> PathBuf { - artifact_id(descriminator).path(&PathBuf::from(std::env::temp_dir())).to_owned() + fn artifact_path(discriminator: u32) -> PathBuf { + let pvf = PvfPrepData::from_discriminator(discriminator); + let checksum = blake3::hash(pvf.code().as_bytes_ref()); + artifact_id(discriminator) + .path(&PathBuf::from(std::env::temp_dir()), checksum.to_hex().as_str()) + .to_owned() } struct Builder { @@ -953,8 +947,6 @@ pub(crate) mod tests { impl Test { fn new(Builder { cleanup_pulse_interval, artifact_ttl, artifacts }: Builder) -> Self { - let cache_path = PathBuf::from(std::env::temp_dir()); - let (to_host_tx, to_host_rx) = mpsc::channel(10); let (to_prepare_queue_tx, to_prepare_queue_rx) = mpsc::channel(10); let (from_prepare_queue_tx, from_prepare_queue_rx) = mpsc::unbounded(); @@ -962,7 +954,6 @@ pub(crate) mod tests { let (to_sweeper_tx, to_sweeper_rx) = mpsc::channel(10); let run = run(Inner { - cache_path, cleanup_pulse_interval, artifact_ttl, artifacts, @@ -1111,12 +1102,18 @@ pub(crate) mod tests { let mut builder = Builder::default(); builder.cleanup_pulse_interval = Duration::from_millis(100); builder.artifact_ttl = Duration::from_millis(500); - builder - .artifacts - .insert_prepared(artifact_id(1), mock_now, PrepareStats::default()); - builder - .artifacts - .insert_prepared(artifact_id(2), mock_now, PrepareStats::default()); + builder.artifacts.insert_prepared( + artifact_id(1), + artifact_path(1), + mock_now, + PrepareStats::default(), + ); + builder.artifacts.insert_prepared( + artifact_id(2), + artifact_path(2), + mock_now, + PrepareStats::default(), + ); let mut test = builder.build(); let mut host = test.host_handle(); @@ -1188,7 +1185,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(1), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); @@ -1204,7 +1201,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(2), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); @@ -1258,7 +1255,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(1), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); @@ -1371,7 +1368,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(2), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); @@ -1527,7 +1524,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(1), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); @@ -1703,7 +1700,7 @@ pub(crate) mod tests { test.from_prepare_queue_tx .send(prepare::FromQueue { artifact_id: artifact_id(1), - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }) .await .unwrap(); diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 102a91dbdad7..7e7a13252548 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -84,7 +84,7 @@ //! A pruning task will run at a fixed interval of time. This task will remove all artifacts that //! weren't used or received a heads up signal for a while. //! -//! ## Execution +//! ## Execution //! //! The execute workers will be fed by the requests from the execution queue, which is basically a //! combination of a path to the compiled artifact and the diff --git a/polkadot/node/core/pvf/src/prepare/pool.rs b/polkadot/node/core/pvf/src/prepare/pool.rs index 8e02f540d321..21af21e5b028 100644 --- a/polkadot/node/core/pvf/src/prepare/pool.rs +++ b/polkadot/node/core/pvf/src/prepare/pool.rs @@ -68,7 +68,7 @@ pub enum ToPool { /// /// In either case, the worker is considered busy and no further `StartWork` messages should be /// sent until either `Concluded` or `Rip` message is received. - StartWork { worker: Worker, pvf: PvfPrepData, artifact_path: PathBuf }, + StartWork { worker: Worker, pvf: PvfPrepData, cache_path: PathBuf }, } /// A message sent from pool to its client. @@ -232,7 +232,7 @@ fn handle_to_pool( .boxed(), ); }, - ToPool::StartWork { worker, pvf, artifact_path } => { + ToPool::StartWork { worker, pvf, cache_path } => { if let Some(data) = spawned.get_mut(worker) { if let Some(idle) = data.idle.take() { let preparation_timer = metrics.time_preparation(); @@ -242,7 +242,7 @@ fn handle_to_pool( worker, idle, pvf, - artifact_path, + cache_path, preparation_timer, ) .boxed(), @@ -303,10 +303,10 @@ async fn start_work_task( worker: Worker, idle: IdleWorker, pvf: PvfPrepData, - artifact_path: PathBuf, + cache_path: PathBuf, _preparation_timer: Option, ) -> PoolEvent { - let outcome = worker_intf::start_work(&metrics, idle, pvf, artifact_path).await; + let outcome = worker_intf::start_work(&metrics, idle, pvf, cache_path).await; PoolEvent::StartWork(worker, outcome) } diff --git a/polkadot/node/core/pvf/src/prepare/queue.rs b/polkadot/node/core/pvf/src/prepare/queue.rs index c38012d74548..c140a6cafda0 100644 --- a/polkadot/node/core/pvf/src/prepare/queue.rs +++ b/polkadot/node/core/pvf/src/prepare/queue.rs @@ -268,12 +268,12 @@ fn find_idle_worker(queue: &mut Queue) -> Option { } async fn handle_from_pool(queue: &mut Queue, from_pool: pool::FromPool) -> Result<(), Fatal> { - use pool::FromPool::*; + use pool::FromPool; match from_pool { - Spawned(worker) => handle_worker_spawned(queue, worker).await?, - Concluded { worker, rip, result } => + FromPool::Spawned(worker) => handle_worker_spawned(queue, worker).await?, + FromPool::Concluded { worker, rip, result } => handle_worker_concluded(queue, worker, rip, result).await?, - Rip(worker) => handle_worker_rip(queue, worker).await?, + FromPool::Rip(worker) => handle_worker_rip(queue, worker).await?, } Ok(()) } @@ -424,17 +424,17 @@ async fn spawn_extra_worker(queue: &mut Queue, critical: bool) -> Result<(), Fat /// Attaches the work to the given worker telling the poll about the job. async fn assign(queue: &mut Queue, worker: Worker, job: Job) -> Result<(), Fatal> { let job_data = &mut queue.jobs[job]; - - let artifact_id = ArtifactId::from_pvf_prep_data(&job_data.pvf); - let artifact_path = artifact_id.path(&queue.cache_path); - job_data.worker = Some(worker); queue.workers[worker].job = Some(job); send_pool( &mut queue.to_pool_tx, - pool::ToPool::StartWork { worker, pvf: job_data.pvf.clone(), artifact_path }, + pool::ToPool::StartWork { + worker, + pvf: job_data.pvf.clone(), + cache_path: queue.cache_path.clone(), + }, ) .await?; @@ -491,7 +491,7 @@ mod tests { use crate::host::tests::TEST_PREPARATION_TIMEOUT; use assert_matches::assert_matches; use futures::{future::BoxFuture, FutureExt}; - use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareStats}; + use polkadot_node_core_pvf_common::{error::PrepareError, prepare::PrepareSuccess}; use slotmap::SlotMap; use std::task::Poll; @@ -612,7 +612,7 @@ mod tests { test.send_from_pool(pool::FromPool::Concluded { worker: w, rip: false, - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }); assert_eq!( @@ -651,7 +651,7 @@ mod tests { test.send_from_pool(pool::FromPool::Concluded { worker: w1, rip: false, - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }); assert_matches!(test.poll_and_recv_to_pool().await, pool::ToPool::StartWork { .. }); @@ -697,7 +697,7 @@ mod tests { test.send_from_pool(pool::FromPool::Concluded { worker: w1, rip: false, - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Kill(w1)); } @@ -731,7 +731,7 @@ mod tests { test.send_from_pool(pool::FromPool::Concluded { worker: w1, rip: true, - result: Ok(PrepareStats::default()), + result: Ok(PrepareSuccess::default()), }); // Since there is still work, the queue requested one extra worker to spawn to handle the diff --git a/polkadot/node/core/pvf/src/prepare/worker_intf.rs b/polkadot/node/core/pvf/src/prepare/worker_intf.rs index a22fa74b2fe1..e7f142a46bb8 100644 --- a/polkadot/node/core/pvf/src/prepare/worker_intf.rs +++ b/polkadot/node/core/pvf/src/prepare/worker_intf.rs @@ -17,6 +17,7 @@ //! Host interface to the prepare worker. use crate::{ + artifacts::ArtifactId, metrics::Metrics, security, worker_intf::{ @@ -27,8 +28,8 @@ use crate::{ }; use parity_scale_codec::{Decode, Encode}; use polkadot_node_core_pvf_common::{ - error::{PrepareError, PrepareResult}, - prepare::PrepareStats, + error::{PrepareError, PrepareResult, PrepareWorkerResult}, + prepare::{PrepareStats, PrepareSuccess, PrepareWorkerSuccess}, pvf::PvfPrepData, worker_dir, SecurityStatus, }; @@ -81,7 +82,7 @@ pub enum Outcome { /// final destination location. RenameTmpFile { worker: IdleWorker, - result: PrepareResult, + result: PrepareWorkerResult, err: String, // Unfortunately `PathBuf` doesn't implement `Encode`/`Decode`, so we do a fallible // conversion to `Option`. @@ -115,7 +116,7 @@ pub async fn start_work( metrics: &Metrics, worker: IdleWorker, pvf: PvfPrepData, - artifact_path: PathBuf, + cache_path: PathBuf, ) -> Outcome { let IdleWorker { stream, pid, worker_dir } = worker; @@ -123,8 +124,8 @@ pub async fn start_work( target: LOG_TARGET, worker_pid = %pid, ?worker_dir, - "starting prepare for {}", - artifact_path.display(), + "starting prepare for {:?}", + pvf, ); with_worker_dir_setup( @@ -135,7 +136,7 @@ pub async fn start_work( let preparation_timeout = pvf.prep_timeout(); let audit_log_file = security::AuditLogFile::try_open_and_seek_to_end().await; - if let Err(err) = send_request(&mut stream, pvf.clone()).await { + if let Err(err) = send_request(&mut stream, &pvf).await { gum::warn!( target: LOG_TARGET, worker_pid = %pid, @@ -159,7 +160,7 @@ pub async fn start_work( match result { // Received bytes from worker within the time limit. - Ok(Ok(prepare_result)) => { + Ok(Ok(prepare_worker_result)) => { // Check if any syscall violations occurred during the job. For now this is only // informative, as we are not enforcing the seccomp policy yet. for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { @@ -175,10 +176,11 @@ pub async fn start_work( handle_response( metrics, IdleWorker { stream, pid, worker_dir }, - prepare_result, + prepare_worker_result, pid, tmp_artifact_file, - artifact_path, + &pvf, + &cache_path, preparation_timeout, ) .await @@ -215,20 +217,22 @@ pub async fn start_work( async fn handle_response( metrics: &Metrics, worker: IdleWorker, - result: PrepareResult, + result: PrepareWorkerResult, worker_pid: u32, tmp_file: PathBuf, - artifact_path: PathBuf, + pvf: &PvfPrepData, + cache_path: &PathBuf, preparation_timeout: Duration, ) -> Outcome { - let PrepareStats { cpu_time_elapsed, memory_stats } = match result.clone() { - Ok(result) => result, - // Timed out on the child. This should already be logged by the child. - Err(PrepareError::TimedOut) => return Outcome::TimedOut, - Err(PrepareError::JobDied(err)) => return Outcome::JobDied(err), - Err(PrepareError::OutOfMemory) => return Outcome::OutOfMemory, - Err(_) => return Outcome::Concluded { worker, result }, - }; + let PrepareWorkerSuccess { checksum, stats: PrepareStats { cpu_time_elapsed, memory_stats } } = + match result.clone() { + Ok(result) => result, + // Timed out on the child. This should already be logged by the child. + Err(PrepareError::TimedOut) => return Outcome::TimedOut, + Err(PrepareError::JobDied(err)) => return Outcome::JobDied(err), + Err(PrepareError::OutOfMemory) => return Outcome::OutOfMemory, + Err(err) => return Outcome::Concluded { worker, result: Err(err) }, + }; if cpu_time_elapsed > preparation_timeout { // The job didn't complete within the timeout. @@ -243,6 +247,9 @@ async fn handle_response( return Outcome::TimedOut } + let artifact_id = ArtifactId::from_pvf_prep_data(pvf); + let artifact_path = artifact_id.path(cache_path, &checksum); + gum::debug!( target: LOG_TARGET, %worker_pid, @@ -252,7 +259,13 @@ async fn handle_response( ); let outcome = match tokio::fs::rename(&tmp_file, &artifact_path).await { - Ok(()) => Outcome::Concluded { worker, result }, + Ok(()) => Outcome::Concluded { + worker, + result: Ok(PrepareSuccess { + path: artifact_path, + stats: PrepareStats { cpu_time_elapsed, memory_stats: memory_stats.clone() }, + }), + }, Err(err) => { gum::warn!( target: LOG_TARGET, @@ -329,14 +342,14 @@ where outcome } -async fn send_request(stream: &mut UnixStream, pvf: PvfPrepData) -> io::Result<()> { +async fn send_request(stream: &mut UnixStream, pvf: &PvfPrepData) -> io::Result<()> { framed_send(stream, &pvf.encode()).await?; Ok(()) } -async fn recv_response(stream: &mut UnixStream, pid: u32) -> io::Result { +async fn recv_response(stream: &mut UnixStream, pid: u32) -> io::Result { let result = framed_recv(stream).await?; - let result = PrepareResult::decode(&mut &result[..]).map_err(|e| { + let result = PrepareWorkerResult::decode(&mut &result[..]).map_err(|e| { // We received invalid bytes from the worker. let bound_bytes = &result[..result.len().min(4)]; gum::warn!( diff --git a/polkadot/node/core/pvf/src/worker_intf.rs b/polkadot/node/core/pvf/src/worker_intf.rs index 8f9a7de354b8..5e589b9abcee 100644 --- a/polkadot/node/core/pvf/src/worker_intf.rs +++ b/polkadot/node/core/pvf/src/worker_intf.rs @@ -198,7 +198,7 @@ pub async fn tmppath_in(prefix: &str, dir: &Path) -> io::Result { /// The same as [`tmppath_in`], but uses [`std::env::temp_dir`] as the directory. pub async fn tmppath(prefix: &str) -> io::Result { - let temp_dir = PathBuf::from(std::env::temp_dir()); + let temp_dir = std::env::temp_dir(); tmppath_in(prefix, &temp_dir).await } @@ -453,7 +453,7 @@ impl Drop for WorkerDir { /// artifacts from previous jobs. pub fn clear_worker_dir_path(worker_dir_path: &Path) -> io::Result<()> { fn remove_dir_contents(path: &Path) -> io::Result<()> { - for entry in std::fs::read_dir(&path)? { + for entry in std::fs::read_dir(path)? { let entry = entry?; let path = entry.path(); diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index d2d842cf84a3..5bdf49cc719e 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -20,8 +20,7 @@ use assert_matches::assert_matches; use parity_scale_codec::Encode as _; use polkadot_node_core_pvf::{ start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, PrepareError, - PrepareJobKind, PrepareStats, PvfPrepData, ValidationError, ValidationHost, - JOB_TIMEOUT_WALL_CLOCK_FACTOR, + PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult}; use polkadot_primitives::{ExecutorParam, ExecutorParams}; @@ -70,7 +69,7 @@ impl TestHost { &self, code: &[u8], executor_params: ExecutorParams, - ) -> Result { + ) -> Result<(), PrepareError> { let (result_tx, result_rx) = futures::channel::oneshot::channel(); let code = sp_maybe_compressed_blob::decompress(code, 16 * 1024 * 1024) diff --git a/substrate/utils/build-script-utils/src/version.rs b/substrate/utils/build-script-utils/src/version.rs index f6a9ff9554ab..549e499b1102 100644 --- a/substrate/utils/build-script-utils/src/version.rs +++ b/substrate/utils/build-script-utils/src/version.rs @@ -59,3 +59,34 @@ fn get_version(impl_commit: &str) -> String { impl_commit ) } + +/// Generate `SUBSTRATE_WASMTIME_VERSION` +pub fn generate_wasmtime_version() { + generate_dependency_version("wasmtime", "SUBSTRATE_WASMTIME_VERSION"); +} + +fn generate_dependency_version(dep: &str, env_var: &str) { + // we only care about the root + match std::process::Command::new("cargo") + .args(["tree", "--depth=0", "--locked", "--package", dep]) + .output() + { + Ok(output) if output.status.success() => { + let version = String::from_utf8_lossy(&output.stdout); + + // vX.X.X + if let Some(ver) = version.strip_prefix(&format!("{} v", dep)) { + println!("cargo:rustc-env={}={}", env_var, ver); + } else { + println!("cargo:warning=Unexpected result {}", version); + } + }, + + // command errors out when it could not find the given dependency + // or when having multiple versions of it + Ok(output) => + println!("cargo:warning=`cargo tree` {}", String::from_utf8_lossy(&output.stderr)), + + Err(err) => println!("cargo:warning=Could not run `cargo tree`: {}", err), + } +} From aa5705bff973522a7bd4e39f2ec1a8da56f59f95 Mon Sep 17 00:00:00 2001 From: Ross Bulat Date: Mon, 20 Nov 2023 09:19:34 +0000 Subject: [PATCH 19/48] Pools: Add `MaxUnbonding` to metadata (#2397) --- substrate/frame/nomination-pools/src/lib.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/substrate/frame/nomination-pools/src/lib.rs b/substrate/frame/nomination-pools/src/lib.rs index 909a930e3821..ab4bd51ffc0e 100644 --- a/substrate/frame/nomination-pools/src/lib.rs +++ b/substrate/frame/nomination-pools/src/lib.rs @@ -1626,6 +1626,10 @@ pub mod pallet { #[pallet::constant] type MaxPointsToBalance: Get; + /// The maximum number of simultaneous unbonding chunks that can exist per member. + #[pallet::constant] + type MaxUnbonding: Get; + /// Infallible method for converting `Currency::Balance` to `U256`. type BalanceToU256: Convert, U256>; @@ -1644,9 +1648,6 @@ pub mod pallet { /// The maximum length, in bytes, that a pools metadata maybe. type MaxMetadataLen: Get; - - /// The maximum number of simultaneous unbonding chunks that can exist per member. - type MaxUnbonding: Get; } /// The sum of funds across all pools. From b35300c377d502e75e9e3fb6f15934bc0f31ef53 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:25:58 +0200 Subject: [PATCH 20/48] Bump docker/build-push-action from 5.0.0 to 5.1.0 (#2404) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [docker/build-push-action](https://github.com/docker/build-push-action) from 5.0.0 to 5.1.0.
Release notes

Sourced from docker/build-push-action's releases.

v5.1.0

Full Changelog: https://github.com/docker/build-push-action/compare/v5.0.0...v5.1.0

Commits
  • 4a13e50 Merge pull request #1006 from docker/dependabot/npm_and_yarn/docker/actions-t...
  • 7416668 chore: update generated content
  • b4f76a5 chore(deps): Bump @​docker/actions-toolkit from 0.13.0 to 0.14.0
  • b7feb76 Merge pull request #1005 from crazy-max/ci-inspect
  • fae8018 ci: inspect sbom and provenance
  • b625868 Merge pull request #1004 from crazy-max/ci-update-buildx
  • 5193ef1 ci: update buildx to latest
  • d3afd77 Merge pull request #991 from docker/dependabot/npm_and_yarn/babel/traverse-7....
  • 7a786bb Merge pull request #992 from crazy-max/annotations
  • c66ae3a chore: update generated content
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=docker/build-push-action&package-manager=github_actions&previous-version=5.0.0&new-version=5.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release-50_publish-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-50_publish-docker.yml b/.github/workflows/release-50_publish-docker.yml index 891f43e605c0..567e996b8fd9 100644 --- a/.github/workflows/release-50_publish-docker.yml +++ b/.github/workflows/release-50_publish-docker.yml @@ -268,7 +268,7 @@ jobs: - name: Build and push id: docker_build - uses: docker/build-push-action@0565240e2d4ab88bba5387d719585280857ece09 # v5.0.0 + uses: docker/build-push-action@4a13e500e55cf31b7a5d59a38ab2040ab0f42f56 # v5.1.0 with: push: true file: docker/dockerfiles/polkadot/polkadot_injected_debian.Dockerfile From ede4a362622dfa69eb167eaa876246b1289f4b41 Mon Sep 17 00:00:00 2001 From: jserrat <35823283+Jpserrat@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:00:19 +0000 Subject: [PATCH 21/48] remove retry from backers on failed candidate validation (#2182) Hey guys, as discussed I've changed the name to a more general one `PvfExecKind`, is this good or too general? Creating this as a draft, I still have to fix the tests. Closes #1585 Kusama address: FkB6QEo8VnV3oifugNj5NeVG3Mvq1zFbrUu4P5YwRoe5mQN --------- Co-authored-by: command-bot <> Co-authored-by: Marcin S --- polkadot/node/core/approval-voting/src/lib.rs | 4 +- .../node/core/approval-voting/src/tests.rs | 4 +- polkadot/node/core/backing/src/lib.rs | 4 +- polkadot/node/core/backing/src/tests/mod.rs | 38 ++-- .../src/tests/prospective_parachains.rs | 4 +- .../node/core/candidate-validation/src/lib.rs | 105 ++++++---- .../core/candidate-validation/src/tests.rs | 190 ++++++++++-------- .../src/participation/mod.rs | 4 +- .../src/participation/tests.rs | 16 +- polkadot/node/malus/src/variants/common.rs | 36 ++-- .../node/overseer/examples/minimal-example.rs | 4 +- polkadot/node/overseer/src/tests.rs | 6 +- polkadot/node/subsystem-types/src/messages.rs | 10 +- polkadot/primitives/src/lib.rs | 10 +- polkadot/primitives/src/v6/executor_params.rs | 39 ++-- polkadot/primitives/src/v6/mod.rs | 26 +-- .../src/node/utility/pvf-host-and-workers.md | 4 + .../src/configuration/benchmarking.rs | 10 +- 18 files changed, 271 insertions(+), 243 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 94f7fcaf9411..ef01727b7eb6 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -54,7 +54,7 @@ use polkadot_node_subsystem_util::{ }; use polkadot_primitives::{ ApprovalVote, BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, DisputeStatement, - ExecutorParams, GroupIndex, Hash, PvfExecTimeoutKind, SessionIndex, SessionInfo, + ExecutorParams, GroupIndex, Hash, PvfExecKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; @@ -2867,7 +2867,7 @@ async fn launch_approval( candidate_receipt: candidate.clone(), pov: available_data.pov, executor_params, - exec_timeout_kind: PvfExecTimeoutKind::Approval, + exec_kind: PvfExecKind::Approval, response_sender: val_tx, }) .await; diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 0c0dcfde9b66..11bcba9c3882 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -2705,10 +2705,10 @@ async fn handle_double_assignment_import( assert_matches!( overseer_recv(virtual_overseer).await, AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive { - exec_timeout_kind, + exec_kind, response_sender, .. - }) if exec_timeout_kind == PvfExecTimeoutKind::Approval => { + }) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index a91eefe5e04f..434051f1b00f 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -106,7 +106,7 @@ use polkadot_node_subsystem_util::{ use polkadot_primitives::{ BackedCandidate, CandidateCommitments, CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, CoreState, ExecutorParams, Hash, Id as ParaId, - PersistedValidationData, PvfExecTimeoutKind, SigningContext, ValidationCode, ValidatorId, + PersistedValidationData, PvfExecKind, SigningContext, ValidationCode, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation, }; use sp_keystore::KeystorePtr; @@ -566,7 +566,7 @@ async fn request_candidate_validation( candidate_receipt, pov, executor_params, - exec_timeout_kind: PvfExecTimeoutKind::Backing, + exec_kind: PvfExecKind::Backing, response_sender: tx, }) .await; diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index caa85c12989c..c12be72556e3 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -33,7 +33,7 @@ use polkadot_node_subsystem::{ }; use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{ - CandidateDescriptor, GroupRotationInfo, HeadData, PersistedValidationData, PvfExecTimeoutKind, + CandidateDescriptor, GroupRotationInfo, HeadData, PersistedValidationData, PvfExecKind, ScheduledCore, SessionIndex, LEGACY_MIN_BACKING_VOTES, }; use sp_application_crypto::AppCrypto; @@ -344,14 +344,14 @@ async fn assert_validate_from_exhaustive( validation_data, validation_code, candidate_receipt, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -550,14 +550,14 @@ fn backing_works() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -729,14 +729,14 @@ fn backing_works_while_validation_ongoing() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -890,14 +890,14 @@ fn backing_misbehavior_works() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1057,14 +1057,14 @@ fn backing_dont_second_invalid() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1097,14 +1097,14 @@ fn backing_dont_second_invalid() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1224,14 +1224,14 @@ fn backing_second_after_first_fails_works() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1368,14 +1368,14 @@ fn backing_works_after_failed_validation() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }, ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -1634,13 +1634,13 @@ fn retry_works() { validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, .. }, ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index fc4bd7d98e7d..e7c29e11bb47 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -232,14 +232,14 @@ async fn assert_validate_seconded_candidate( validation_code, candidate_receipt, pov, - exec_timeout_kind, + exec_kind, response_sender, .. }) if &validation_data == assert_pvd && &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_timeout_kind == PvfExecTimeoutKind::Backing && + exec_kind == PvfExecKind::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 96ab07d3b824..f5d17af6c689 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -49,8 +49,8 @@ use polkadot_primitives::{ DEFAULT_LENIENT_PREPARATION_TIMEOUT, DEFAULT_PRECHECK_PREPARATION_TIMEOUT, }, CandidateCommitments, CandidateDescriptor, CandidateReceipt, ExecutorParams, Hash, - OccupiedCoreAssumption, PersistedValidationData, PvfExecTimeoutKind, PvfPrepTimeoutKind, - ValidationCode, ValidationCodeHash, + OccupiedCoreAssumption, PersistedValidationData, PvfExecKind, PvfPrepKind, ValidationCode, + ValidationCodeHash, }; use parity_scale_codec::Encode; @@ -73,12 +73,6 @@ mod tests; const LOG_TARGET: &'static str = "parachain::candidate-validation"; -/// The amount of time to wait before retrying after a retry-able backing validation error. We use a -/// lower value for the backing case, to fit within the lower backing timeout. -#[cfg(not(test))] -const PVF_BACKING_EXECUTION_RETRY_DELAY: Duration = Duration::from_millis(500); -#[cfg(test)] -const PVF_BACKING_EXECUTION_RETRY_DELAY: Duration = Duration::from_millis(200); /// The amount of time to wait before retrying after a retry-able approval validation error. We use /// a higher value for the approval case since we have more time, and if we wait longer it is more /// likely that transient conditions will resolve. @@ -163,7 +157,7 @@ async fn run( candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, .. } => { @@ -180,7 +174,7 @@ async fn run( candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, &metrics, ) .await; @@ -198,7 +192,7 @@ async fn run( candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, .. } => { @@ -215,7 +209,7 @@ async fn run( candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, &metrics, ) .await; @@ -357,7 +351,7 @@ where return PreCheckOutcome::Invalid }; - let timeout = pvf_prep_timeout(&executor_params, PvfPrepTimeoutKind::Precheck); + let timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Precheck); let pvf = match sp_maybe_compressed_blob::decompress( &validation_code.0, @@ -501,7 +495,7 @@ async fn validate_from_chain_state( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_timeout_kind: PvfExecTimeoutKind, + exec_kind: PvfExecKind, metrics: &Metrics, ) -> Result where @@ -521,7 +515,7 @@ where candidate_receipt.clone(), pov, executor_params, - exec_timeout_kind, + exec_kind, metrics, ) .await; @@ -557,7 +551,7 @@ async fn validate_candidate_exhaustive( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_timeout_kind: PvfExecTimeoutKind, + exec_kind: PvfExecKind, metrics: &Metrics, ) -> Result { let _timer = metrics.time_validate_candidate_exhaustive(); @@ -616,15 +610,32 @@ async fn validate_candidate_exhaustive( relay_parent_storage_root: persisted_validation_data.relay_parent_storage_root, }; - let result = validation_backend - .validate_candidate_with_retry( - raw_validation_code.to_vec(), - pvf_exec_timeout(&executor_params, exec_timeout_kind), - exec_timeout_kind, - params, - executor_params, - ) - .await; + let result = match exec_kind { + // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and + // honest backers getting slashed. + PvfExecKind::Backing => { + let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); + let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind); + let pvf = PvfPrepData::from_code( + raw_validation_code.to_vec(), + executor_params, + prep_timeout, + PrepareJobKind::Compilation, + ); + + validation_backend.validate_candidate(pvf, exec_timeout, params.encode()).await + }, + PvfExecKind::Approval => + validation_backend + .validate_candidate_with_retry( + raw_validation_code.to_vec(), + pvf_exec_timeout(&executor_params, exec_kind), + params, + executor_params, + PVF_APPROVAL_EXECUTION_RETRY_DELAY, + ) + .await, + }; if let Err(ref error) = result { gum::info!(target: LOG_TARGET, ?para_id, ?error, "Failed to validate candidate"); @@ -709,8 +720,8 @@ trait ValidationBackend { encoded_params: Vec, ) -> Result; - /// Tries executing a PVF. Will retry once if an error is encountered that may have been - /// transient. + /// Tries executing a PVF for the approval subsystem. Will retry once if an error is encountered + /// that may have been transient. /// /// NOTE: Should retry only on errors that are a result of execution itself, and not of /// preparation. @@ -718,11 +729,11 @@ trait ValidationBackend { &mut self, raw_validation_code: Vec, exec_timeout: Duration, - exec_timeout_kind: PvfExecTimeoutKind, params: ValidationParams, executor_params: ExecutorParams, + retry_delay: Duration, ) -> Result { - let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepTimeoutKind::Lenient); + let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. let pvf = PvfPrepData::from_code( raw_validation_code, @@ -740,11 +751,6 @@ trait ValidationBackend { return validation_result } - let retry_delay = match exec_timeout_kind { - PvfExecTimeoutKind::Backing => PVF_BACKING_EXECUTION_RETRY_DELAY, - PvfExecTimeoutKind::Approval => PVF_APPROVAL_EXECUTION_RETRY_DELAY, - }; - // Allow limited retries for each kind of error. let mut num_death_retries_left = 1; let mut num_job_error_retries_left = 1; @@ -867,22 +873,41 @@ fn perform_basic_checks( Ok(()) } -fn pvf_prep_timeout(executor_params: &ExecutorParams, kind: PvfPrepTimeoutKind) -> Duration { +/// To determine the amount of timeout time for the pvf execution. +/// +/// Precheck +/// The time period after which the preparation worker is considered +/// unresponsive and will be killed. +/// +/// Prepare +///The time period after which the preparation worker is considered +/// unresponsive and will be killed. +fn pvf_prep_timeout(executor_params: &ExecutorParams, kind: PvfPrepKind) -> Duration { if let Some(timeout) = executor_params.pvf_prep_timeout(kind) { return timeout } match kind { - PvfPrepTimeoutKind::Precheck => DEFAULT_PRECHECK_PREPARATION_TIMEOUT, - PvfPrepTimeoutKind::Lenient => DEFAULT_LENIENT_PREPARATION_TIMEOUT, + PvfPrepKind::Precheck => DEFAULT_PRECHECK_PREPARATION_TIMEOUT, + PvfPrepKind::Prepare => DEFAULT_LENIENT_PREPARATION_TIMEOUT, } } -fn pvf_exec_timeout(executor_params: &ExecutorParams, kind: PvfExecTimeoutKind) -> Duration { +/// To determine the amount of timeout time for the pvf execution. +/// +/// Backing subsystem +/// The amount of time to spend on execution during backing. +/// +/// Approval subsystem +/// The amount of time to spend on execution during approval or disputes. +/// This should be much longer than the backing execution timeout to ensure that in the +/// absence of extremely large disparities between hardware, blocks that pass backing are +/// considered executable by approval checkers or dispute participants. +fn pvf_exec_timeout(executor_params: &ExecutorParams, kind: PvfExecKind) -> Duration { if let Some(timeout) = executor_params.pvf_exec_timeout(kind) { return timeout } match kind { - PvfExecTimeoutKind::Backing => DEFAULT_BACKING_EXECUTION_TIMEOUT, - PvfExecTimeoutKind::Approval => DEFAULT_APPROVAL_EXECUTION_TIMEOUT, + PvfExecKind::Backing => DEFAULT_BACKING_EXECUTION_TIMEOUT, + PvfExecKind::Approval => DEFAULT_APPROVAL_EXECUTION_TIMEOUT, } } diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 801c581938a6..5e2585d68735 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -436,7 +436,7 @@ fn candidate_validation_ok_is_ok() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -488,7 +488,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -496,23 +496,20 @@ fn candidate_validation_bad_return_is_invalid() { assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::Timeout)); } -// Test that we vote valid if we get `AmbiguousWorkerDeath`, retry, and then succeed. -#[test] -fn candidate_validation_one_ambiguous_error_is_valid() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let head_data = HeadData(vec![1, 1, 1]); - let validation_code = ValidationCode(vec![2; 16]); - +fn perform_basic_checks_on_valid_candidate( + pov: &PoV, + validation_code: &ValidationCode, + validation_data: &PersistedValidationData, + head_data_hash: Hash, +) -> CandidateDescriptor { let descriptor = make_valid_candidate_descriptor( ParaId::from(1_u32), dummy_hash(), validation_data.hash(), pov.hash(), validation_code.hash(), - head_data.hash(), - dummy_hash(), + head_data_hash, + head_data_hash, Sr25519Keyring::Alice, ); @@ -523,6 +520,24 @@ fn candidate_validation_one_ambiguous_error_is_valid() { &validation_code.hash(), ); assert!(check.is_ok()); + descriptor +} + +// Test that we vote valid if we get `AmbiguousWorkerDeath`, retry, and then succeed. +#[test] +fn candidate_validation_one_ambiguous_error_is_valid() { + let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; + + let pov = PoV { block_data: BlockData(vec![1; 32]) }; + let head_data = HeadData(vec![1, 1, 1]); + let validation_code = ValidationCode(vec![2; 16]); + + let descriptor = perform_basic_checks_on_valid_candidate( + &pov, + &validation_code, + &validation_data, + head_data.hash(), + ); let validation_result = WasmValidationResult { head_data, @@ -554,7 +569,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Approval, &Default::default(), )) .unwrap(); @@ -576,24 +591,12 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { let pov = PoV { block_data: BlockData(vec![1; 32]) }; let validation_code = ValidationCode(vec![2; 16]); - let descriptor = make_valid_candidate_descriptor( - ParaId::from(1_u32), - dummy_hash(), - validation_data.hash(), - pov.hash(), - validation_code.hash(), - dummy_hash(), - dummy_hash(), - Sr25519Keyring::Alice, - ); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, + let descriptor = perform_basic_checks_on_valid_candidate( &pov, - &validation_code.hash(), + &validation_code, + &validation_data, + dummy_hash(), ); - assert!(check.is_ok()); let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; @@ -607,7 +610,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Approval, &Default::default(), )) .unwrap(); @@ -615,58 +618,79 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { assert_matches!(v, ValidationResult::Invalid(InvalidCandidate::ExecutionError(_))); } -// Test that we retry on internal errors. +// Test that we retry for approval on internal errors. #[test] fn candidate_validation_retry_internal_errors() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let validation_code = ValidationCode(vec![2; 16]); - - let descriptor = make_valid_candidate_descriptor( - ParaId::from(1_u32), - dummy_hash(), - validation_data.hash(), - pov.hash(), - validation_code.hash(), - dummy_hash(), - dummy_hash(), - Sr25519Keyring::Alice, - ); - - let check = perform_basic_checks( - &descriptor, - validation_data.max_pov_size, - &pov, - &validation_code.hash(), + let v = candidate_validation_retry_on_error_helper( + PvfExecKind::Approval, + vec![ + Err(InternalValidationError::HostCommunication("foo".into()).into()), + // Throw an AJD error, we should still retry again. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath( + "baz".into(), + ))), + // Throw another internal error. + Err(InternalValidationError::HostCommunication("bar".into()).into()), + ], ); - assert!(check.is_ok()); - - let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; + assert_matches!(v, Err(ValidationFailed(s)) if s.contains("bar")); +} - let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result_list(vec![ +// Test that we don't retry for backing on internal errors. +#[test] +fn candidate_validation_dont_retry_internal_errors() { + let v = candidate_validation_retry_on_error_helper( + PvfExecKind::Backing, + vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), // Throw another internal error. Err(InternalValidationError::HostCommunication("bar".into()).into()), - ]), - validation_data, - validation_code, - candidate_receipt, - Arc::new(pov), - ExecutorParams::default(), - PvfExecTimeoutKind::Backing, - &Default::default(), - )); + ], + ); - assert_matches!(v, Err(ValidationFailed(s)) if s.contains("bar")); + assert_matches!(v, Err(ValidationFailed(s)) if s.contains("foo")); } -// Test that we retry on panic errors. +// Test that we retry for approval on panic errors. #[test] fn candidate_validation_retry_panic_errors() { + let v = candidate_validation_retry_on_error_helper( + PvfExecKind::Approval, + vec![ + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + // Throw an AWD error, we should still retry again. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + // Throw another panic error. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + ], + ); + + assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(s))) if s == "bar".to_string()); +} + +// Test that we don't retry for backing on panic errors. +#[test] +fn candidate_validation_dont_retry_panic_errors() { + let v = candidate_validation_retry_on_error_helper( + PvfExecKind::Backing, + vec![ + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + // Throw an AWD error, we should still retry again. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + // Throw another panic error. + Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + ], + ); + + assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(s))) if s == "foo".to_string()); +} + +fn candidate_validation_retry_on_error_helper( + exec_kind: PvfExecKind, + mock_errors: Vec>, +) -> Result { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; let pov = PoV { block_data: BlockData(vec![1; 32]) }; @@ -693,26 +717,16 @@ fn candidate_validation_retry_panic_errors() { let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; - let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result_list(vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), - // Throw an AJD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath( - "baz".into(), - ))), - // Throw another panic error. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), - ]), + return executor::block_on(validate_candidate_exhaustive( + MockValidateCandidateBackend::with_hardcoded_result_list(mock_errors), validation_data, validation_code, candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + exec_kind, &Default::default(), )); - - assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(s))) if s == "bar".to_string()); } #[test] @@ -752,7 +766,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )); @@ -797,7 +811,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -846,7 +860,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -903,7 +917,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )); @@ -954,7 +968,7 @@ fn code_decompression_failure_is_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )); @@ -1006,7 +1020,7 @@ fn pov_decompression_failure_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecTimeoutKind::Backing, + PvfExecKind::Backing, &Default::default(), )); diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 90268516e9df..05ea7323af14 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -32,7 +32,7 @@ use polkadot_node_subsystem::{ }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, Hash, PvfExecTimeoutKind, SessionIndex, + BlockNumber, CandidateHash, CandidateReceipt, Hash, PvfExecKind, SessionIndex, }; use crate::LOG_TARGET; @@ -386,7 +386,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_timeout_kind: PvfExecTimeoutKind::Approval, + exec_kind: PvfExecKind::Approval, response_sender: validation_tx, }) .await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index 0aa0d7720051..012df51d0cd3 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -115,8 +115,8 @@ pub async fn participation_full_happy_path( assert_matches!( ctx_handle.recv().await, AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_timeout_kind, response_sender, .. } - ) if exec_timeout_kind == PvfExecTimeoutKind::Approval => { + CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. } + ) if exec_kind == PvfExecKind::Approval => { if expected_commitments_hash != candidate_receipt.commitments_hash { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); } else { @@ -450,8 +450,8 @@ fn cast_invalid_vote_if_validation_fails_or_is_invalid() { assert_matches!( ctx_handle.recv().await, AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive { exec_timeout_kind, response_sender, .. } - ) if exec_timeout_kind == PvfExecTimeoutKind::Approval => { + CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } + ) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); }, "overseer did not receive candidate validation message", @@ -487,8 +487,8 @@ fn cast_invalid_vote_if_commitments_dont_match() { assert_matches!( ctx_handle.recv().await, AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive { exec_timeout_kind, response_sender, .. } - ) if exec_timeout_kind == PvfExecTimeoutKind::Approval => { + CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } + ) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); }, "overseer did not receive candidate validation message", @@ -524,8 +524,8 @@ fn cast_valid_vote_if_validation_passes() { assert_matches!( ctx_handle.recv().await, AllMessages::CandidateValidation( - CandidateValidationMessage::ValidateFromExhaustive { exec_timeout_kind, response_sender, .. } - ) if exec_timeout_kind == PvfExecTimeoutKind::Approval => { + CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } + ) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap(); }, "overseer did not receive candidate validation message", diff --git a/polkadot/node/malus/src/variants/common.rs b/polkadot/node/malus/src/variants/common.rs index 474887ee8df7..20b6654638e7 100644 --- a/polkadot/node/malus/src/variants/common.rs +++ b/polkadot/node/malus/src/variants/common.rs @@ -30,7 +30,7 @@ use polkadot_node_subsystem::{ use polkadot_primitives::{ CandidateCommitments, CandidateDescriptor, CandidateReceipt, PersistedValidationData, - PvfExecTimeoutKind, + PvfExecKind, }; use futures::channel::oneshot; @@ -90,10 +90,10 @@ impl FakeCandidateValidation { } } - fn should_misbehave(&self, timeout: PvfExecTimeoutKind) -> bool { + fn should_misbehave(&self, timeout: PvfExecKind) -> bool { match timeout { - PvfExecTimeoutKind::Backing => self.includes_backing(), - PvfExecTimeoutKind::Approval => self.includes_approval(), + PvfExecKind::Backing => self.includes_backing(), + PvfExecKind::Approval => self.includes_approval(), } } } @@ -279,13 +279,13 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, .. }, } => { match self.fake_validation { - x if x.misbehaves_valid() && x.should_misbehave(exec_timeout_kind) => { + x if x.misbehaves_valid() && x.should_misbehave(exec_kind) => { // Behave normally if the `PoV` is not known to be malicious. if pov.block_data.0.as_slice() != MALICIOUS_POV { return Some(FromOrchestra::Communication { @@ -295,7 +295,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }) @@ -333,14 +333,14 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }) }, } }, - x if x.misbehaves_invalid() && x.should_misbehave(exec_timeout_kind) => { + x if x.misbehaves_invalid() && x.should_misbehave(exec_kind) => { // Set the validation result to invalid with probability `p` and trigger a // dispute let behave_maliciously = self.distribution.sample(&mut rand::thread_rng()); @@ -373,7 +373,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }) @@ -388,7 +388,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }), @@ -401,13 +401,13 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, .. }, } => { match self.fake_validation { - x if x.misbehaves_valid() && x.should_misbehave(exec_timeout_kind) => { + x if x.misbehaves_valid() && x.should_misbehave(exec_kind) => { // Behave normally if the `PoV` is not known to be malicious. if pov.block_data.0.as_slice() != MALICIOUS_POV { return Some(FromOrchestra::Communication { @@ -415,7 +415,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }) @@ -445,13 +445,13 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }), } }, - x if x.misbehaves_invalid() && x.should_misbehave(exec_timeout_kind) => { + x if x.misbehaves_invalid() && x.should_misbehave(exec_kind) => { // Maliciously set the validation result to invalid for a valid candidate // with probability `p` let behave_maliciously = self.distribution.sample(&mut rand::thread_rng()); @@ -479,7 +479,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }) @@ -491,7 +491,7 @@ where candidate_receipt, pov, executor_params, - exec_timeout_kind, + exec_kind, response_sender, }, }), diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index b2c0ea2f75a8..857cdba673db 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -32,7 +32,7 @@ use polkadot_overseer::{ gen::{FromOrchestra, SpawnedSubsystem}, HeadSupportsParachains, SubsystemError, }; -use polkadot_primitives::{CandidateReceipt, Hash, PvfExecTimeoutKind}; +use polkadot_primitives::{CandidateReceipt, Hash, PvfExecKind}; struct AlwaysSupportsParachains; @@ -77,7 +77,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_timeout_kind: PvfExecTimeoutKind::Backing, + exec_kind: PvfExecKind::Backing, response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 254f5fe45120..0494274367d9 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -30,7 +30,7 @@ use polkadot_node_subsystem_types::messages::{ }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, - PvfExecTimeoutKind, SessionIndex, ValidDisputeStatementKind, ValidatorIndex, + PvfExecKind, SessionIndex, ValidDisputeStatementKind, ValidatorIndex, }; use crate::{ @@ -106,7 +106,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_timeout_kind: PvfExecTimeoutKind::Backing, + exec_kind: PvfExecKind::Backing, response_sender: tx, }) .await; @@ -804,7 +804,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_timeout_kind: PvfExecTimeoutKind::Backing, + exec_kind: PvfExecKind::Backing, response_sender, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 43456daec302..44c6f27b17cc 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -47,7 +47,7 @@ use polkadot_primitives::{ CommittedCandidateReceipt, CoreState, DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, OccupiedCoreAssumption, PersistedValidationData, - PvfCheckStatement, PvfExecTimeoutKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, + PvfCheckStatement, PvfExecKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; @@ -150,8 +150,8 @@ pub enum CandidateValidationMessage { pov: Arc, /// Session's executor parameters executor_params: ExecutorParams, - /// Execution timeout kind (backing/approvals) - exec_timeout_kind: PvfExecTimeoutKind, + /// Execution kind, used for timeouts and retries (backing/approvals) + exec_kind: PvfExecKind, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -175,8 +175,8 @@ pub enum CandidateValidationMessage { pov: Arc, /// Session's executor parameters executor_params: ExecutorParams, - /// Execution timeout kind (backing/approvals) - exec_timeout_kind: PvfExecTimeoutKind, + /// Execution kind, used for timeouts and retries (backing/approvals) + exec_kind: PvfExecKind, /// The sending side of the response channel response_sender: oneshot::Sender>, }, diff --git a/polkadot/primitives/src/lib.rs b/polkadot/primitives/src/lib.rs index 4ba8b8b031fc..2570bcadf606 100644 --- a/polkadot/primitives/src/lib.rs +++ b/polkadot/primitives/src/lib.rs @@ -48,11 +48,11 @@ pub use v6::{ HorizontalMessages, HrmpChannelId, Id, InboundDownwardMessage, InboundHrmpMessage, IndexedVec, InherentData, InvalidDisputeStatementKind, Moment, MultiDisputeStatementSet, Nonce, OccupiedCore, OccupiedCoreAssumption, OutboundHrmpMessage, ParathreadClaim, ParathreadEntry, - PersistedValidationData, PvfCheckStatement, PvfExecTimeoutKind, PvfPrepTimeoutKind, - RuntimeMetricLabel, RuntimeMetricLabelValue, RuntimeMetricLabelValues, RuntimeMetricLabels, - RuntimeMetricOp, RuntimeMetricUpdate, ScheduledCore, ScrapedOnChainVotes, SessionIndex, - SessionInfo, Signature, Signed, SignedAvailabilityBitfield, SignedAvailabilityBitfields, - SignedStatement, SigningContext, Slot, UncheckedSigned, UncheckedSignedAvailabilityBitfield, + PersistedValidationData, PvfCheckStatement, PvfExecKind, PvfPrepKind, RuntimeMetricLabel, + RuntimeMetricLabelValue, RuntimeMetricLabelValues, RuntimeMetricLabels, RuntimeMetricOp, + RuntimeMetricUpdate, ScheduledCore, ScrapedOnChainVotes, SessionIndex, SessionInfo, Signature, + Signed, SignedAvailabilityBitfield, SignedAvailabilityBitfields, SignedStatement, + SigningContext, Slot, UncheckedSigned, UncheckedSignedAvailabilityBitfield, UncheckedSignedAvailabilityBitfields, UncheckedSignedStatement, UpgradeGoAhead, UpgradeRestriction, UpwardMessage, ValidDisputeStatementKind, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, ValidityAttestation, diff --git a/polkadot/primitives/src/v6/executor_params.rs b/polkadot/primitives/src/v6/executor_params.rs index bb9980f68796..112a529f62b0 100644 --- a/polkadot/primitives/src/v6/executor_params.rs +++ b/polkadot/primitives/src/v6/executor_params.rs @@ -21,7 +21,7 @@ //! by the first element of the vector). Decoding to a usable semantics structure is //! done in `polkadot-node-core-pvf`. -use crate::{BlakeTwo256, HashT as _, PvfExecTimeoutKind, PvfPrepTimeoutKind}; +use crate::{BlakeTwo256, HashT as _, PvfExecKind, PvfPrepKind}; use parity_scale_codec::{Decode, Encode}; use polkadot_core_primitives::Hash; use scale_info::TypeInfo; @@ -45,7 +45,7 @@ pub const PRECHECK_MEM_MAX_LO: u64 = 256 * 1024 * 1024; pub const PRECHECK_MEM_MAX_HI: u64 = 16 * 1024 * 1024 * 1024; // Default PVF timeouts. Must never be changed! Use executor environment parameters to adjust them. -// See also `PvfPrepTimeoutKind` and `PvfExecTimeoutKind` docs. +// See also `PvfPrepKind` and `PvfExecKind` docs. /// Default PVF preparation timeout for prechecking requests. pub const DEFAULT_PRECHECK_PREPARATION_TIMEOUT: Duration = Duration::from_secs(60); @@ -99,12 +99,12 @@ pub enum ExecutorParam { /// Always ensure that `precheck_timeout` < `lenient_timeout`. /// When absent, the default values will be used. #[codec(index = 5)] - PvfPrepTimeout(PvfPrepTimeoutKind, u64), + PvfPrepTimeout(PvfPrepKind, u64), /// PVF execution timeouts, in millisecond. /// Always ensure that `backing_timeout` < `approval_timeout`. /// When absent, the default values will be used. #[codec(index = 6)] - PvfExecTimeout(PvfExecTimeoutKind, u64), + PvfExecTimeout(PvfExecKind, u64), /// Enables WASM bulk memory proposal #[codec(index = 7)] WasmExtBulkMemory, @@ -174,7 +174,7 @@ impl ExecutorParams { } /// Returns a PVF preparation timeout, if any - pub fn pvf_prep_timeout(&self, kind: PvfPrepTimeoutKind) -> Option { + pub fn pvf_prep_timeout(&self, kind: PvfPrepKind) -> Option { for param in &self.0 { if let ExecutorParam::PvfPrepTimeout(k, timeout) = param { if kind == *k { @@ -186,7 +186,7 @@ impl ExecutorParams { } /// Returns a PVF execution timeout, if any - pub fn pvf_exec_timeout(&self, kind: PvfExecTimeoutKind) -> Option { + pub fn pvf_exec_timeout(&self, kind: PvfExecKind) -> Option { for param in &self.0 { if let ExecutorParam::PvfExecTimeout(k, timeout) = param { if kind == *k { @@ -242,12 +242,12 @@ impl ExecutorParams { StackNativeMax(_) => "StackNativeMax", PrecheckingMaxMemory(_) => "PrecheckingMaxMemory", PvfPrepTimeout(kind, _) => match kind { - PvfPrepTimeoutKind::Precheck => "PvfPrepTimeoutKind::Precheck", - PvfPrepTimeoutKind::Lenient => "PvfPrepTimeoutKind::Lenient", + PvfPrepKind::Precheck => "PvfPrepKind::Precheck", + PvfPrepKind::Prepare => "PvfPrepKind::Prepare", }, PvfExecTimeout(kind, _) => match kind { - PvfExecTimeoutKind::Backing => "PvfExecTimeoutKind::Backing", - PvfExecTimeoutKind::Approval => "PvfExecTimeoutKind::Approval", + PvfExecKind::Backing => "PvfExecKind::Backing", + PvfExecKind::Approval => "PvfExecKind::Approval", }, WasmExtBulkMemory => "WasmExtBulkMemory", }; @@ -297,30 +297,23 @@ impl ExecutorParams { } if let (Some(precheck), Some(lenient)) = ( - seen.get("PvfPrepTimeoutKind::Precheck") + seen.get("PvfPrepKind::Precheck") .or(Some(&DEFAULT_PRECHECK_PREPARATION_TIMEOUT_MS)), - seen.get("PvfPrepTimeoutKind::Lenient") + seen.get("PvfPrepKind::Prepare") .or(Some(&DEFAULT_LENIENT_PREPARATION_TIMEOUT_MS)), ) { if *precheck >= *lenient { - return Err(IncompatibleValues( - "PvfPrepTimeoutKind::Precheck", - "PvfPrepTimeoutKind::Lenient", - )) + return Err(IncompatibleValues("PvfPrepKind::Precheck", "PvfPrepKind::Prepare")) } } if let (Some(backing), Some(approval)) = ( - seen.get("PvfExecTimeoutKind::Backing") - .or(Some(&DEFAULT_BACKING_EXECUTION_TIMEOUT_MS)), - seen.get("PvfExecTimeoutKind::Approval") + seen.get("PvfExecKind::Backing").or(Some(&DEFAULT_BACKING_EXECUTION_TIMEOUT_MS)), + seen.get("PvfExecKind::Approval") .or(Some(&DEFAULT_APPROVAL_EXECUTION_TIMEOUT_MS)), ) { if *backing >= *approval { - return Err(IncompatibleValues( - "PvfExecTimeoutKind::Backing", - "PvfExecTimeoutKind::Approval", - )) + return Err(IncompatibleValues("PvfExecKind::Backing", "PvfExecKind::Approval")) } } diff --git a/polkadot/primitives/src/v6/mod.rs b/polkadot/primitives/src/v6/mod.rs index 9371b3db406b..83b590dc3203 100644 --- a/polkadot/primitives/src/v6/mod.rs +++ b/polkadot/primitives/src/v6/mod.rs @@ -1781,30 +1781,22 @@ impl WellKnownKey { } } -/// Type discriminator for PVF preparation timeouts +/// Type discriminator for PVF preparation. #[derive(Encode, Decode, TypeInfo, Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum PvfPrepTimeoutKind { - /// For prechecking requests, the time period after which the preparation worker is considered - /// unresponsive and will be killed. +pub enum PvfPrepKind { + /// For prechecking requests. Precheck, - /// For execution and heads-up requests, the time period after which the preparation worker is - /// considered unresponsive and will be killed. More lenient than the timeout for prechecking - /// to prevent honest validators from timing out on valid PVFs. - Lenient, + /// For execution and heads-up requests. + Prepare, } -/// Type discriminator for PVF execution timeouts +/// Type discriminator for PVF execution. #[derive(Encode, Decode, TypeInfo, Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize)] -pub enum PvfExecTimeoutKind { - /// The amount of time to spend on execution during backing. +pub enum PvfExecKind { + /// For backing requests. Backing, - - /// The amount of time to spend on execution during approval or disputes. - /// - /// This should be much longer than the backing execution timeout to ensure that in the - /// absence of extremely large disparities between hardware, blocks that pass backing are - /// considered executable by approval checkers or dispute participants. + /// For approval and dispute request. Approval, } diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md index 4dbb7980c1be..74d88ba3ad99 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md @@ -31,6 +31,10 @@ hopefully resolve. We use a more brief delay here (1 second as opposed to 15 minutes for preparation (see above)), because a successful execution must happen in a short amount of time. +If the execution fails during the backing phase, we won't retry to reduce the chance of +supporting nondeterministic candidates. This reduces the chance of nondeterministic blocks +getting backed and honest backers getting slashed. + We currently know of the following specific cases that will lead to a retried execution request: diff --git a/polkadot/runtime/parachains/src/configuration/benchmarking.rs b/polkadot/runtime/parachains/src/configuration/benchmarking.rs index 508e0579a09d..67daf1c45988 100644 --- a/polkadot/runtime/parachains/src/configuration/benchmarking.rs +++ b/polkadot/runtime/parachains/src/configuration/benchmarking.rs @@ -17,7 +17,7 @@ use crate::configuration::*; use frame_benchmarking::{benchmarks, BenchmarkError, BenchmarkResult}; use frame_system::RawOrigin; -use primitives::{ExecutorParam, ExecutorParams, PvfExecTimeoutKind, PvfPrepTimeoutKind}; +use primitives::{ExecutorParam, ExecutorParams, PvfExecKind, PvfPrepKind}; use sp_runtime::traits::One; benchmarks! { @@ -41,10 +41,10 @@ benchmarks! { ExecutorParam::StackNativeMax(256 * 1024 * 1024), ExecutorParam::WasmExtBulkMemory, ExecutorParam::PrecheckingMaxMemory(2 * 1024 * 1024 * 1024), - ExecutorParam::PvfPrepTimeout(PvfPrepTimeoutKind::Precheck, 60_000), - ExecutorParam::PvfPrepTimeout(PvfPrepTimeoutKind::Lenient, 360_000), - ExecutorParam::PvfExecTimeout(PvfExecTimeoutKind::Backing, 2_000), - ExecutorParam::PvfExecTimeout(PvfExecTimeoutKind::Approval, 12_000), + ExecutorParam::PvfPrepTimeout(PvfPrepKind::Precheck, 60_000), + ExecutorParam::PvfPrepTimeout(PvfPrepKind::Prepare, 360_000), + ExecutorParam::PvfExecTimeout(PvfExecKind::Backing, 2_000), + ExecutorParam::PvfExecTimeout(PvfExecKind::Approval, 12_000), ][..])) set_config_with_perbill {}: set_on_demand_fee_variability(RawOrigin::Root, Perbill::from_percent(100)) From 126f64a91ee0c0333f368fe67bb00d56d68ecb34 Mon Sep 17 00:00:00 2001 From: Javier Viola Date: Tue, 21 Nov 2023 07:38:04 -0300 Subject: [PATCH 22/48] bump zombienet version `v1.3.82` (#2396) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Includes: - bump `pjs` modules versions - re-enable test disable in https://github.com/paritytech/polkadot-sdk/pull/2294 --------- Co-authored-by: Bastian Köcher --- .gitlab-ci.yml | 4 ---- .gitlab/pipeline/zombienet.yml | 5 +++++ .gitlab/pipeline/zombienet/substrate.yml | 15 +++++++-------- .../skip-feeless-payment/src/lib.rs | 15 +++++++++------ 4 files changed, 21 insertions(+), 18 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index b485ca3317ed..e698aba81d55 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,6 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final - ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.80" DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}" default: @@ -199,9 +198,6 @@ default: - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs - if: $CI_COMMIT_REF_NAME =~ /^gh-readonly-queue.*$/ # merge queues -.zombienet-refs: - extends: .build-refs - include: # check jobs - .gitlab/pipeline/check.yml diff --git a/.gitlab/pipeline/zombienet.yml b/.gitlab/pipeline/zombienet.yml index 64210d6a00ab..fc3fd1256194 100644 --- a/.gitlab/pipeline/zombienet.yml +++ b/.gitlab/pipeline/zombienet.yml @@ -1,3 +1,8 @@ +.zombienet-refs: + extends: .build-refs + variables: + ZOMBIENET_IMAGE: "docker.io/paritytech/zombienet:v1.3.82" + include: # substrate tests - .gitlab/pipeline/zombienet/substrate.yml diff --git a/.gitlab/pipeline/zombienet/substrate.yml b/.gitlab/pipeline/zombienet/substrate.yml index 6334e7db9a32..b687576267de 100644 --- a/.gitlab/pipeline/zombienet/substrate.yml +++ b/.gitlab/pipeline/zombienet/substrate.yml @@ -40,14 +40,13 @@ tags: - zombienet-polkadot-integration-test -# Skip this one until PolkadotJS includes `SkipCheckIfFeeless` extension -# zombienet-substrate-0000-block-building: -# extends: -# - .zombienet-substrate-common -# script: -# - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh -# --local-dir="${LOCAL_DIR}/0000-block-building" -# --test="block-building.zndsl" +zombienet-substrate-0000-block-building: + extends: + - .zombienet-substrate-common + script: + - /home/nonroot/zombie-net/scripts/ci/run-test-local-env-manager.sh + --local-dir="${LOCAL_DIR}/0000-block-building" + --test="block-building.zndsl" zombienet-substrate-0001-basic-warp-sync: extends: diff --git a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs index 923c7e7ebc29..55afaaf93d78 100644 --- a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs +++ b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs @@ -77,9 +77,9 @@ pub mod pallet { /// A [`SignedExtension`] that skips the wrapped extension if the dispatchable is feeless. #[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] #[scale_info(skip_type_params(T))] -pub struct SkipCheckIfFeeless(pub S, sp_std::marker::PhantomData); +pub struct SkipCheckIfFeeless(pub S, sp_std::marker::PhantomData); -impl sp_std::fmt::Debug for SkipCheckIfFeeless { +impl sp_std::fmt::Debug for SkipCheckIfFeeless { #[cfg(feature = "std")] fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { write!(f, "SkipCheckIfFeeless<{:?}>", self.0.encode()) @@ -90,9 +90,8 @@ impl sp_std::fmt::Debug for SkipCheckIfFeeless SkipCheckIfFeeless { - /// utility constructor. Used only in client/factory code. - pub fn from(s: S) -> Self { +impl From for SkipCheckIfFeeless { + fn from(s: S) -> Self { Self(s, sp_std::marker::PhantomData) } } @@ -106,7 +105,11 @@ where type Call = S::Call; type AdditionalSigned = S::AdditionalSigned; type Pre = (Self::AccountId, Option<::Pre>); - const IDENTIFIER: &'static str = "SkipCheckIfFeeless"; + // From the outside this extension should be "invisible", because it just extends the wrapped + // extension with an extra check in `pre_dispatch` and `post_dispatch`. Thus, we should forward + // the identifier of the wrapped extension to let wallets see this extension as it would only be + // the wrapped extension itself. + const IDENTIFIER: &'static str = S::IDENTIFIER; fn additional_signed(&self) -> Result { self.0.additional_signed() From 2fd8c51ebcfcf7d10cab2a2827211583418a90a0 Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:51:23 +0100 Subject: [PATCH 23/48] `chain-spec-builder`: cleanup (#2174) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR removes: - `New`, `Generate`, `Edit` commands, - `kitchensink` dependency from the `chain-spec-builder` util. New `convert-to-raw`, `update-code` commands were added. Additionally renames the `runtime` command (which was added in #1256) to `create`. --------- Co-authored-by: Sebastian Kunert Co-authored-by: Bastian Köcher Co-authored-by: command-bot <> --- Cargo.lock | 7 - .../bin/utils/chain-spec-builder/Cargo.toml | 7 - .../bin/utils/chain-spec-builder/bin/main.rs | 127 ++------ .../bin/utils/chain-spec-builder/src/lib.rs | 295 +++++------------- 4 files changed, 116 insertions(+), 320 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b2d655e6f41c..697f232f9719 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -18315,18 +18315,11 @@ checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" name = "staging-chain-spec-builder" version = "2.0.0" dependencies = [ - "ansi_term", "clap 4.4.6", - "kitchensink-runtime", "log", - "rand 0.8.5", "sc-chain-spec", - "sc-keystore", "serde_json", - "sp-core", - "sp-keystore", "sp-tracing 10.0.0", - "staging-node-cli", ] [[package]] diff --git a/substrate/bin/utils/chain-spec-builder/Cargo.toml b/substrate/bin/utils/chain-spec-builder/Cargo.toml index 0b373b8e9247..f587989e0039 100644 --- a/substrate/bin/utils/chain-spec-builder/Cargo.toml +++ b/substrate/bin/utils/chain-spec-builder/Cargo.toml @@ -20,15 +20,8 @@ name = "chain-spec-builder" crate-type = ["rlib"] [dependencies] -ansi_term = "0.12.1" clap = { version = "4.4.6", features = ["derive"] } -rand = "0.8" -kitchensink-runtime = { version = "3.0.0-dev", path = "../../node/runtime" } log = "0.4.17" -node-cli = { package = "staging-node-cli", path = "../../node/cli" } sc-chain-spec = { path = "../../../client/chain-spec" } -sc-keystore = { path = "../../../client/keystore" } serde_json = "1.0.108" -sp-core = { path = "../../../primitives/core" } -sp-keystore = { path = "../../../primitives/keystore" } sp-tracing = { version = "10.0.0", path = "../../../primitives/tracing" } diff --git a/substrate/bin/utils/chain-spec-builder/bin/main.rs b/substrate/bin/utils/chain-spec-builder/bin/main.rs index 83892afd6ace..986293179a91 100644 --- a/substrate/bin/utils/chain-spec-builder/bin/main.rs +++ b/substrate/bin/utils/chain-spec-builder/bin/main.rs @@ -17,14 +17,11 @@ // along with this program. If not, see . use chain_spec_builder::{ - generate_authority_keys_and_store, generate_chain_spec, generate_chain_spec_for_runtime, - print_seeds, ChainSpecBuilder, ChainSpecBuilderCmd, EditCmd, GenerateCmd, NewCmd, VerifyCmd, + generate_chain_spec_for_runtime, ChainSpecBuilder, ChainSpecBuilderCmd, ConvertToRawCmd, + UpdateCodeCmd, VerifyCmd, }; use clap::Parser; -use node_cli::chain_spec; -use rand::{distributions::Alphanumeric, rngs::OsRng, Rng}; use sc_chain_spec::{update_code_in_json_chain_spec, GenericChainSpec}; -use sp_core::{crypto::Ss58Codec, sr25519}; use staging_chain_spec_builder as chain_spec_builder; use std::fs; @@ -32,110 +29,48 @@ fn main() -> Result<(), String> { sp_tracing::try_init_simple(); let builder = ChainSpecBuilder::parse(); - #[cfg(build_type = "debug")] - if matches!(builder.command, ChainSpecBuilderCmd::Generate(_) | ChainSpecBuilderCmd::New(_)) { - println!( - "The chain spec builder builds a chain specification that includes a Substrate runtime \ - compiled as WASM. To ensure proper functioning of the included runtime compile (or run) \ - the chain spec builder binary in `--release` mode.\n", - ); - } - let chain_spec_path = builder.chain_spec_path.to_path_buf(); - let mut write_chain_spec = true; - - let chain_spec_json = match builder.command { - ChainSpecBuilderCmd::Generate(GenerateCmd { - authorities, - nominators, - endowed, - keystore_path, - }) => { - let authorities = authorities.max(1); - let rand_str = || -> String { - OsRng.sample_iter(&Alphanumeric).take(32).map(char::from).collect() - }; - - let authority_seeds = (0..authorities).map(|_| rand_str()).collect::>(); - let nominator_seeds = (0..nominators).map(|_| rand_str()).collect::>(); - let endowed_seeds = (0..endowed).map(|_| rand_str()).collect::>(); - let sudo_seed = rand_str(); - - print_seeds(&authority_seeds, &nominator_seeds, &endowed_seeds, &sudo_seed); - - if let Some(keystore_path) = keystore_path { - generate_authority_keys_and_store(&authority_seeds, &keystore_path)?; - } - - let nominator_accounts = nominator_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - - let endowed_accounts = endowed_seeds - .into_iter() - .map(|seed| { - chain_spec::get_account_id_from_seed::(&seed).to_ss58check() - }) - .collect(); - let sudo_account = - chain_spec::get_account_id_from_seed::(&sudo_seed).to_ss58check(); - - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account) + match builder.command { + ChainSpecBuilderCmd::Create(cmd) => { + let chain_spec_json = generate_chain_spec_for_runtime(&cmd)?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; }, - ChainSpecBuilderCmd::New(NewCmd { - authority_seeds, - nominator_accounts, - endowed_accounts, - sudo_account, - }) => - generate_chain_spec(authority_seeds, nominator_accounts, endowed_accounts, sudo_account), - ChainSpecBuilderCmd::Runtime(cmd) => generate_chain_spec_for_runtime(&cmd), - ChainSpecBuilderCmd::Edit(EditCmd { + ChainSpecBuilderCmd::UpdateCode(UpdateCodeCmd { ref input_chain_spec, ref runtime_wasm_path, - convert_to_raw, }) => { let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; let mut chain_spec_json = - serde_json::from_str::(&chain_spec.as_json(convert_to_raw)?) + serde_json::from_str::(&chain_spec.as_json(false)?) .map_err(|e| format!("Conversion to json failed: {e}"))?; - if let Some(path) = runtime_wasm_path { - update_code_in_json_chain_spec( - &mut chain_spec_json, - &fs::read(path.as_path()) - .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], - ); - } - - serde_json::to_string_pretty(&chain_spec_json) - .map_err(|e| format!("to pretty failed: {e}")) + update_code_in_json_chain_spec( + &mut chain_spec_json, + &fs::read(runtime_wasm_path.as_path()) + .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], + ); + + let chain_spec_json = serde_json::to_string_pretty(&chain_spec_json) + .map_err(|e| format!("to pretty failed: {e}"))?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; }, - ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec, ref runtime_wasm_path }) => { - write_chain_spec = false; + ChainSpecBuilderCmd::ConvertToRaw(ConvertToRawCmd { ref input_chain_spec }) => { let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; - let mut chain_spec_json = + + let chain_spec_json = serde_json::from_str::(&chain_spec.as_json(true)?) .map_err(|e| format!("Conversion to json failed: {e}"))?; - if let Some(path) = runtime_wasm_path { - update_code_in_json_chain_spec( - &mut chain_spec_json, - &fs::read(path.as_path()) - .map_err(|e| format!("Wasm blob file could not be read: {e}"))?[..], - ); - }; - serde_json::to_string_pretty(&chain_spec_json) - .map_err(|e| format!("to pretty failed: {e}")) - }, - }?; - if write_chain_spec { - fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string()) - } else { - Ok(()) - } + let chain_spec_json = serde_json::to_string_pretty(&chain_spec_json) + .map_err(|e| format!("Conversion to pretty failed: {e}"))?; + fs::write(chain_spec_path, chain_spec_json).map_err(|err| err.to_string())?; + }, + ChainSpecBuilderCmd::Verify(VerifyCmd { ref input_chain_spec }) => { + let chain_spec = GenericChainSpec::<()>::from_json_file(input_chain_spec.clone())?; + let _ = serde_json::from_str::(&chain_spec.as_json(true)?) + .map_err(|e| format!("Conversion to json failed: {e}"))?; + }, + }; + Ok(()) } diff --git a/substrate/bin/utils/chain-spec-builder/src/lib.rs b/substrate/bin/utils/chain-spec-builder/src/lib.rs index 6f21b68c3684..60ec0f1a656b 100644 --- a/substrate/bin/utils/chain-spec-builder/src/lib.rs +++ b/substrate/bin/utils/chain-spec-builder/src/lib.rs @@ -21,31 +21,69 @@ //! A chain-spec is short for `chain-configuration`. See the [`sc-chain-spec`] for more information. //! //! Note that this binary is analogous to the `build-spec` subcommand, contained in typical -//! substrate-based nodes. This particular binary is capable of building a more sophisticated chain -//! specification that can be used with the substrate-node, ie. [`node-cli`]. +//! substrate-based nodes. This particular binary is capable of interacting with +//! [`sp-genesis-builder`] implementation of any provided runtime allowing to build chain-spec JSON +//! files. //! -//! See [`ChainSpecBuilder`] for a list of available commands. +//! See [`ChainSpecBuilderCmd`] for a list of available commands. +//! +//! ## Typical use-cases. +//! ##### Get default config from runtime. +//! +//! Query the default genesis config from the provided `runtime.wasm` and use it in the chain +//! spec. Tool can also store runtime's default genesis config in given file: +//! ```text +//! chain-spec-builder create -r runtime.wasm default /dev/stdout +//! ``` +//! +//! _Note:_ [`GenesisBuilder::create_default_config`][sp-genesis-builder-create] runtime function is called. +//! +//! +//! ##### Generate raw storage chain spec using genesis config patch. +//! +//! Patch the runtime's default genesis config with provided `patch.json` and generate raw +//! storage (`-s`) version of chain spec: +//! ```text +//! chain-spec-builder create -s -r runtime.wasm patch patch.json +//! ``` +//! +//! _Note:_ [`GenesisBuilder::build_config`][sp-genesis-builder-build] runtime function is called. +//! +//! ##### Generate raw storage chain spec using full genesis config. +//! +//! Build the chain spec using provided full genesis config json file. No defaults will be used: +//! ```text +//! chain-spec-builder create -s -r runtime.wasm full full-genesis-config.json +//! ``` +//! +//! _Note_: [`GenesisBuilder::build_config`][sp-genesis-builder-build] runtime function is called. +//! +//! ##### Generate human readable chain spec using provided genesis config patch. +//! ```text +//! chain-spec-builder create -r runtime.wasm patch patch.json +//! ``` +//! +//! ##### Generate human readable chain spec using provided full genesis config. +//! ```text +//! chain-spec-builder create -r runtime.wasm full full-genesis-config.json +//! ``` +//! +//! ##### Extra tools. +//! The `chain-spec-builder` provides also some extra utilities: [`VerifyCmd`], [`ConvertToRawCmd`], [`UpdateCodeCmd`]. //! //! [`sc-chain-spec`]: ../sc_chain_spec/index.html //! [`node-cli`]: ../node_cli/index.html +//! [`sp-genesis-builder`]: ../sp_genesis_builder/index.html +//! [sp-genesis-builder-create]: ../sp_genesis_builder/trait.GenesisBuilder.html#method.create_default_config +//! [sp-genesis-builder-build]: ../sp_genesis_builder/trait.GenesisBuilder.html#method.build_config -use std::{ - fs, - path::{Path, PathBuf}, -}; +use std::{fs, path::PathBuf}; -use ansi_term::Style; use clap::{Parser, Subcommand}; -use sc_chain_spec::GenesisConfigBuilderRuntimeCaller; - -use node_cli::chain_spec::{self, AccountId}; -use sc_keystore::LocalKeystore; +use sc_chain_spec::{GenericChainSpec, GenesisConfigBuilderRuntimeCaller}; use serde_json::Value; -use sp_core::crypto::{ByteArray, Ss58Codec}; -use sp_keystore::KeystorePtr; -/// A utility to easily create a testnet chain spec definition with a given set -/// of authorities and endowed accounts and/or generate random accounts. +/// A utility to easily create a chain spec definition. #[derive(Debug, Parser)] #[command(rename_all = "kebab-case")] pub struct ChainSpecBuilder { @@ -59,70 +97,25 @@ pub struct ChainSpecBuilder { #[derive(Debug, Subcommand)] #[command(rename_all = "kebab-case")] pub enum ChainSpecBuilderCmd { - New(NewCmd), - Generate(GenerateCmd), - Runtime(RuntimeCmd), - Edit(EditCmd), + Create(CreateCmd), Verify(VerifyCmd), -} - -/// Create a new chain spec with the given authorities, endowed and sudo -/// accounts. Only works for kitchen-sink runtime -#[derive(Parser, Debug)] -#[command(rename_all = "kebab-case")] -pub struct NewCmd { - /// Authority key seed. - #[arg(long, short, required = true)] - pub authority_seeds: Vec, - /// Active nominators (SS58 format), each backing a random subset of the aforementioned - /// authorities. - #[arg(long, short, default_value = "0")] - pub nominator_accounts: Vec, - /// Endowed account address (SS58 format). - #[arg(long, short)] - pub endowed_accounts: Vec, - /// Sudo account address (SS58 format). - #[arg(long, short)] - pub sudo_account: String, -} - -/// Create a new chain spec with the given number of authorities and endowed -/// accounts. Random keys will be generated as required. -#[derive(Parser, Debug)] -pub struct GenerateCmd { - /// The number of authorities. - #[arg(long, short)] - pub authorities: usize, - /// The number of nominators backing the aforementioned authorities. - /// - /// Will nominate a random subset of `authorities`. - #[arg(long, short, default_value_t = 0)] - pub nominators: usize, - /// The number of endowed accounts. - #[arg(long, short, default_value_t = 0)] - pub endowed: usize, - /// Path to use when saving generated keystores for each authority. - /// - /// At this path, a new folder will be created for each authority's - /// keystore named `auth-$i` where `i` is the authority index, i.e. - /// `auth-0`, `auth-1`, etc. - #[arg(long, short)] - pub keystore_path: Option, + UpdateCode(UpdateCodeCmd), + ConvertToRaw(ConvertToRawCmd), } /// Create a new chain spec by interacting with the provided runtime wasm blob. #[derive(Parser, Debug)] -pub struct RuntimeCmd { - /// The name of chain +pub struct CreateCmd { + /// The name of chain. #[arg(long, short = 'n', default_value = "Custom")] chain_name: String, - /// The chain id + /// The chain id. #[arg(long, short = 'i', default_value = "custom")] chain_id: String, - /// The path to runtime wasm blob + /// The path to runtime wasm blob. #[arg(long, short)] runtime_wasm_path: PathBuf, - /// Export chainspec as raw storage + /// Export chainspec as raw storage. #[arg(long, short = 's')] raw_storage: bool, /// Verify the genesis config. This silently generates the raw storage from genesis config. Any @@ -144,7 +137,6 @@ enum GenesisBuildAction { #[derive(Parser, Debug, Clone)] struct PatchCmd { /// The path to the runtime genesis config patch. - #[arg(long, short)] patch_path: PathBuf, } @@ -152,7 +144,6 @@ struct PatchCmd { #[derive(Parser, Debug, Clone)] struct FullCmd { /// The path to the full runtime genesis config json file. - #[arg(long, short)] config_path: PathBuf, } @@ -163,161 +154,45 @@ struct FullCmd { struct DefaultCmd { /// If provided stores the default genesis config json file at given path (in addition to /// chain-spec). - #[arg(long, short)] default_config_path: Option, } -/// Edits provided input chain spec. Input can be converted into raw storage chain-spec. The code -/// can be updated with the runtime provided in the command line. +/// Updates the code in the provided input chain spec. +/// +/// The code field of the chain spec will be updated with the runtime provided in the +/// command line. This operation supports both plain and raw formats. #[derive(Parser, Debug, Clone)] -pub struct EditCmd { - /// Chain spec to be edited - #[arg(long, short)] +pub struct UpdateCodeCmd { + /// Chain spec to be updated. pub input_chain_spec: PathBuf, - /// The path to new runtime wasm blob to be stored into chain-spec - #[arg(long, short = 'r')] - pub runtime_wasm_path: Option, - /// Convert genesis spec to raw format - #[arg(long, short = 's')] - pub convert_to_raw: bool, + /// The path to new runtime wasm blob to be stored into chain-spec. + pub runtime_wasm_path: PathBuf, } -/// Verifies provided input chain spec. If the runtime is provided verification is performed against -/// new runtime. +/// Converts the given chain spec into the raw format. #[derive(Parser, Debug, Clone)] -pub struct VerifyCmd { - /// Chain spec to be edited - #[arg(long, short)] +pub struct ConvertToRawCmd { + /// Chain spec to be converted. pub input_chain_spec: PathBuf, - /// The path to new runtime wasm blob to be stored into chain-spec - #[arg(long, short = 'r')] - pub runtime_wasm_path: Option, -} - -/// Generate the chain spec using the given seeds and accounts. -pub fn generate_chain_spec( - authority_seeds: Vec, - nominator_accounts: Vec, - endowed_accounts: Vec, - sudo_account: String, -) -> Result { - let parse_account = |address: String| { - AccountId::from_string(&address) - .map_err(|err| format!("Failed to parse account address: {:?}", err)) - }; - - let nominator_accounts = nominator_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let endowed_accounts = endowed_accounts - .into_iter() - .map(parse_account) - .collect::, String>>()?; - - let sudo_account = parse_account(sudo_account)?; - - let authorities = authority_seeds - .iter() - .map(AsRef::as_ref) - .map(chain_spec::authority_keys_from_seed) - .collect::>(); - - chain_spec::ChainSpec::builder(kitchensink_runtime::wasm_binary_unwrap(), Default::default()) - .with_name("Custom") - .with_id("custom") - .with_chain_type(sc_chain_spec::ChainType::Live) - .with_genesis_config_patch(chain_spec::testnet_genesis( - authorities, - nominator_accounts, - sudo_account, - Some(endowed_accounts), - )) - .build() - .as_json(false) -} - -/// Generate the authority keys and store them in the given `keystore_path`. -pub fn generate_authority_keys_and_store( - seeds: &[String], - keystore_path: &Path, -) -> Result<(), String> { - for (n, seed) in seeds.iter().enumerate() { - let keystore: KeystorePtr = - LocalKeystore::open(keystore_path.join(format!("auth-{}", n)), None) - .map_err(|err| err.to_string())? - .into(); - - let (_, _, grandpa, babe, im_online, authority_discovery, mixnet) = - chain_spec::authority_keys_from_seed(seed); - - let insert_key = |key_type, public| { - keystore - .insert(key_type, &format!("//{}", seed), public) - .map_err(|_| format!("Failed to insert key: {}", grandpa)) - }; - - insert_key(sp_core::crypto::key_types::BABE, babe.as_slice())?; - - insert_key(sp_core::crypto::key_types::GRANDPA, grandpa.as_slice())?; - - insert_key(sp_core::crypto::key_types::IM_ONLINE, im_online.as_slice())?; - - insert_key( - sp_core::crypto::key_types::AUTHORITY_DISCOVERY, - authority_discovery.as_slice(), - )?; - - insert_key(sp_core::crypto::key_types::MIXNET, mixnet.as_slice())?; - } - - Ok(()) } -/// Print the given seeds -pub fn print_seeds( - authority_seeds: &[String], - nominator_seeds: &[String], - endowed_seeds: &[String], - sudo_seed: &str, -) { - let header = Style::new().bold().underline(); - let entry = Style::new().bold(); - - println!("{}", header.paint("Authority seeds")); - - for (n, seed) in authority_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("auth-{}:", n)), seed); - } - - println!("{}", header.paint("Nominator seeds")); - - for (n, seed) in nominator_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("nom-{}:", n)), seed); - } - - println!(); - - if !endowed_seeds.is_empty() { - println!("{}", header.paint("Endowed seeds")); - for (n, seed) in endowed_seeds.iter().enumerate() { - println!("{} //{}", entry.paint(format!("endowed-{}:", n)), seed); - } - - println!(); - } - - println!("{}", header.paint("Sudo seed")); - println!("//{}", sudo_seed); +/// Verifies the provided input chain spec. +/// +/// Silently checks if given input chain spec can be converted to raw. It allows to check if all +/// RuntimeGenesisConfig fiels are properly initialized and if the json does not contain invalid +/// fields. +#[derive(Parser, Debug, Clone)] +pub struct VerifyCmd { + /// Chain spec to be verified. + pub input_chain_spec: PathBuf, } -/// Processes `RuntimeCmd` and returns JSON version of `ChainSpec` -pub fn generate_chain_spec_for_runtime(cmd: &RuntimeCmd) -> Result { +/// Processes `CreateCmd` and returns JSON version of `ChainSpec`. +pub fn generate_chain_spec_for_runtime(cmd: &CreateCmd) -> Result { let code = fs::read(cmd.runtime_wasm_path.as_path()) .map_err(|e| format!("wasm blob shall be readable {e}"))?; - let builder = chain_spec::ChainSpec::builder(&code[..], Default::default()) + let builder = GenericChainSpec::<()>::builder(&code[..], Default::default()) .with_name(&cmd.chain_name[..]) .with_id(&cmd.chain_id[..]) .with_chain_type(sc_chain_spec::ChainType::Live); From 06190498a00ff913975fa8853c3c6a94783c4124 Mon Sep 17 00:00:00 2001 From: Julian Eager Date: Tue, 21 Nov 2023 19:10:59 +0800 Subject: [PATCH 24/48] Refactor `ValidationError` (#2406) --- .../node/core/candidate-validation/src/lib.rs | 60 +++++++++++-------- .../core/candidate-validation/src/tests.rs | 40 ++++++------- .../benches/host_prepare_rococo_runtime.rs | 3 +- .../node/core/pvf/common/src/executor_intf.rs | 19 +++--- polkadot/node/core/pvf/src/error.rs | 37 ++++++++---- polkadot/node/core/pvf/src/execute/queue.rs | 14 ++--- polkadot/node/core/pvf/src/host.rs | 39 +++++------- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/tests/it/main.rs | 8 +-- polkadot/node/core/pvf/tests/it/process.rs | 10 ++-- 10 files changed, 122 insertions(+), 110 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index f5d17af6c689..9f7b17f61299 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -24,8 +24,8 @@ #![warn(missing_docs)] use polkadot_node_core_pvf::{ - InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PrepareError, - PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, + InternalValidationError, InvalidCandidate as WasmInvalidCandidate, PossiblyInvalidError, + PrepareError, PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, }; use polkadot_node_primitives::{ BlockData, InvalidCandidate, PoV, ValidationResult, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT, @@ -642,7 +642,7 @@ async fn validate_candidate_exhaustive( } match result { - Err(ValidationError::InternalError(e)) => { + Err(ValidationError::Internal(e)) => { gum::warn!( target: LOG_TARGET, ?para_id, @@ -651,34 +651,29 @@ async fn validate_candidate_exhaustive( ); Err(ValidationFailed(e.to_string())) }, - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout)) => + Err(ValidationError::Invalid(WasmInvalidCandidate::HardTimeout)) => Ok(ValidationResult::Invalid(InvalidCandidate::Timeout)), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::WorkerReportedInvalid(e))) => + Err(ValidationError::Invalid(WasmInvalidCandidate::WorkerReportedInvalid(e))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(e))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError( "ambiguous worker death".to_string(), ))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError(err))) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(err))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath(err))) => + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) => Ok(ValidationResult::Invalid(InvalidCandidate::ExecutionError(format!( "ambiguous job death: {err}" )))), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::PrepareError(e))) => { - // In principle if preparation of the `WASM` fails, the current candidate can not be the - // reason for that. So we can't say whether it is invalid or not. In addition, with - // pre-checking enabled only valid runtimes should ever get enacted, so we can be - // reasonably sure that this is some local problem on the current node. However, as this - // particular error *seems* to indicate a deterministic error, we raise a warning. + Err(ValidationError::Preparation(e)) => { gum::warn!( target: LOG_TARGET, ?para_id, ?e, "Deterministic error occurred during preparation (should have been ruled out by pre-checking phase)", ); - Err(ValidationFailed(e)) + Err(ValidationFailed(e.to_string())) }, Ok(res) => if res.head_data.hash() != candidate_receipt.descriptor.para_head { @@ -762,16 +757,31 @@ trait ValidationBackend { } match validation_result { - Err(ValidationError::InvalidCandidate( - WasmInvalidCandidate::AmbiguousWorkerDeath | - WasmInvalidCandidate::AmbiguousJobDeath(_), - )) if num_death_retries_left > 0 => num_death_retries_left -= 1, - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError(_))) - if num_job_error_retries_left > 0 => - num_job_error_retries_left -= 1, - Err(ValidationError::InternalError(_)) if num_internal_retries_left > 0 => - num_internal_retries_left -= 1, - _ => break, + Err(ValidationError::PossiblyInvalid( + PossiblyInvalidError::AmbiguousWorkerDeath | + PossiblyInvalidError::AmbiguousJobDeath(_), + )) => + if num_death_retries_left > 0 { + num_death_retries_left -= 1; + } else { + break; + }, + + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(_))) => + if num_job_error_retries_left > 0 { + num_job_error_retries_left -= 1; + } else { + break; + }, + + Err(ValidationError::Internal(_)) => + if num_internal_retries_left > 0 { + num_internal_retries_left -= 1; + } else { + break; + }, + + Ok(_) | Err(ValidationError::Invalid(_) | ValidationError::Preparation(_)) => break, } // If we got a possibly transient error, retry once after a brief delay, on the diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 5e2585d68735..110785804652 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -480,9 +480,9 @@ fn candidate_validation_bad_return_is_invalid() { let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, @@ -561,7 +561,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { let v = executor::block_on(validate_candidate_exhaustive( MockValidateCandidateBackend::with_hardcoded_result_list(vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), Ok(validation_result), ]), validation_data.clone(), @@ -602,8 +602,8 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { let v = executor::block_on(validate_candidate_exhaustive( MockValidateCandidateBackend::with_hardcoded_result_list(vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), ]), validation_data, validation_code, @@ -626,7 +626,7 @@ fn candidate_validation_retry_internal_errors() { vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousJobDeath( + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath( "baz".into(), ))), // Throw another internal error. @@ -644,7 +644,7 @@ fn candidate_validation_dont_retry_internal_errors() { vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another internal error. Err(InternalValidationError::HostCommunication("bar".into()).into()), ], @@ -659,11 +659,11 @@ fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( PvfExecKind::Approval, vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another panic error. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("bar".into()))), ], ); @@ -676,11 +676,11 @@ fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( PvfExecKind::Backing, vec![ - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("foo".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), // Throw another panic error. - Err(ValidationError::InvalidCandidate(WasmInvalidCandidate::JobError("bar".into()))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("bar".into()))), ], ); @@ -758,9 +758,9 @@ fn candidate_validation_timeout_is_internal_error() { let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, @@ -852,9 +852,9 @@ fn candidate_validation_code_mismatch_is_invalid() { let (_ctx, _ctx_handle) = test_helpers::make_subsystem_context::(pool.clone()); let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Err( - ValidationError::InvalidCandidate(WasmInvalidCandidate::HardTimeout), - )), + MockValidateCandidateBackend::with_hardcoded_result(Err(ValidationError::Invalid( + WasmInvalidCandidate::HardTimeout, + ))), validation_data, validation_code, candidate_receipt, diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs index c02a0b595da3..f2551b701c2c 100644 --- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs +++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs @@ -18,8 +18,7 @@ use criterion::{criterion_group, criterion_main, BatchSize, Criterion, SamplingMode}; use polkadot_node_core_pvf::{ - start, testing, Config, Metrics, PrepareError, PrepareJobKind, PrepareStats, PvfPrepData, - ValidationHost, + start, testing, Config, Metrics, PrepareError, PrepareJobKind, PvfPrepData, ValidationHost, }; use polkadot_primitives::ExecutorParams; use rococo_runtime::WASM_BINARY; diff --git a/polkadot/node/core/pvf/common/src/executor_intf.rs b/polkadot/node/core/pvf/common/src/executor_intf.rs index 3a1d3ac1ba07..4bafc3f4291a 100644 --- a/polkadot/node/core/pvf/common/src/executor_intf.rs +++ b/polkadot/node/core/pvf/common/src/executor_intf.rs @@ -140,8 +140,7 @@ pub unsafe fn create_runtime_from_artifact_bytes( executor_params: &ExecutorParams, ) -> Result { let mut config = DEFAULT_CONFIG.clone(); - config.semantics = - params_to_wasmtime_semantics(executor_params).map_err(|err| WasmError::Other(err))?; + config.semantics = params_to_wasmtime_semantics(executor_params); sc_executor_wasmtime::create_runtime_from_artifact_bytes::( compiled_artifact_blob, @@ -149,13 +148,12 @@ pub unsafe fn create_runtime_from_artifact_bytes( ) } -pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Result { +pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Semantics { let mut sem = DEFAULT_CONFIG.semantics.clone(); - let mut stack_limit = if let Some(stack_limit) = sem.deterministic_stack_limit.clone() { - stack_limit - } else { - return Err("No default stack limit set".to_owned()) - }; + let mut stack_limit = sem + .deterministic_stack_limit + .expect("There is a comment to not change the default stack limit; it should always be available; qed") + .clone(); for p in par.iter() { match p { @@ -172,7 +170,7 @@ pub fn params_to_wasmtime_semantics(par: &ExecutorParams) -> Result Result, sc_executor_common::error::WasmError> { - let semantics = params_to_wasmtime_semantics(executor_params) - .map_err(|e| sc_executor_common::error::WasmError::Other(e))?; + let semantics = params_to_wasmtime_semantics(executor_params); sc_executor_wasmtime::prepare_runtime_artifact(blob, &semantics) } diff --git a/polkadot/node/core/pvf/src/error.rs b/polkadot/node/core/pvf/src/error.rs index 7fdb8c56ec92..442443f326e9 100644 --- a/polkadot/node/core/pvf/src/error.rs +++ b/polkadot/node/core/pvf/src/error.rs @@ -19,31 +19,44 @@ use polkadot_node_core_pvf_common::error::{InternalValidationError, PrepareError /// A error raised during validation of the candidate. #[derive(Debug, Clone)] pub enum ValidationError { - /// The error was raised because the candidate is invalid. + /// Deterministic preparation issue. In practice, most of the problems should be caught by + /// prechecking, so this may be a sign of internal conditions. /// - /// Whenever we are unsure if the error was due to the candidate or not, we must vote invalid. - InvalidCandidate(InvalidCandidate), - /// Some internal error occurred. - InternalError(InternalValidationError), + /// In principle if preparation of the `WASM` fails, the current candidate cannot be the + /// reason for that. So we can't say whether it is invalid or not. In addition, with + /// pre-checking enabled only valid runtimes should ever get enacted, so we can be + /// reasonably sure that this is some local problem on the current node. However, as this + /// particular error *seems* to indicate a deterministic error, we raise a warning. + Preparation(PrepareError), + /// The error was raised because the candidate is invalid. Should vote against. + Invalid(InvalidCandidate), + /// Possibly transient issue that may resolve after retries. Should vote against when retries + /// fail. + PossiblyInvalid(PossiblyInvalidError), + /// Preparation or execution issue caused by an internal condition. Should not vote against. + Internal(InternalValidationError), } /// A description of an error raised during executing a PVF and can be attributed to the combination /// of the candidate [`polkadot_parachain_primitives::primitives::ValidationParams`] and the PVF. #[derive(Debug, Clone)] pub enum InvalidCandidate { - /// PVF preparation ended up with a deterministic error. - PrepareError(String), /// The candidate is reported to be invalid by the execution worker. The string contains the /// error message. WorkerReportedInvalid(String), + /// PVF execution (compilation is not included) took more time than was allotted. + HardTimeout, +} + +/// Possibly transient issue that may resolve after retries. +#[derive(Debug, Clone)] +pub enum PossiblyInvalidError { /// The worker process (not the job) has died during validation of a candidate. /// /// It's unlikely that this is caused by malicious code since workers spawn separate job /// processes, and those job processes are sandboxed. But, it is possible. We retry in this /// case, and if the error persists, we assume it's caused by the candidate and vote against. AmbiguousWorkerDeath, - /// PVF execution (compilation is not included) took more time than was allotted. - HardTimeout, /// The job process (not the worker) has died for one of the following reasons: /// /// (a) A seccomp violation occurred, most likely due to an attempt by malicious code to @@ -68,7 +81,7 @@ pub enum InvalidCandidate { impl From for ValidationError { fn from(error: InternalValidationError) -> Self { - Self::InternalError(error) + Self::Internal(error) } } @@ -77,9 +90,9 @@ impl From for ValidationError { // Here we need to classify the errors into two errors: deterministic and non-deterministic. // See [`PrepareError::is_deterministic`]. if error.is_deterministic() { - Self::InvalidCandidate(InvalidCandidate::PrepareError(error.to_string())) + Self::Preparation(error) } else { - Self::InternalError(InternalValidationError::NonDeterministicPrepareError(error)) + Self::Internal(InternalValidationError::NonDeterministicPrepareError(error)) } } } diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 257377df3f48..a0c24fd44323 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -22,7 +22,7 @@ use crate::{ host::ResultSender, metrics::Metrics, worker_intf::{IdleWorker, WorkerHandle}, - InvalidCandidate, ValidationError, LOG_TARGET, + InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ channel::mpsc, @@ -342,27 +342,27 @@ fn handle_job_finish( }, Outcome::InvalidCandidate { err, idle_worker } => ( Some(idle_worker), - Err(ValidationError::InvalidCandidate(InvalidCandidate::WorkerReportedInvalid(err))), + Err(ValidationError::Invalid(InvalidCandidate::WorkerReportedInvalid(err))), None, ), - Outcome::InternalError { err } => (None, Err(ValidationError::InternalError(err)), None), + Outcome::InternalError { err } => (None, Err(ValidationError::Internal(err)), None), // Either the worker or the job timed out. Kill the worker in either case. Treated as // definitely-invalid, because if we timed out, there's no time left for a retry. Outcome::HardTimeout => - (None, Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)), None), + (None, Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), None), // "Maybe invalid" errors (will retry). Outcome::WorkerIntfErr => ( None, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)), None, ), Outcome::JobDied { err } => ( None, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousJobDeath(err))), + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))), None, ), Outcome::JobError { err } => - (None, Err(ValidationError::InvalidCandidate(InvalidCandidate::JobError(err))), None), + (None, Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError(err))), None), }; queue.metrics.execute_finished(); diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index f67934e4171c..6049f51834e3 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -873,7 +873,7 @@ fn pulse_every(interval: std::time::Duration) -> impl futures::Stream #[cfg(test)] pub(crate) mod tests { use super::*; - use crate::InvalidCandidate; + use crate::PossiblyInvalidError; use assert_matches::assert_matches; use futures::future::BoxFuture; use polkadot_node_core_pvf_common::{ @@ -1211,27 +1211,27 @@ pub(crate) mod tests { ); result_tx_pvf_1_1 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_1_1.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); result_tx_pvf_1_2 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_1_2.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); result_tx_pvf_2 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_pvf_2.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); } @@ -1337,7 +1337,7 @@ pub(crate) mod tests { assert_matches!(result_rx.now_or_never().unwrap().unwrap(), Err(PrepareError::TimedOut)); assert_matches!( result_rx_execute.now_or_never().unwrap().unwrap(), - Err(ValidationError::InternalError(_)) + Err(ValidationError::Internal(_)) ); // Reversed case: first send multiple precheck requests, then ask for an execution. @@ -1479,7 +1479,7 @@ pub(crate) mod tests { // The result should contain the error. let result = test.poll_and_recv_result(result_rx).await; - assert_matches!(result, Err(ValidationError::InternalError(_))); + assert_matches!(result, Err(ValidationError::Internal(_))); // Submit another execute request. We shouldn't try to prepare again, yet. let (result_tx_2, result_rx_2) = oneshot::channel(); @@ -1498,7 +1498,7 @@ pub(crate) mod tests { // The result should contain the original error. let result = test.poll_and_recv_result(result_rx_2).await; - assert_matches!(result, Err(ValidationError::InternalError(_))); + assert_matches!(result, Err(ValidationError::Internal(_))); // Pause for enough time to reset the cooldown for this failed prepare request. futures_timer::Delay::new(PREPARE_FAILURE_COOLDOWN).await; @@ -1538,11 +1538,11 @@ pub(crate) mod tests { // Send an error for the execution here, just so we can check the result receiver is still // alive. result_tx_3 - .send(Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath))) + .send(Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath))) .unwrap(); assert_matches!( result_rx_3.now_or_never().unwrap().unwrap(), - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); } @@ -1581,10 +1581,7 @@ pub(crate) mod tests { // The result should contain the error. let result = test.poll_and_recv_result(result_rx).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); // Submit another execute request. let (result_tx_2, result_rx_2) = oneshot::channel(); @@ -1603,10 +1600,7 @@ pub(crate) mod tests { // The result should contain the original error. let result = test.poll_and_recv_result(result_rx_2).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); // Pause for enough time to reset the cooldown for this failed prepare request. futures_timer::Delay::new(PREPARE_FAILURE_COOLDOWN).await; @@ -1628,10 +1622,7 @@ pub(crate) mod tests { // The result should still contain the original error. let result = test.poll_and_recv_result(result_rx_3).await; - assert_matches!( - result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::PrepareError(_))) - ); + assert_matches!(result, Err(ValidationError::Preparation(_))); } // Test that multiple heads-up requests trigger preparation retries if the first one failed. diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 7e7a13252548..3306a2461c15 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -103,7 +103,7 @@ mod worker_intf; #[cfg(feature = "test-utils")] pub mod testing; -pub use error::{InvalidCandidate, ValidationError}; +pub use error::{InvalidCandidate, PossiblyInvalidError, ValidationError}; pub use host::{start, Config, ValidationHost, EXECUTE_BINARY_NAME, PREPARE_BINARY_NAME}; pub use metrics::Metrics; pub use priority::Priority; diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 5bdf49cc719e..4c81ac502dd4 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -162,7 +162,7 @@ async fn execute_job_terminates_on_timeout() { .await; match result { - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, r => panic!("{:?}", r), } @@ -202,8 +202,8 @@ async fn ensure_parallel_execution() { assert_matches!( (res1, res2), ( - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)), - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)), + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) ) ); @@ -344,7 +344,7 @@ async fn deleting_prepared_artifact_does_not_dispute() { .await; match result { - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) => {}, + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) => {}, r => panic!("{:?}", r), } } diff --git a/polkadot/node/core/pvf/tests/it/process.rs b/polkadot/node/core/pvf/tests/it/process.rs index 725d060ab916..075d94373df0 100644 --- a/polkadot/node/core/pvf/tests/it/process.rs +++ b/polkadot/node/core/pvf/tests/it/process.rs @@ -19,7 +19,9 @@ use super::TestHost; use assert_matches::assert_matches; -use polkadot_node_core_pvf::{InvalidCandidate, PrepareError, ValidationError}; +use polkadot_node_core_pvf::{ + InvalidCandidate, PossiblyInvalidError, PrepareError, ValidationError, +}; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams}; use procfs::process; use rusty_fork::rusty_fork_test; @@ -151,7 +153,7 @@ rusty_fork_test! { assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::HardTimeout)) + Err(ValidationError::Invalid(InvalidCandidate::HardTimeout)) ); }) } @@ -217,7 +219,7 @@ rusty_fork_test! { assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousWorkerDeath)) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousWorkerDeath)) ); }) } @@ -288,7 +290,7 @@ rusty_fork_test! { // Note that we get a more specific error if the job died than if the whole worker died. assert_matches!( result, - Err(ValidationError::InvalidCandidate(InvalidCandidate::AmbiguousJobDeath(err))) + Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::AmbiguousJobDeath(err))) if err == "received signal: SIGKILL" ); }) From 40afc77c4e6efff56649174a35384499f601f8d4 Mon Sep 17 00:00:00 2001 From: Javier Viola Date: Tue, 21 Nov 2023 08:40:55 -0300 Subject: [PATCH 25/48] Add output positional arg to undying-collator cli (#2375) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Follow up from https://github.com/paritytech/polkadot-sdk/pull/2370 to add `output` positional argument to undying-collator. cc @JoshOrndorff --------- Co-authored-by: Bastian Köcher --- .../undying/collator/src/cli.rs | 11 +++++++- .../undying/collator/src/main.rs | 26 ++++++++++++++++--- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs index cd16133dbf19..d04122f2f689 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/cli.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/cli.rs @@ -18,6 +18,7 @@ use clap::Parser; use sc_cli::SubstrateCli; +use std::path::PathBuf; /// Sub-commands supported by the collator. #[derive(Debug, Parser)] @@ -34,6 +35,10 @@ pub enum Subcommand { /// Command for exporting the genesis state of the parachain #[derive(Debug, Parser)] pub struct ExportGenesisStateCommand { + /// Output file name or stdout if unspecified. + #[arg()] + pub output: Option, + /// Id of the parachain this collator collates for. #[arg(long, default_value_t = 100)] pub parachain_id: u32, @@ -50,7 +55,11 @@ pub struct ExportGenesisStateCommand { /// Command for exporting the genesis wasm file. #[derive(Debug, Parser)] -pub struct ExportGenesisWasmCommand {} +pub struct ExportGenesisWasmCommand { + /// Output file name or stdout if unspecified. + #[arg()] + pub output: Option, +} #[allow(missing_docs)] #[derive(Debug, Parser)] diff --git a/polkadot/parachain/test-parachains/undying/collator/src/main.rs b/polkadot/parachain/test-parachains/undying/collator/src/main.rs index e564e221f013..c6b338a20dc0 100644 --- a/polkadot/parachain/test-parachains/undying/collator/src/main.rs +++ b/polkadot/parachain/test-parachains/undying/collator/src/main.rs @@ -22,6 +22,10 @@ use polkadot_node_subsystem::messages::{CollationGenerationMessage, CollatorProt use polkadot_primitives::Id as ParaId; use sc_cli::{Error as SubstrateCliError, SubstrateCli}; use sp_core::hexdisplay::HexDisplay; +use std::{ + fs, + io::{self, Write}, +}; use test_parachain_undying_collator::Collator; mod cli; @@ -35,14 +39,30 @@ fn main() -> Result<()> { // `pov_size` and `pvf_complexity` need to match the ones that we start the collator // with. let collator = Collator::new(params.pov_size, params.pvf_complexity); - println!("0x{:?}", HexDisplay::from(&collator.genesis_head())); + + let output_buf = + format!("0x{:?}", HexDisplay::from(&collator.genesis_head())).into_bytes(); + + if let Some(output) = params.output { + std::fs::write(output, output_buf)?; + } else { + std::io::stdout().write_all(&output_buf)?; + } Ok::<_, Error>(()) }, - Some(cli::Subcommand::ExportGenesisWasm(_params)) => { + Some(cli::Subcommand::ExportGenesisWasm(params)) => { // We pass some dummy values for `pov_size` and `pvf_complexity` as these don't // matter for `wasm` export. - println!("0x{:?}", HexDisplay::from(&Collator::default().validation_code())); + let output_buf = + format!("0x{:?}", HexDisplay::from(&Collator::default().validation_code())) + .into_bytes(); + + if let Some(output) = params.output { + fs::write(output, output_buf)?; + } else { + io::stdout().write_all(&output_buf)?; + } Ok(()) }, From 552be4800d9e4b480f79a300fc531783e04be099 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Tue, 21 Nov 2023 12:52:46 +0100 Subject: [PATCH 26/48] PVF worker: switch on seccomp networking restrictions (#2221) --- polkadot/node/core/pvf/common/src/error.rs | 7 +- polkadot/node/core/pvf/common/src/execute.rs | 2 +- polkadot/node/core/pvf/common/src/lib.rs | 2 +- .../node/core/pvf/common/src/worker/mod.rs | 2 +- .../pvf/common/src/worker/security/seccomp.rs | 10 +- .../node/core/pvf/execute-worker/src/lib.rs | 26 ++-- .../node/core/pvf/prepare-worker/src/lib.rs | 24 ++-- .../node/core/pvf/src/execute/worker_intf.rs | 111 ++++++++++-------- polkadot/node/core/pvf/src/prepare/pool.rs | 4 +- .../node/core/pvf/src/prepare/worker_intf.rs | 45 ++++--- polkadot/node/core/pvf/src/security.rs | 75 ++++++++---- polkadot/node/core/pvf/tests/it/process.rs | 2 +- .../src/node/utility/pvf-host-and-workers.md | 38 +++--- 13 files changed, 202 insertions(+), 146 deletions(-) diff --git a/polkadot/node/core/pvf/common/src/error.rs b/polkadot/node/core/pvf/common/src/error.rs index 6bf05ece78ef..7db7f9a59451 100644 --- a/polkadot/node/core/pvf/common/src/error.rs +++ b/polkadot/node/core/pvf/common/src/error.rs @@ -77,7 +77,7 @@ pub enum PrepareError { #[codec(index = 9)] ClearWorkerDir(String), /// The preparation job process died, due to OOM, a seccomp violation, or some other factor. - JobDied(String), + JobDied { err: String, job_pid: i32 }, #[codec(index = 10)] /// Some error occurred when interfacing with the kernel. #[codec(index = 11)] @@ -96,7 +96,7 @@ impl PrepareError { match self { Prevalidation(_) | Preparation(_) | JobError(_) | OutOfMemory => true, IoErr(_) | - JobDied(_) | + JobDied { .. } | CreateTmpFile(_) | RenameTmpFile { .. } | ClearWorkerDir(_) | @@ -119,7 +119,8 @@ impl fmt::Display for PrepareError { JobError(err) => write!(f, "panic: {}", err), TimedOut => write!(f, "prepare: timeout"), IoErr(err) => write!(f, "prepare: io error while receiving response: {}", err), - JobDied(err) => write!(f, "prepare: prepare job died: {}", err), + JobDied { err, job_pid } => + write!(f, "prepare: prepare job with pid {job_pid} died: {err}"), CreateTmpFile(err) => write!(f, "prepare: error creating tmp file: {}", err), RenameTmpFile { err, src, dest } => write!(f, "prepare: error renaming tmp file ({:?} -> {:?}): {}", src, dest, err), diff --git a/polkadot/node/core/pvf/common/src/execute.rs b/polkadot/node/core/pvf/common/src/execute.rs index 89e7c8e471a5..aa1c1c539682 100644 --- a/polkadot/node/core/pvf/common/src/execute.rs +++ b/polkadot/node/core/pvf/common/src/execute.rs @@ -46,7 +46,7 @@ pub enum WorkerResponse { /// /// We cannot treat this as an internal error because malicious code may have killed the job. /// We still retry it, because in the non-malicious case it is likely spurious. - JobDied(String), + JobDied { err: String, job_pid: i32 }, /// An unexpected error occurred in the job process, e.g. failing to spawn a thread, panic, /// etc. /// diff --git a/polkadot/node/core/pvf/common/src/lib.rs b/polkadot/node/core/pvf/common/src/lib.rs index 282d2f7c41d0..278fa3fe821c 100644 --- a/polkadot/node/core/pvf/common/src/lib.rs +++ b/polkadot/node/core/pvf/common/src/lib.rs @@ -53,7 +53,7 @@ pub struct SecurityStatus { pub can_enable_landlock: bool, /// Whether the seccomp features we use are fully available on this system. pub can_enable_seccomp: bool, - // Whether we are able to unshare the user namespace and change the filesystem root. + /// Whether we are able to unshare the user namespace and change the filesystem root. pub can_unshare_user_namespace_and_change_root: bool, } diff --git a/polkadot/node/core/pvf/common/src/worker/mod.rs b/polkadot/node/core/pvf/common/src/worker/mod.rs index 86f47acccac6..d4f9bbc27ea6 100644 --- a/polkadot/node/core/pvf/common/src/worker/mod.rs +++ b/polkadot/node/core/pvf/common/src/worker/mod.rs @@ -219,7 +219,7 @@ pub fn run_worker( #[cfg_attr(not(target_os = "linux"), allow(unused_mut))] mut worker_dir_path: PathBuf, node_version: Option<&str>, worker_version: Option<&str>, - #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] security_status: &SecurityStatus, + security_status: &SecurityStatus, mut event_loop: F, ) where F: FnMut(UnixStream, PathBuf) -> io::Result, diff --git a/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs b/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs index 5539ad284400..c3822d3c4c69 100644 --- a/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs +++ b/polkadot/node/core/pvf/common/src/worker/security/seccomp.rs @@ -67,11 +67,9 @@ //! //! # Action on syscall violations //! -//! On syscall violations we currently only log, to make sure this works correctly before enforcing. -//! -//! In the future, when a forbidden syscall is attempted we immediately kill the process in order to -//! prevent the attacker from doing anything else. In execution, this will result in voting against -//! the candidate. +//! When a forbidden syscall is attempted we immediately kill the process in order to prevent the +//! attacker from doing anything else. In execution, this will result in voting against the +//! candidate. use crate::{ worker::{stringify_panic_payload, WorkerKind}, @@ -82,7 +80,7 @@ use std::{collections::BTreeMap, path::Path}; /// The action to take on caught syscalls. #[cfg(not(test))] -const CAUGHT_ACTION: SeccompAction = SeccompAction::Log; +const CAUGHT_ACTION: SeccompAction = SeccompAction::KillProcess; /// Don't kill the process when testing. #[cfg(test)] const CAUGHT_ACTION: SeccompAction = SeccompAction::Errno(libc::EACCES as u32); diff --git a/polkadot/node/core/pvf/execute-worker/src/lib.rs b/polkadot/node/core/pvf/execute-worker/src/lib.rs index 9ec811686b89..d82a8fca65d3 100644 --- a/polkadot/node/core/pvf/execute-worker/src/lib.rs +++ b/polkadot/node/core/pvf/execute-worker/src/lib.rs @@ -222,12 +222,6 @@ pub fn worker_entrypoint( }, }; - gum::trace!( - target: LOG_TARGET, - %worker_pid, - "worker: sending response to host: {:?}", - response - ); send_response(&mut stream, response)?; } }, @@ -360,7 +354,7 @@ fn handle_child_process( /// - The response, either `Ok` or some error state. fn handle_parent_process( mut pipe_read: PipeReader, - child: Pid, + job_pid: Pid, worker_pid: u32, usage_before: Usage, timeout: Duration, @@ -373,10 +367,11 @@ fn handle_parent_process( // Should retry at any rate. .map_err(|err| io::Error::new(io::ErrorKind::Other, err.to_string()))?; - let status = nix::sys::wait::waitpid(child, None); + let status = nix::sys::wait::waitpid(job_pid, None); gum::trace!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute worker received wait status from job: {:?}", status, ); @@ -396,6 +391,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute job took {}ms cpu time, exceeded execute timeout {}ms", cpu_tv.as_millis(), timeout.as_millis(), @@ -428,6 +424,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "execute job error: {}", job_error, ); @@ -443,15 +440,18 @@ fn handle_parent_process( // // The job gets SIGSYS on seccomp violations, but this signal may have been sent for some // other reason, so we still need to check for seccomp violations elsewhere. - Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => - Ok(WorkerResponse::JobDied(format!("received signal: {signal:?}"))), + Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => Ok(WorkerResponse::JobDied { + err: format!("received signal: {signal:?}"), + job_pid: job_pid.as_raw(), + }), Err(errno) => Ok(internal_error_from_errno("waitpid", errno)), // It is within an attacker's power to send an unexpected exit status. So we cannot treat // this as an internal error (which would make us abstain), but must vote against. - Ok(unexpected_wait_status) => Ok(WorkerResponse::JobDied(format!( - "unexpected status from wait: {unexpected_wait_status:?}" - ))), + Ok(unexpected_wait_status) => Ok(WorkerResponse::JobDied { + err: format!("unexpected status from wait: {unexpected_wait_status:?}"), + job_pid: job_pid.as_raw(), + }), } } diff --git a/polkadot/node/core/pvf/prepare-worker/src/lib.rs b/polkadot/node/core/pvf/prepare-worker/src/lib.rs index 34e6a78c26ae..499e87c6e207 100644 --- a/polkadot/node/core/pvf/prepare-worker/src/lib.rs +++ b/polkadot/node/core/pvf/prepare-worker/src/lib.rs @@ -257,9 +257,9 @@ pub fn worker_entrypoint( handle_parent_process( pipe_reader, + worker_pid, child, temp_artifact_dest.clone(), - worker_pid, usage_before, preparation_timeout, ) @@ -506,9 +506,9 @@ fn handle_child_process( /// - If the child process timeout, it returns `PrepareError::TimedOut`. fn handle_parent_process( mut pipe_read: PipeReader, - child: Pid, - temp_artifact_dest: PathBuf, worker_pid: u32, + job_pid: Pid, + temp_artifact_dest: PathBuf, usage_before: Usage, timeout: Duration, ) -> Result { @@ -518,10 +518,11 @@ fn handle_parent_process( .read_to_end(&mut received_data) .map_err(|err| PrepareError::IoErr(err.to_string()))?; - let status = nix::sys::wait::waitpid(child, None); + let status = nix::sys::wait::waitpid(job_pid, None); gum::trace!( target: LOG_TARGET, %worker_pid, + %job_pid, "prepare worker received wait status from job: {:?}", status, ); @@ -539,6 +540,7 @@ fn handle_parent_process( gum::warn!( target: LOG_TARGET, %worker_pid, + %job_pid, "prepare job took {}ms cpu time, exceeded prepare timeout {}ms", cpu_tv.as_millis(), timeout.as_millis(), @@ -573,6 +575,7 @@ fn handle_parent_process( gum::debug!( target: LOG_TARGET, %worker_pid, + %job_pid, "worker: writing artifact to {}", temp_artifact_dest.display(), ); @@ -593,15 +596,18 @@ fn handle_parent_process( // // The job gets SIGSYS on seccomp violations, but this signal may have been sent for some // other reason, so we still need to check for seccomp violations elsewhere. - Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => - Err(PrepareError::JobDied(format!("received signal: {signal:?}"))), + Ok(WaitStatus::Signaled(_pid, signal, _core_dump)) => Err(PrepareError::JobDied { + err: format!("received signal: {signal:?}"), + job_pid: job_pid.as_raw(), + }), Err(errno) => Err(error_from_errno("waitpid", errno)), // An attacker can make the child process return any exit status it wants. So we can treat // all unexpected cases the same way. - Ok(unexpected_wait_status) => Err(PrepareError::JobDied(format!( - "unexpected status from wait: {unexpected_wait_status:?}" - ))), + Ok(unexpected_wait_status) => Err(PrepareError::JobDied { + err: format!("unexpected status from wait: {unexpected_wait_status:?}"), + job_pid: job_pid.as_raw(), + }), } } diff --git a/polkadot/node/core/pvf/src/execute/worker_intf.rs b/polkadot/node/core/pvf/src/execute/worker_intf.rs index bf44ba017250..4faaf13e62f7 100644 --- a/polkadot/node/core/pvf/src/execute/worker_intf.rs +++ b/polkadot/node/core/pvf/src/execute/worker_intf.rs @@ -33,7 +33,7 @@ use polkadot_node_core_pvf_common::{ execute::{Handshake, WorkerResponse}, worker_dir, SecurityStatus, }; -use polkadot_parachain_primitives::primitives::ValidationResult; +use polkadot_parachain_primitives::primitives::{ValidationCodeHash, ValidationResult}; use polkadot_primitives::ExecutorParams; use std::{path::Path, time::Duration}; use tokio::{io, net::UnixStream}; @@ -156,6 +156,16 @@ pub async fn start_work( let response = futures::select! { response = recv_response(&mut stream).fuse() => { match response { + Ok(response) => + handle_response( + response, + pid, + &artifact.id.code_hash, + &artifact_path, + execution_timeout, + audit_log_file + ) + .await, Err(error) => { gum::warn!( target: LOG_TARGET, @@ -164,56 +174,9 @@ pub async fn start_work( ?error, "failed to recv an execute response", ); - // The worker died. Check if it was due to a seccomp violation. - // - // NOTE: Log, but don't change the outcome. Not all validators may have - // auditing enabled, so we don't want attackers to abuse a non-deterministic - // outcome. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - validation_code_hash = ?artifact.id.code_hash, - ?artifact_path, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } return Outcome::WorkerIntfErr }, - Ok(response) => { - // Check if any syscall violations occurred during the job. For now this is - // only informative, as we are not enforcing the seccomp policy yet. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - validation_code_hash = ?artifact.id.code_hash, - ?artifact_path, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } - - if let WorkerResponse::Ok{duration, ..} = response { - if duration > execution_timeout { - // The job didn't complete within the timeout. - gum::warn!( - target: LOG_TARGET, - worker_pid = %pid, - "execute job took {}ms cpu time, exceeded execution timeout {}ms.", - duration.as_millis(), - execution_timeout.as_millis(), - ); - - // Return a timeout error. - return Outcome::HardTimeout - } - } - - response - }, } }, _ = Delay::new(timeout).fuse() => { @@ -238,7 +201,7 @@ pub async fn start_work( idle_worker: IdleWorker { stream, pid, worker_dir }, }, WorkerResponse::JobTimedOut => Outcome::HardTimeout, - WorkerResponse::JobDied(err) => Outcome::JobDied { err }, + WorkerResponse::JobDied { err, job_pid: _ } => Outcome::JobDied { err }, WorkerResponse::JobError(err) => Outcome::JobError { err }, WorkerResponse::InternalError(err) => Outcome::InternalError { err }, @@ -247,6 +210,56 @@ pub async fn start_work( .await } +/// Handles the case where we successfully received response bytes on the host from the child. +/// +/// Here we know the artifact exists, but is still located in a temporary file which will be cleared +/// by [`with_worker_dir_setup`]. +async fn handle_response( + response: WorkerResponse, + worker_pid: u32, + validation_code_hash: &ValidationCodeHash, + artifact_path: &Path, + execution_timeout: Duration, + audit_log_file: Option, +) -> WorkerResponse { + if let WorkerResponse::Ok { duration, .. } = response { + if duration > execution_timeout { + // The job didn't complete within the timeout. + gum::warn!( + target: LOG_TARGET, + worker_pid, + "execute job took {}ms cpu time, exceeded execution timeout {}ms.", + duration.as_millis(), + execution_timeout.as_millis(), + ); + + // Return a timeout error. + return WorkerResponse::JobTimedOut + } + } + + if let WorkerResponse::JobDied { err: _, job_pid } = response { + // The job died. Check if it was due to a seccomp violation. + // + // NOTE: Log, but don't change the outcome. Not all validators may have + // auditing enabled, so we don't want attackers to abuse a non-deterministic + // outcome. + for syscall in security::check_seccomp_violations_for_job(audit_log_file, job_pid).await { + gum::error!( + target: LOG_TARGET, + %worker_pid, + %job_pid, + %syscall, + ?validation_code_hash, + ?artifact_path, + "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" + ); + } + } + + response +} + /// Create a temporary file for an artifact in the worker cache, execute the given future/closure /// passing the file path in, and clean up the worker cache. /// diff --git a/polkadot/node/core/pvf/src/prepare/pool.rs b/polkadot/node/core/pvf/src/prepare/pool.rs index 21af21e5b028..4901be9fe1b7 100644 --- a/polkadot/node/core/pvf/src/prepare/pool.rs +++ b/polkadot/node/core/pvf/src/prepare/pool.rs @@ -388,14 +388,14 @@ fn handle_mux( Ok(()) }, // The worker might still be usable, but we kill it just in case. - Outcome::JobDied(err) => { + Outcome::JobDied { err, job_pid } => { if attempt_retire(metrics, spawned, worker) { reply( from_pool, FromPool::Concluded { worker, rip: true, - result: Err(PrepareError::JobDied(err)), + result: Err(PrepareError::JobDied { err, job_pid }), }, )?; } diff --git a/polkadot/node/core/pvf/src/prepare/worker_intf.rs b/polkadot/node/core/pvf/src/prepare/worker_intf.rs index e7f142a46bb8..318aea7295de 100644 --- a/polkadot/node/core/pvf/src/prepare/worker_intf.rs +++ b/polkadot/node/core/pvf/src/prepare/worker_intf.rs @@ -104,7 +104,7 @@ pub enum Outcome { /// The preparation job process died, due to OOM, a seccomp violation, or some other factor. /// /// The worker might still be usable, but we kill it just in case. - JobDied(String), + JobDied { err: String, job_pid: i32 }, } /// Given the idle token of a worker and parameters of work, communicates with the worker and @@ -160,19 +160,7 @@ pub async fn start_work( match result { // Received bytes from worker within the time limit. - Ok(Ok(prepare_worker_result)) => { - // Check if any syscall violations occurred during the job. For now this is only - // informative, as we are not enforcing the seccomp policy yet. - for syscall in security::check_seccomp_violations_for_worker(audit_log_file, pid).await { - gum::error!( - target: LOG_TARGET, - worker_pid = %pid, - %syscall, - ?pvf, - "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" - ); - } - + Ok(Ok(prepare_worker_result)) => handle_response( metrics, IdleWorker { stream, pid, worker_dir }, @@ -182,9 +170,9 @@ pub async fn start_work( &pvf, &cache_path, preparation_timeout, + audit_log_file, ) - .await - }, + .await, Ok(Err(err)) => { // Communication error within the time limit. gum::warn!( @@ -221,15 +209,36 @@ async fn handle_response( worker_pid: u32, tmp_file: PathBuf, pvf: &PvfPrepData, - cache_path: &PathBuf, + cache_path: &Path, preparation_timeout: Duration, + audit_log_file: Option, ) -> Outcome { let PrepareWorkerSuccess { checksum, stats: PrepareStats { cpu_time_elapsed, memory_stats } } = match result.clone() { Ok(result) => result, // Timed out on the child. This should already be logged by the child. Err(PrepareError::TimedOut) => return Outcome::TimedOut, - Err(PrepareError::JobDied(err)) => return Outcome::JobDied(err), + Err(PrepareError::JobDied { err, job_pid }) => { + // The job died. Check if it was due to a seccomp violation. + // + // NOTE: Log, but don't change the outcome. Not all validators may have + // auditing enabled, so we don't want attackers to abuse a non-deterministic + // outcome. + for syscall in + security::check_seccomp_violations_for_job(audit_log_file, job_pid).await + { + gum::error!( + target: LOG_TARGET, + %worker_pid, + %job_pid, + %syscall, + ?pvf, + "A forbidden syscall was attempted! This is a violation of our seccomp security policy. Report an issue ASAP!" + ); + } + + return Outcome::JobDied { err, job_pid } + }, Err(PrepareError::OutOfMemory) => return Outcome::OutOfMemory, Err(err) => return Outcome::Concluded { worker, result: Err(err) }, }; diff --git a/polkadot/node/core/pvf/src/security.rs b/polkadot/node/core/pvf/src/security.rs index 0c0c5f401663..8c06c68392f4 100644 --- a/polkadot/node/core/pvf/src/security.rs +++ b/polkadot/node/core/pvf/src/security.rs @@ -178,9 +178,15 @@ async fn check_can_unshare_user_namespace_and_change_root( let stderr = std::str::from_utf8(&output.stderr) .expect("child process writes a UTF-8 string to stderr; qed") .trim(); - Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( - format!("not available: {}", stderr) - )) + if stderr.is_empty() { + Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( + "not available".into() + )) + } else { + Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( + format!("not available: {}", stderr) + )) + } }, Err(err) => Err(SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( @@ -208,16 +214,25 @@ async fn check_landlock( if #[cfg(target_os = "linux")] { match tokio::process::Command::new(prepare_worker_program_path) .arg("--check-can-enable-landlock") - .status() + .output() .await { - Ok(status) if status.success() => Ok(()), - Ok(_status) => { + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { let abi = polkadot_node_core_pvf_common::worker::security::landlock::LANDLOCK_ABI as u8; - Err(SecureModeError::CannotEnableLandlock( - format!("landlock ABI {} not available", abi) - )) + let stderr = std::str::from_utf8(&output.stderr) + .expect("child process writes a UTF-8 string to stderr; qed") + .trim(); + if stderr.is_empty() { + Err(SecureModeError::CannotEnableLandlock( + format!("landlock ABI {} not available", abi) + )) + } else { + Err(SecureModeError::CannotEnableLandlock( + format!("not available: {}", stderr) + )) + } }, Err(err) => Err(SecureModeError::CannotEnableLandlock( @@ -238,7 +253,7 @@ async fn check_landlock( /// to running the check in a worker, we try it... in a worker. The expected return status is 0 on /// success and -1 on failure. async fn check_seccomp( - #[cfg_attr(not(target_os = "linux"), allow(unused_variables))] + #[cfg_attr(not(all(target_os = "linux", target_arch = "x86_64")), allow(unused_variables))] prepare_worker_program_path: &Path, ) -> SecureModeResult { cfg_if::cfg_if! { @@ -247,14 +262,24 @@ async fn check_seccomp( if #[cfg(target_arch = "x86_64")] { match tokio::process::Command::new(prepare_worker_program_path) .arg("--check-can-enable-seccomp") - .status() + .output() .await { - Ok(status) if status.success() => Ok(()), - Ok(_status) => - Err(SecureModeError::CannotEnableSeccomp( - "not available".into() - )), + Ok(output) if output.status.success() => Ok(()), + Ok(output) => { + let stderr = std::str::from_utf8(&output.stderr) + .expect("child process writes a UTF-8 string to stderr; qed") + .trim(); + if stderr.is_empty() { + Err(SecureModeError::CannotEnableSeccomp( + "not available".into() + )) + } else { + Err(SecureModeError::CannotEnableSeccomp( + format!("not available: {}", stderr) + )) + } + }, Err(err) => Err(SecureModeError::CannotEnableSeccomp( format!("could not start child process: {}", err) @@ -320,25 +345,25 @@ impl AuditLogFile { } } -/// Check if a seccomp violation occurred for the given worker. As the syslog may be in a different -/// location, or seccomp auditing may be disabled, this function provides a best-effort attempt -/// only. +/// Check if a seccomp violation occurred for the given job process. As the syslog may be in a +/// different location, or seccomp auditing may be disabled, this function provides a best-effort +/// attempt only. /// /// The `audit_log_file` must have been obtained before the job started. It only allows reading /// entries that were written since it was obtained, so that we do not consider events from previous /// processes with the same pid. This can still be racy, but it's unlikely and fine for a /// best-effort attempt. -pub async fn check_seccomp_violations_for_worker( +pub async fn check_seccomp_violations_for_job( audit_log_file: Option, - worker_pid: u32, + job_pid: i32, ) -> Vec { - let audit_event_pid_field = format!("pid={worker_pid}"); + let audit_event_pid_field = format!("pid={job_pid}"); let audit_log_file = match audit_log_file { Some(file) => { - gum::debug!( + gum::trace!( target: LOG_TARGET, - %worker_pid, + %job_pid, audit_log_path = ?file.path, "checking audit log for seccomp violations", ); @@ -347,7 +372,7 @@ pub async fn check_seccomp_violations_for_worker( None => { gum::warn!( target: LOG_TARGET, - %worker_pid, + %job_pid, "could not open either {AUDIT_LOG_PATH} or {SYSLOG_PATH} for reading audit logs" ); return vec![] diff --git a/polkadot/node/core/pvf/tests/it/process.rs b/polkadot/node/core/pvf/tests/it/process.rs index 075d94373df0..b742acb15d02 100644 --- a/polkadot/node/core/pvf/tests/it/process.rs +++ b/polkadot/node/core/pvf/tests/it/process.rs @@ -248,7 +248,7 @@ rusty_fork_test! { // Note that we get a more specific error if the job died than if the whole worker died. assert_matches!( result, - Err(PrepareError::JobDied(err)) if err == "received signal: SIGKILL" + Err(PrepareError::JobDied{ err, job_pid: _ }) if err == "received signal: SIGKILL" ); }) } diff --git a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md index 74d88ba3ad99..56bdd48bc0c3 100644 --- a/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md +++ b/polkadot/roadmap/implementers-guide/src/node/utility/pvf-host-and-workers.md @@ -38,16 +38,22 @@ getting backed and honest backers getting slashed. We currently know of the following specific cases that will lead to a retried execution request: -1. **OOM:** The host might have been temporarily low on memory due to other - processes running on the same machine. **NOTE:** This case will lead to - voting against the candidate (and possibly a dispute) if the retry is still - not successful. -2. **Artifact missing:** The prepared artifact might have been deleted due to +1. **OOM:** We have memory limits to try to prevent attackers from exhausting + host memory. If the memory limit is hit, we kill the job process and retry + the job. Alternatively, the host might have been temporarily low on memory + due to other processes running on the same machine. **NOTE:** This case will + lead to voting against the candidate (and possibly a dispute) if the retry is + still not successful. +2. **Syscall violations:** If the job attempts a system call that is blocked by + the sandbox's security policy, the job process is immediately killed and we + retry. **NOTE:** In the future, if we have a proper way to detect that the + job died due to a security violation, it might make sense not to retry in + this case. +3. **Artifact missing:** The prepared artifact might have been deleted due to operator error or some bug in the system. -3. **Job errors:** For example, the worker thread panicked for some - indeterminate reason, which may or may not be independent of the candidate or - PVF. -4. **Internal errors:** See "Internal Errors" section. In this case, after the +4. **Job errors:** For example, the job process panicked for some indeterminate + reason, which may or may not be independent of the candidate or PVF. +5. **Internal errors:** See "Internal Errors" section. In this case, after the retry we abstain from voting. ### Preparation timeouts @@ -159,16 +165,14 @@ data on the host machine. *Currently this is only supported on Linux.* - +### Restricting networking - +We also disable networking on PVF threads by disabling certain syscalls, such as +the creation of sockets. This prevents attackers from either downloading +payloads or communicating sensitive data from the validator's machine to the +outside world. - - - - - - +*Currently this is only supported on Linux.* ### Clearing env vars From b25d29a50247f284cde5b24e270b6db7b4a81e7e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 15:22:42 +0100 Subject: [PATCH 27/48] cumulus-consensus-common: block import: `delayed_best_block` flag added (#2001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR adds the `delayed_best_block` flag to `ParachainBlockImport`. If not set, the `params.fork_choice` is not updated (to `ForkChoiceStrategy::Custom`) during the block import. When `delayed_best_block` is set to `false` all parachain blocks on the [longest fork](https://github.com/paritytech/polkadot-sdk/blob/552be4800d9e4b480f79a300fc531783e04be099/substrate/client/service/src/client/client.rs#L708-L709) will be notified as the best block, allowing transaction pool to be updated with every imported block. Otherwise imported blocks will not be notified as best blocks (`fork_choice=ForkChoiceStrategy::Custom(false)`), and transaction pool would be updated only with best block received from relay-chain. Improvement for: #1202 --------- Co-authored-by: Bastian Köcher --- cumulus/client/consensus/common/src/lib.rs | 39 +++++++++++++++----- cumulus/client/consensus/common/src/tests.rs | 6 ++- 2 files changed, 33 insertions(+), 12 deletions(-) diff --git a/cumulus/client/consensus/common/src/lib.rs b/cumulus/client/consensus/common/src/lib.rs index 08bceabb2bd4..cebe34e7ea58 100644 --- a/cumulus/client/consensus/common/src/lib.rs +++ b/cumulus/client/consensus/common/src/lib.rs @@ -111,12 +111,15 @@ impl ParachainConsensus for Box + Send + /// Parachain specific block import. /// -/// This is used to set `block_import_params.fork_choice` to `false` as long as the block origin is -/// not `NetworkInitialSync`. The best block for parachains is determined by the relay chain. -/// Meaning we will update the best block, as it is included by the relay-chain. +/// Specialized block import for parachains. It supports to delay setting the best block until the +/// relay chain has included a candidate in its best block. By default the delayed best block +/// setting is disabled. The block import also monitors the imported blocks and prunes by default if +/// there are too many blocks at the same height. Too many blocks at the same height can for example +/// happen if the relay chain is rejecting the parachain blocks in the validation. pub struct ParachainBlockImport { inner: BI, monitor: Option>>, + delayed_best_block: bool, } impl> ParachainBlockImport { @@ -141,13 +144,27 @@ impl> ParachainBlockImport let monitor = level_limit.map(|level_limit| SharedData::new(LevelMonitor::new(level_limit, backend))); - Self { inner, monitor } + Self { inner, monitor, delayed_best_block: false } + } + + /// Create a new instance which delays setting the best block. + /// + /// The number of leaves per level limit is set to `LevelLimit::Default`. + pub fn new_with_delayed_best_block(inner: BI, backend: Arc) -> Self { + Self { + delayed_best_block: true, + ..Self::new_with_limit(inner, backend, LevelLimit::Default) + } } } impl Clone for ParachainBlockImport { fn clone(&self) -> Self { - ParachainBlockImport { inner: self.inner.clone(), monitor: self.monitor.clone() } + ParachainBlockImport { + inner: self.inner.clone(), + monitor: self.monitor.clone(), + delayed_best_block: self.delayed_best_block, + } } } @@ -182,11 +199,13 @@ where params.finalized = true; } - // Best block is determined by the relay chain, or if we are doing the initial sync - // we import all blocks as new best. - params.fork_choice = Some(sc_consensus::ForkChoiceStrategy::Custom( - params.origin == sp_consensus::BlockOrigin::NetworkInitialSync, - )); + if self.delayed_best_block { + // Best block is determined by the relay chain, or if we are doing the initial sync + // we import all blocks as new best. + params.fork_choice = Some(sc_consensus::ForkChoiceStrategy::Custom( + params.origin == sp_consensus::BlockOrigin::NetworkInitialSync, + )); + } let maybe_lock = self.monitor.as_ref().map(|monitor_lock| { let mut monitor = monitor_lock.shared_data_locked(); diff --git a/cumulus/client/consensus/common/src/tests.rs b/cumulus/client/consensus/common/src/tests.rs index 9658a0add790..597d1ab2acc2 100644 --- a/cumulus/client/consensus/common/src/tests.rs +++ b/cumulus/client/consensus/common/src/tests.rs @@ -1124,7 +1124,8 @@ fn find_potential_parents_aligned_with_pending() { let backend = Arc::new(Backend::new_test(1000, 1)); let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); - let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let mut para_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let relay_parent = relay_hash_from_block_num(10); // Choose different relay parent for alternative chain to get new hashes. @@ -1279,7 +1280,8 @@ fn find_potential_parents_aligned_no_pending() { let backend = Arc::new(Backend::new_test(1000, 1)); let client = Arc::new(TestClientBuilder::with_backend(backend.clone()).build()); - let mut para_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let mut para_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let relay_parent = relay_hash_from_block_num(10); // Choose different relay parent for alternative chain to get new hashes. From b3841b6b71cd3707d80e90492059d1a2e3bd33a6 Mon Sep 17 00:00:00 2001 From: Francisco Aguirre Date: Tue, 21 Nov 2023 16:09:40 +0100 Subject: [PATCH 28/48] Different XCM builders, default one requires fee payment (#2253) Adding on top of the new builder pattern for creating XCM programs, I'm adding some more APIs: ```rust let paying_fees: Xcm<()> = Xcm::builder() // Only allow paying for fees .withdraw_asset() // First instruction has to load the holding register .buy_execution() // Second instruction has to be `buy_execution` .build(); let paying_fees_invalid: Xcm<()> = Xcm::builder() .withdraw_asset() .build(); // Invalid, need to pay for fees let not_paying_fees: Xcm<()> = Xcm::builder_unpaid() .unpaid_execution() // Needed .withdraw_asset() .deposit_asset() .build(); let all_goes: Xcm<()> = Xcm::builder_unsafe() // You can do anything .withdraw_asset() .deposit_asset() .build(); ``` The invalid bits are because the methods don't even exist on the types that you'd want to call them on. --------- Co-authored-by: command-bot <> --- Cargo.lock | 1 + polkadot/xcm/procedural/Cargo.toml | 1 + .../xcm/procedural/src/builder_pattern.rs | 287 ++++++++++++++++-- polkadot/xcm/procedural/src/lib.rs | 6 +- .../xcm/procedural/tests/builder_pattern.rs | 81 +++++ polkadot/xcm/procedural/tests/ui.rs | 2 +- .../badly_formatted_attribute.rs | 32 ++ .../badly_formatted_attribute.stderr | 5 + .../buy_execution_named_fields.rs | 30 ++ .../buy_execution_named_fields.stderr | 5 + .../loads_holding_no_operands.rs | 32 ++ .../loads_holding_no_operands.stderr | 6 + .../ui/builder_pattern/no_buy_execution.rs | 29 ++ .../builder_pattern/no_buy_execution.stderr | 6 + .../ui/builder_pattern/no_unpaid_execution.rs | 29 ++ .../no_unpaid_execution.stderr | 6 + .../builder_pattern/unexpected_attribute.rs | 32 ++ .../unexpected_attribute.stderr | 5 + .../unpaid_execution_named_fields.rs | 30 ++ .../unpaid_execution_named_fields.stderr | 5 + .../wrong_target.rs} | 0 .../wrong_target.stderr} | 2 +- polkadot/xcm/src/v3/mod.rs | 4 + polkadot/xcm/xcm-simulator/example/src/lib.rs | 19 -- prdoc/pr_2253.prdoc | 24 ++ 25 files changed, 625 insertions(+), 54 deletions(-) create mode 100644 polkadot/xcm/procedural/tests/builder_pattern.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs create mode 100644 polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr rename polkadot/xcm/procedural/tests/ui/{builder_pattern.rs => builder_pattern/wrong_target.rs} (100%) rename polkadot/xcm/procedural/tests/ui/{builder_pattern.stderr => builder_pattern/wrong_target.stderr} (63%) create mode 100644 prdoc/pr_2253.prdoc diff --git a/Cargo.lock b/Cargo.lock index 697f232f9719..88cefe7bbef5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21512,6 +21512,7 @@ dependencies = [ "Inflector", "proc-macro2", "quote", + "staging-xcm", "syn 2.0.38", "trybuild", ] diff --git a/polkadot/xcm/procedural/Cargo.toml b/polkadot/xcm/procedural/Cargo.toml index 33c2a94be0e4..8ab27c91dae1 100644 --- a/polkadot/xcm/procedural/Cargo.toml +++ b/polkadot/xcm/procedural/Cargo.toml @@ -18,3 +18,4 @@ Inflector = "0.11.4" [dev-dependencies] trybuild = { version = "1.0.74", features = ["diff"] } +xcm = { package = "staging-xcm", path = ".." } diff --git a/polkadot/xcm/procedural/src/builder_pattern.rs b/polkadot/xcm/procedural/src/builder_pattern.rs index ebad54e972b6..1cb795ea9b20 100644 --- a/polkadot/xcm/procedural/src/builder_pattern.rs +++ b/polkadot/xcm/procedural/src/builder_pattern.rs @@ -17,56 +17,83 @@ //! Derive macro for creating XCMs with a builder pattern use inflector::Inflector; -use proc_macro::TokenStream; use proc_macro2::TokenStream as TokenStream2; use quote::{format_ident, quote}; use syn::{ - parse_macro_input, Data, DeriveInput, Error, Expr, ExprLit, Fields, Lit, Meta, MetaNameValue, + Data, DataEnum, DeriveInput, Error, Expr, ExprLit, Fields, Ident, Lit, Meta, MetaNameValue, + Result, Variant, }; -pub fn derive(input: TokenStream) -> TokenStream { - let input = parse_macro_input!(input as DeriveInput); - let builder_impl = match &input.data { - Data::Enum(data_enum) => generate_methods_for_enum(input.ident, data_enum), - _ => - return Error::new_spanned(&input, "Expected the `Instruction` enum") - .to_compile_error() - .into(), +pub fn derive(input: DeriveInput) -> Result { + let data_enum = match &input.data { + Data::Enum(data_enum) => data_enum, + _ => return Err(Error::new_spanned(&input, "Expected the `Instruction` enum")), }; + let builder_raw_impl = generate_builder_raw_impl(&input.ident, data_enum); + let builder_impl = generate_builder_impl(&input.ident, data_enum)?; + let builder_unpaid_impl = generate_builder_unpaid_impl(&input.ident, data_enum)?; let output = quote! { - pub struct XcmBuilder(Vec>); + /// A trait for types that track state inside the XcmBuilder + pub trait XcmBuilderState {} + + /// Access to all the instructions + pub enum AnythingGoes {} + /// You need to pay for execution + pub enum PaymentRequired {} + /// The holding register was loaded, now to buy execution + pub enum LoadedHolding {} + /// Need to explicitly state it won't pay for fees + pub enum ExplicitUnpaidRequired {} + + impl XcmBuilderState for AnythingGoes {} + impl XcmBuilderState for PaymentRequired {} + impl XcmBuilderState for LoadedHolding {} + impl XcmBuilderState for ExplicitUnpaidRequired {} + + /// Type used to build XCM programs + pub struct XcmBuilder { + pub(crate) instructions: Vec>, + pub state: core::marker::PhantomData, + } + impl Xcm { - pub fn builder() -> XcmBuilder { - XcmBuilder::(Vec::new()) + pub fn builder() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } + } + pub fn builder_unpaid() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } + } + pub fn builder_unsafe() -> XcmBuilder { + XcmBuilder:: { + instructions: Vec::new(), + state: core::marker::PhantomData, + } } } #builder_impl + #builder_unpaid_impl + #builder_raw_impl }; - output.into() + Ok(output) } -fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> TokenStream2 { +fn generate_builder_raw_impl(name: &Ident, data_enum: &DataEnum) -> TokenStream2 { let methods = data_enum.variants.iter().map(|variant| { let variant_name = &variant.ident; let method_name_string = &variant_name.to_string().to_snake_case(); let method_name = syn::Ident::new(&method_name_string, variant_name.span()); - let docs: Vec<_> = variant - .attrs - .iter() - .filter_map(|attr| match &attr.meta { - Meta::NameValue(MetaNameValue { - value: Expr::Lit(ExprLit { lit: Lit::Str(literal), .. }), - .. - }) if attr.path().is_ident("doc") => Some(literal.value()), - _ => None, - }) - .map(|doc| syn::parse_str::(&format!("/// {}", doc)).unwrap()) - .collect(); + let docs = get_doc_comments(&variant); let method = match &variant.fields { Fields::Unit => { quote! { pub fn #method_name(mut self) -> Self { - self.0.push(#name::::#variant_name); + self.instructions.push(#name::::#variant_name); self } } @@ -81,7 +108,7 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok let arg_types: Vec<_> = fields.unnamed.iter().map(|field| &field.ty).collect(); quote! { pub fn #method_name(mut self, #(#arg_names: #arg_types),*) -> Self { - self.0.push(#name::::#variant_name(#(#arg_names),*)); + self.instructions.push(#name::::#variant_name(#(#arg_names),*)); self } } @@ -91,7 +118,7 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); quote! { pub fn #method_name(mut self, #(#arg_names: #arg_types),*) -> Self { - self.0.push(#name::::#variant_name { #(#arg_names),* }); + self.instructions.push(#name::::#variant_name { #(#arg_names),* }); self } } @@ -103,13 +130,209 @@ fn generate_methods_for_enum(name: syn::Ident, data_enum: &syn::DataEnum) -> Tok } }); let output = quote! { - impl XcmBuilder { + impl XcmBuilder { #(#methods)* pub fn build(self) -> Xcm { - Xcm(self.0) + Xcm(self.instructions) } } }; output } + +fn generate_builder_impl(name: &Ident, data_enum: &DataEnum) -> Result { + // We first require an instruction that load the holding register + let load_holding_variants = data_enum + .variants + .iter() + .map(|variant| { + let maybe_builder_attr = variant.attrs.iter().find(|attr| match attr.meta { + Meta::List(ref list) => { + return list.path.is_ident("builder"); + }, + _ => false, + }); + let builder_attr = match maybe_builder_attr { + Some(builder) => builder.clone(), + None => return Ok(None), /* It's not going to be an instruction that loads the + * holding register */ + }; + let Meta::List(ref list) = builder_attr.meta else { unreachable!("We checked before") }; + let inner_ident: Ident = syn::parse2(list.tokens.clone().into()).map_err(|_| { + Error::new_spanned(&builder_attr, "Expected `builder(loads_holding)`") + })?; + let ident_to_match: Ident = syn::parse_quote!(loads_holding); + if inner_ident == ident_to_match { + Ok(Some(variant)) + } else { + Err(Error::new_spanned(&builder_attr, "Expected `builder(loads_holding)`")) + } + }) + .collect::>>()?; + + let load_holding_methods = load_holding_variants + .into_iter() + .flatten() + .map(|variant| { + let variant_name = &variant.ident; + let method_name_string = &variant_name.to_string().to_snake_case(); + let method_name = syn::Ident::new(&method_name_string, variant_name.span()); + let docs = get_doc_comments(&variant); + let method = match &variant.fields { + Fields::Unnamed(fields) => { + let arg_names: Vec<_> = fields + .unnamed + .iter() + .enumerate() + .map(|(index, _)| format_ident!("arg{}", index)) + .collect(); + let arg_types: Vec<_> = fields.unnamed.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name(#(#arg_names),*)); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + Fields::Named(fields) => { + let arg_names: Vec<_> = fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + _ => + return Err(Error::new_spanned( + &variant, + "Instructions that load the holding register should take operands", + )), + }; + Ok(method) + }) + .collect::, _>>()?; + + let first_impl = quote! { + impl XcmBuilder { + #(#load_holding_methods)* + } + }; + + // Then we require fees to be paid + let buy_execution_method = data_enum + .variants + .iter() + .find(|variant| variant.ident.to_string() == "BuyExecution") + .map_or( + Err(Error::new_spanned(&data_enum.variants, "No BuyExecution instruction")), + |variant| { + let variant_name = &variant.ident; + let method_name_string = &variant_name.to_string().to_snake_case(); + let method_name = syn::Ident::new(&method_name_string, variant_name.span()); + let docs = get_doc_comments(&variant); + let fields = match &variant.fields { + Fields::Named(fields) => { + let arg_names: Vec<_> = + fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = + fields.named.iter().map(|field| &field.ty).collect(); + quote! { + #(#docs)* + pub fn #method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#variant_name { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }, + _ => + return Err(Error::new_spanned( + &variant, + "BuyExecution should have named fields", + )), + }; + Ok(fields) + }, + )?; + + let second_impl = quote! { + impl XcmBuilder { + #buy_execution_method + } + }; + + let output = quote! { + #first_impl + #second_impl + }; + + Ok(output) +} + +fn generate_builder_unpaid_impl(name: &Ident, data_enum: &DataEnum) -> Result { + let unpaid_execution_variant = data_enum + .variants + .iter() + .find(|variant| variant.ident.to_string() == "UnpaidExecution") + .ok_or(Error::new_spanned(&data_enum.variants, "No UnpaidExecution instruction"))?; + let unpaid_execution_ident = &unpaid_execution_variant.ident; + let unpaid_execution_method_name = Ident::new( + &unpaid_execution_ident.to_string().to_snake_case(), + unpaid_execution_ident.span(), + ); + let docs = get_doc_comments(&unpaid_execution_variant); + let fields = match &unpaid_execution_variant.fields { + Fields::Named(fields) => fields, + _ => + return Err(Error::new_spanned( + &unpaid_execution_variant, + "UnpaidExecution should have named fields", + )), + }; + let arg_names: Vec<_> = fields.named.iter().map(|field| &field.ident).collect(); + let arg_types: Vec<_> = fields.named.iter().map(|field| &field.ty).collect(); + Ok(quote! { + impl XcmBuilder { + #(#docs)* + pub fn #unpaid_execution_method_name(self, #(#arg_names: #arg_types),*) -> XcmBuilder { + let mut new_instructions = self.instructions; + new_instructions.push(#name::::#unpaid_execution_ident { #(#arg_names),* }); + XcmBuilder { + instructions: new_instructions, + state: core::marker::PhantomData, + } + } + } + }) +} + +fn get_doc_comments(variant: &Variant) -> Vec { + variant + .attrs + .iter() + .filter_map(|attr| match &attr.meta { + Meta::NameValue(MetaNameValue { + value: Expr::Lit(ExprLit { lit: Lit::Str(literal), .. }), + .. + }) if attr.path().is_ident("doc") => Some(literal.value()), + _ => None, + }) + .map(|doc| syn::parse_str::(&format!("/// {}", doc)).unwrap()) + .collect() +} diff --git a/polkadot/xcm/procedural/src/lib.rs b/polkadot/xcm/procedural/src/lib.rs index 83cc6cdf98ff..7600e817d0e6 100644 --- a/polkadot/xcm/procedural/src/lib.rs +++ b/polkadot/xcm/procedural/src/lib.rs @@ -17,6 +17,7 @@ //! Procedural macros used in XCM. use proc_macro::TokenStream; +use syn::{parse_macro_input, DeriveInput}; mod builder_pattern; mod v2; @@ -56,7 +57,10 @@ pub fn impl_conversion_functions_for_junctions_v3(input: TokenStream) -> TokenSt /// .buy_execution(fees, weight_limit) /// .deposit_asset(assets, beneficiary) /// .build(); -#[proc_macro_derive(Builder)] +#[proc_macro_derive(Builder, attributes(builder))] pub fn derive_builder(input: TokenStream) -> TokenStream { + let input = parse_macro_input!(input as DeriveInput); builder_pattern::derive(input) + .unwrap_or_else(syn::Error::into_compile_error) + .into() } diff --git a/polkadot/xcm/procedural/tests/builder_pattern.rs b/polkadot/xcm/procedural/tests/builder_pattern.rs new file mode 100644 index 000000000000..eab9d67121f6 --- /dev/null +++ b/polkadot/xcm/procedural/tests/builder_pattern.rs @@ -0,0 +1,81 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test the methods generated by the Builder derive macro. +//! Tests directly on the actual Xcm struct and Instruction enum. + +use xcm::latest::prelude::*; + +#[test] +fn builder_pattern_works() { + let asset: MultiAsset = (Here, 100u128).into(); + let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); + let message: Xcm<()> = Xcm::builder() + .receive_teleported_asset(asset.clone().into()) + .buy_execution(asset.clone(), Unlimited) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); + assert_eq!( + message, + Xcm(vec![ + ReceiveTeleportedAsset(asset.clone().into()), + BuyExecution { fees: asset.clone(), weight_limit: Unlimited }, + DepositAsset { assets: asset.into(), beneficiary }, + ]) + ); +} + +#[test] +fn default_builder_requires_buy_execution() { + let asset: MultiAsset = (Here, 100u128).into(); + let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); + // This is invalid, since it doesn't pay for fees. + // This is enforced by the runtime, because the build() method doesn't exist + // on the resulting type. + // let message: Xcm<()> = Xcm::builder() + // .withdraw_asset(asset.clone().into()) + // .deposit_asset(asset.into(), beneficiary) + // .build(); + + // To be able to do that, we need to use the explicitly unpaid variant + let message: Xcm<()> = Xcm::builder_unpaid() + .unpaid_execution(Unlimited, None) + .withdraw_asset(asset.clone().into()) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); // This works + assert_eq!( + message, + Xcm(vec![ + UnpaidExecution { weight_limit: Unlimited, check_origin: None }, + WithdrawAsset(asset.clone().into()), + DepositAsset { assets: asset.clone().into(), beneficiary }, + ]) + ); + + // The other option doesn't have any limits whatsoever, so it should + // only be used when you really know what you're doing. + let message: Xcm<()> = Xcm::builder_unsafe() + .withdraw_asset(asset.clone().into()) + .deposit_asset(asset.clone().into(), beneficiary) + .build(); + assert_eq!( + message, + Xcm(vec![ + WithdrawAsset(asset.clone().into()), + DepositAsset { assets: asset.clone().into(), beneficiary }, + ]) + ); +} diff --git a/polkadot/xcm/procedural/tests/ui.rs b/polkadot/xcm/procedural/tests/ui.rs index a6ec35d0862a..a30f4d7dee5d 100644 --- a/polkadot/xcm/procedural/tests/ui.rs +++ b/polkadot/xcm/procedural/tests/ui.rs @@ -28,5 +28,5 @@ fn ui() { std::env::set_var("SKIP_WASM_BUILD", "1"); let t = trybuild::TestCases::new(); - t.compile_fail("tests/ui/*.rs"); + t.compile_fail("tests/ui/**/*.rs"); } diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs new file mode 100644 index 000000000000..3a103f3ddc45 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when using a badly formatted attribute. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(funds_holding = 2)] + WithdrawAsset(u128), + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr new file mode 100644 index 000000000000..978faf2e868d --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/badly_formatted_attribute.stderr @@ -0,0 +1,5 @@ +error: Expected `builder(loads_holding)` + --> tests/ui/builder_pattern/badly_formatted_attribute.rs:25:5 + | +25 | #[builder(funds_holding = 2)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs new file mode 100644 index 000000000000..dc5c679a96e7 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.rs @@ -0,0 +1,30 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when the `BuyExecution` instruction doesn't take named fields. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution(u128), + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr new file mode 100644 index 000000000000..dc8246770ba3 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/buy_execution_named_fields.stderr @@ -0,0 +1,5 @@ +error: BuyExecution should have named fields + --> tests/ui/builder_pattern/buy_execution_named_fields.rs:25:5 + | +25 | BuyExecution(u128), + | ^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs new file mode 100644 index 000000000000..070f0be6bacc --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when an instruction that loads the holding register doesn't take operands. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(loads_holding)] + WithdrawAsset, + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr new file mode 100644 index 000000000000..0358a35ad3dd --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/loads_holding_no_operands.stderr @@ -0,0 +1,6 @@ +error: Instructions that load the holding register should take operands + --> tests/ui/builder_pattern/loads_holding_no_operands.rs:25:5 + | +25 | / #[builder(loads_holding)] +26 | | WithdrawAsset, + | |_________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs new file mode 100644 index 000000000000..1ed8dd38cbad --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.rs @@ -0,0 +1,29 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when there's no `BuyExecution` instruction. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr new file mode 100644 index 000000000000..d8798c8223f1 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_buy_execution.stderr @@ -0,0 +1,6 @@ +error: No BuyExecution instruction + --> tests/ui/builder_pattern/no_buy_execution.rs:25:5 + | +25 | / UnpaidExecution { weight_limit: (u32, u32) }, +26 | | Transact { call: Call }, + | |____________________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs new file mode 100644 index 000000000000..d542102d2d35 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.rs @@ -0,0 +1,29 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when there's no `UnpaidExecution` instruction. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution { fees: u128 }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr new file mode 100644 index 000000000000..c8c0748da722 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/no_unpaid_execution.stderr @@ -0,0 +1,6 @@ +error: No UnpaidExecution instruction + --> tests/ui/builder_pattern/no_unpaid_execution.rs:25:5 + | +25 | / BuyExecution { fees: u128 }, +26 | | Transact { call: Call }, + | |____________________________^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs new file mode 100644 index 000000000000..5808ec571ce7 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.rs @@ -0,0 +1,32 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when using wrong attribute. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + #[builder(funds_holding)] + WithdrawAsset(u128), + BuyExecution { fees: u128 }, + UnpaidExecution { weight_limit: (u32, u32) }, + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr new file mode 100644 index 000000000000..1ff9d1851368 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unexpected_attribute.stderr @@ -0,0 +1,5 @@ +error: Expected `builder(loads_holding)` + --> tests/ui/builder_pattern/unexpected_attribute.rs:25:5 + | +25 | #[builder(funds_holding)] + | ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs new file mode 100644 index 000000000000..bb98d603fd91 --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.rs @@ -0,0 +1,30 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +//! Test error when the `BuyExecution` instruction doesn't take named fields. + +use xcm_procedural::Builder; + +struct Xcm(pub Vec>); + +#[derive(Builder)] +enum Instruction { + BuyExecution { fees: u128 }, + UnpaidExecution(u32, u32), + Transact { call: Call }, +} + +fn main() {} diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr new file mode 100644 index 000000000000..0a3c0a40a33b --- /dev/null +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/unpaid_execution_named_fields.stderr @@ -0,0 +1,5 @@ +error: UnpaidExecution should have named fields + --> tests/ui/builder_pattern/unpaid_execution_named_fields.rs:26:5 + | +26 | UnpaidExecution(u32, u32), + | ^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern.rs b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.rs similarity index 100% rename from polkadot/xcm/procedural/tests/ui/builder_pattern.rs rename to polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.rs diff --git a/polkadot/xcm/procedural/tests/ui/builder_pattern.stderr b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr similarity index 63% rename from polkadot/xcm/procedural/tests/ui/builder_pattern.stderr rename to polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr index 439b40f31cae..007aa0b5ff30 100644 --- a/polkadot/xcm/procedural/tests/ui/builder_pattern.stderr +++ b/polkadot/xcm/procedural/tests/ui/builder_pattern/wrong_target.stderr @@ -1,5 +1,5 @@ error: Expected the `Instruction` enum - --> tests/ui/builder_pattern.rs:23:1 + --> tests/ui/builder_pattern/wrong_target.rs:23:1 | 23 | struct SomeStruct; | ^^^^^^^^^^^^^^^^^^ diff --git a/polkadot/xcm/src/v3/mod.rs b/polkadot/xcm/src/v3/mod.rs index 4217528f2273..bbdd504ceb0f 100644 --- a/polkadot/xcm/src/v3/mod.rs +++ b/polkadot/xcm/src/v3/mod.rs @@ -426,6 +426,7 @@ pub enum Instruction { /// Kind: *Command*. /// /// Errors: + #[builder(loads_holding)] WithdrawAsset(MultiAssets), /// Asset(s) (`assets`) have been received into the ownership of this system on the `origin` @@ -439,6 +440,7 @@ pub enum Instruction { /// Kind: *Trusted Indication*. /// /// Errors: + #[builder(loads_holding)] ReserveAssetDeposited(MultiAssets), /// Asset(s) (`assets`) have been destroyed on the `origin` system and equivalent assets should @@ -452,6 +454,7 @@ pub enum Instruction { /// Kind: *Trusted Indication*. /// /// Errors: + #[builder(loads_holding)] ReceiveTeleportedAsset(MultiAssets), /// Respond with information that the local system is expecting. @@ -776,6 +779,7 @@ pub enum Instruction { /// Kind: *Command* /// /// Errors: + #[builder(loads_holding)] ClaimAsset { assets: MultiAssets, ticket: MultiLocation }, /// Always throws an error of type `Trap`. diff --git a/polkadot/xcm/xcm-simulator/example/src/lib.rs b/polkadot/xcm/xcm-simulator/example/src/lib.rs index 03e7c19a9148..85b8ad1c5cb7 100644 --- a/polkadot/xcm/xcm-simulator/example/src/lib.rs +++ b/polkadot/xcm/xcm-simulator/example/src/lib.rs @@ -649,23 +649,4 @@ mod tests { ); }); } - - #[test] - fn builder_pattern_works() { - let asset: MultiAsset = (Here, 100u128).into(); - let beneficiary: MultiLocation = AccountId32 { id: [0u8; 32], network: None }.into(); - let message: Xcm<()> = Xcm::builder() - .withdraw_asset(asset.clone().into()) - .buy_execution(asset.clone(), Unlimited) - .deposit_asset(asset.clone().into(), beneficiary) - .build(); - assert_eq!( - message, - Xcm(vec![ - WithdrawAsset(asset.clone().into()), - BuyExecution { fees: asset.clone(), weight_limit: Unlimited }, - DepositAsset { assets: asset.into(), beneficiary }, - ]) - ); - } } diff --git a/prdoc/pr_2253.prdoc b/prdoc/pr_2253.prdoc new file mode 100644 index 000000000000..398b0a29066b --- /dev/null +++ b/prdoc/pr_2253.prdoc @@ -0,0 +1,24 @@ +# Schema: Parity PR Documentation Schema (prdoc) +# See doc at https://github.com/paritytech/prdoc + +title: Different builder pattern constructors for XCM + +doc: + - audience: Core Dev + description: | + The `builder()` constructor for XCM programs now only allows building messages that pay for fees, + i.e. messages that would pass the `AllowTopLevelPaidExecutionFrom` barrier. + Another constructor, `builder_unpaid()` requires an explicit `UnpaidExecution` instruction before + anything else. + For building messages without any restriction, `builder_unsafe` can be used. + This has been named like that since in general the other two should be used instead, but it's okay + to use it for teaching purposes or for experimenting. + +migrations: + db: [] + + runtime: [] + +crates: [] + +host_functions: [] From f5ad32e406e5108ea87f05a0b99406b80ed3226c Mon Sep 17 00:00:00 2001 From: Javier Bullrich Date: Tue, 21 Nov 2023 16:49:47 +0100 Subject: [PATCH 29/48] added action to pass review bot on merge queue (#2414) This action will trigger when a merge queue is created and will add a positive status check under the `review-bot` account, allowing the PR to pass the required check for the merge. --------- Co-authored-by: Chevdor --- .github/workflows/merge-queue.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/merge-queue.yml diff --git a/.github/workflows/merge-queue.yml b/.github/workflows/merge-queue.yml new file mode 100644 index 000000000000..f3fb7765ca67 --- /dev/null +++ b/.github/workflows/merge-queue.yml @@ -0,0 +1,24 @@ +name: Merge-Queue + +on: + merge_group: + +jobs: + trigger-merge-queue-action: + runs-on: ubuntu-latest + environment: master + steps: + - name: Generate token + id: app_token + uses: tibdex/github-app-token@3beb63f4bd073e61482598c45c71c1019b59b73a # v2.1.0 + with: + app_id: ${{ secrets.REVIEW_APP_ID }} + private_key: ${{ secrets.REVIEW_APP_KEY }} + - name: Add Merge Queue status check + uses: billyjbryant/create-status-check@3e6fa0ac599d10d9588cf9516ca4330ef669b858 # v2 + with: + authToken: ${{ steps.app_token.outputs.token }} + context: 'review-bot' + description: 'PRs for merge queue gets approved' + state: 'success' + sha: ${{ github.event.merge_group.head_commit.id }} From 50811d6b424185e1e6c0762a8b1c95db893c6750 Mon Sep 17 00:00:00 2001 From: Sophia Gold Date: Tue, 21 Nov 2023 11:34:05 -0500 Subject: [PATCH 30/48] Update tick collator for async backing (#1497) This updates the tick runtime and polkadot-parachain collator to use async backing. --- Cargo.lock | 1 + .../testing/rococo-parachain/Cargo.toml | 2 + .../testing/rococo-parachain/src/lib.rs | 47 ++++++++++++------- cumulus/polkadot-parachain/src/service.rs | 34 +++++++++----- 4 files changed, 55 insertions(+), 29 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 88cefe7bbef5..903e9463cf6c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14530,6 +14530,7 @@ dependencies = [ "cumulus-pallet-xcm", "cumulus-pallet-xcmp-queue", "cumulus-ping", + "cumulus-primitives-aura", "cumulus-primitives-core", "cumulus-primitives-utility", "frame-benchmarking", diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml index 5e9d347a25de..08fa8b691274 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/Cargo.toml @@ -52,6 +52,7 @@ cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system cumulus-pallet-xcm = { path = "../../../../pallets/xcm", default-features = false } cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-features = false } cumulus-ping = { path = "../../../pallets/ping", default-features = false } +cumulus-primitives-aura = { path = "../../../../primitives/aura", default-features = false } cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } cumulus-primitives-utility = { path = "../../../../primitives/utility", default-features = false } parachains-common = { path = "../../../common", default-features = false } @@ -70,6 +71,7 @@ std = [ "cumulus-pallet-xcm/std", "cumulus-pallet-xcmp-queue/std", "cumulus-ping/std", + "cumulus-primitives-aura/std", "cumulus-primitives-core/std", "cumulus-primitives-utility/std", "frame-benchmarking?/std", diff --git a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs index 6df00d43e8d3..965fb0d6adc5 100644 --- a/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs +++ b/cumulus/parachains/runtimes/testing/rococo-parachain/src/lib.rs @@ -22,13 +22,13 @@ #[cfg(feature = "std")] include!(concat!(env!("OUT_DIR"), "/wasm_binary.rs")); -use cumulus_pallet_parachain_system::RelayNumberStrictlyIncreases; +use cumulus_pallet_parachain_system::RelayNumberMonotonicallyIncreases; use polkadot_runtime_common::xcm_sender::NoPriceForMessageDelivery; use sp_api::impl_runtime_apis; use sp_core::OpaqueMetadata; use sp_runtime::{ create_runtime_str, generic, impl_opaque_keys, - traits::{AccountIdLookup, BlakeTwo256, Block as BlockT}, + traits::{AccountIdLookup, BlakeTwo256, Block as BlockT, Hash as HashT}, transaction_validity::{TransactionSource, TransactionValidity}, ApplyExtrinsicResult, }; @@ -113,7 +113,7 @@ pub const VERSION: RuntimeVersion = RuntimeVersion { state_version: 0, }; -pub const MILLISECS_PER_BLOCK: u64 = 12000; +pub const MILLISECS_PER_BLOCK: u64 = 6000; pub const SLOT_DURATION: u64 = MILLISECS_PER_BLOCK; @@ -143,18 +143,18 @@ const AVERAGE_ON_INITIALIZE_RATIO: Perbill = Perbill::from_percent(10); /// We allow `Normal` extrinsics to fill up the block up to 75%, the rest can be used /// by Operational extrinsics. const NORMAL_DISPATCH_RATIO: Perbill = Perbill::from_percent(75); -/// We allow for .5 seconds of compute with a 12 second average block time. +/// We allow for 2 seconds of compute with a 6 second average block time. const MAXIMUM_BLOCK_WEIGHT: Weight = Weight::from_parts( - WEIGHT_REF_TIME_PER_SECOND.saturating_div(2), + WEIGHT_REF_TIME_PER_SECOND.saturating_mul(2), cumulus_primitives_core::relay_chain::MAX_POV_SIZE as u64, ); /// Maximum number of blocks simultaneously accepted by the Runtime, not yet included /// into the relay chain. -const UNINCLUDED_SEGMENT_CAPACITY: u32 = 1; +const UNINCLUDED_SEGMENT_CAPACITY: u32 = 3; /// How many parachain blocks are processed by the relay chain per parent. Limits the /// number of blocks authored per slot. -const BLOCK_PROCESSING_VELOCITY: u32 = 1; +const BLOCK_PROCESSING_VELOCITY: u32 = 2; /// Relay chain slot duration, in milliseconds. const RELAY_CHAIN_SLOT_DURATION_MILLIS: u32 = 6000; @@ -277,6 +277,13 @@ parameter_types! { pub const RelayOrigin: AggregateMessageOrigin = AggregateMessageOrigin::Parent; } +type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< + Runtime, + RELAY_CHAIN_SLOT_DURATION_MILLIS, + BLOCK_PROCESSING_VELOCITY, + UNINCLUDED_SEGMENT_CAPACITY, +>; + impl cumulus_pallet_parachain_system::Config for Runtime { type WeightInfo = (); type RuntimeEvent = RuntimeEvent; @@ -287,13 +294,8 @@ impl cumulus_pallet_parachain_system::Config for Runtime { type ReservedDmpWeight = ReservedDmpWeight; type XcmpMessageHandler = XcmpQueue; type ReservedXcmpWeight = ReservedXcmpWeight; - type CheckAssociatedRelayNumber = RelayNumberStrictlyIncreases; - type ConsensusHook = cumulus_pallet_aura_ext::FixedVelocityConsensusHook< - Runtime, - RELAY_CHAIN_SLOT_DURATION_MILLIS, - BLOCK_PROCESSING_VELOCITY, - UNINCLUDED_SEGMENT_CAPACITY, - >; + type CheckAssociatedRelayNumber = RelayNumberMonotonicallyIncreases; + type ConsensusHook = ConsensusHook; } impl parachain_info::Config for Runtime {} @@ -584,9 +586,9 @@ impl pallet_aura::Config for Runtime { type AuthorityId = AuraId; type DisabledValidators = (); type MaxAuthorities = ConstU32<100_000>; - type AllowMultipleBlocksPerSlot = ConstBool; + type AllowMultipleBlocksPerSlot = ConstBool; #[cfg(feature = "experimental")] - type SlotDuration = pallet_aura::MinimumPeriodTimesTwo; + type SlotDuration = ConstU64; } construct_runtime! { @@ -624,7 +626,7 @@ pub type Balance = u128; /// Index of a transaction in the chain. pub type Nonce = u32; /// A hash of some data used by the chain. -pub type Hash = sp_core::H256; +pub type Hash = ::Output; /// An index to a block. pub type BlockNumber = u32; /// The address format for describing accounts. @@ -751,7 +753,7 @@ impl_runtime_apis! { impl sp_consensus_aura::AuraApi for Runtime { fn slot_duration() -> sp_consensus_aura::SlotDuration { - sp_consensus_aura::SlotDuration::from_millis(Aura::slot_duration()) + sp_consensus_aura::SlotDuration::from_millis(SLOT_DURATION) } fn authorities() -> Vec { @@ -824,6 +826,15 @@ impl_runtime_apis! { build_config::(config) } } + + impl cumulus_primitives_aura::AuraUnincludedSegmentApi for Runtime { + fn can_build_upon( + included_hash: ::Hash, + slot: cumulus_primitives_aura::Slot, + ) -> bool { + ConsensusHook::can_build_upon(included_hash, slot) + } + } } cumulus_pallet_parachain_system::register_validate_block! { diff --git a/cumulus/polkadot-parachain/src/service.rs b/cumulus/polkadot-parachain/src/service.rs index 3884fce246c3..f580835db55a 100644 --- a/cumulus/polkadot-parachain/src/service.rs +++ b/cumulus/polkadot-parachain/src/service.rs @@ -590,6 +590,7 @@ where CollatorPair, OverseerHandle, Arc>) + Send + Sync>, + Arc, ) -> Result<(), sc_service::Error>, { let parachain_config = prepare_node_config(parachain_config); @@ -723,6 +724,7 @@ where collator_key.expect("Command line arguments do not allow this. qed"), overseer_handle, announce_block, + backend.clone(), )?; } @@ -983,7 +985,8 @@ pub async fn start_rococo_parachain_node( para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( @@ -1002,11 +1005,15 @@ pub async fn start_rococo_parachain_node( client.clone(), ); - let params = BasicAuraParams { + let params = AuraParams { create_inherent_data_providers: move |_, ()| async move { Ok(()) }, block_import, - para_client: client, + para_client: client.clone(), + para_backend: backend.clone(), relay_client: relay_chain_interface, + code_hash_provider: move |block_hash| { + client.code_at(block_hash).ok().map(|c| ValidationCode::from(c).hash()) + }, sync_oracle, keystore, collator_key, @@ -1016,12 +1023,10 @@ pub async fn start_rococo_parachain_node( relay_chain_slot_duration, proposer, collator_service, - // Very limited proposal time. - authoring_duration: Duration::from_millis(500), - collation_request_receiver: None, + authoring_duration: Duration::from_millis(1500), }; - let fut = basic_aura::run::< + let fut = aura::run::< Block, sp_consensus_aura::sr25519::AuthorityPair, _, @@ -1031,6 +1036,8 @@ pub async fn start_rococo_parachain_node( _, _, _, + _, + _, >(params); task_manager.spawn_essential_handle().spawn("aura", None, fut); @@ -1376,7 +1383,8 @@ where para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( @@ -1471,7 +1479,8 @@ where para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let relay_chain_interface2 = relay_chain_interface.clone(); let collator_service = CollatorService::new( @@ -1642,7 +1651,7 @@ where para_backend: backend.clone(), relay_client: relay_chain_interface, code_hash_provider: move |block_hash| { - client.code_at(block_hash).ok().map(ValidationCode).map(|c| c.hash()) + client.code_at(block_hash).ok().map(|c| ValidationCode::from(c).hash()) }, sync_oracle, keystore, @@ -1713,6 +1722,7 @@ where CollatorPair, OverseerHandle, Arc>) + Send + Sync>, + Arc, ) -> Result<(), sc_service::Error>, { let parachain_config = prepare_node_config(parachain_config); @@ -1845,6 +1855,7 @@ where collator_key.expect("Command line arguments do not allow this. qed"), overseer_handle, announce_block, + backend.clone(), )?; } @@ -1923,7 +1934,8 @@ pub async fn start_contracts_rococo_node( para_id, collator_key, overseer_handle, - announce_block| { + announce_block, + _backend| { let slot_duration = cumulus_client_consensus_aura::slot_duration(&*client)?; let proposer_factory = sc_basic_authorship::ProposerFactory::with_proof_recording( From 2183669d05f9b510f979a0cc3c7847707bacba2e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Tue, 21 Nov 2023 20:03:55 +0100 Subject: [PATCH 31/48] cumulus-test-service: block import fix (#2430) This is follow-up for: https://github.com/paritytech/polkadot-sdk/pull/2001 Block import queue for `test-parachain` (`cumulus-test-service`) shall use delayed best block feature. This should fixed broken zombienet tests. --- cumulus/test/service/src/lib.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cumulus/test/service/src/lib.rs b/cumulus/test/service/src/lib.rs index 6fd3e4d43d75..4d4c60d1bda6 100644 --- a/cumulus/test/service/src/lib.rs +++ b/cumulus/test/service/src/lib.rs @@ -211,7 +211,8 @@ pub fn new_partial( sc_service::new_full_parts::(config, None, executor)?; let client = Arc::new(client); - let block_import = ParachainBlockImport::new(client.clone(), backend.clone()); + let block_import = + ParachainBlockImport::new_with_delayed_best_block(client.clone(), backend.clone()); let registry = config.prometheus_registry(); From 7a32f4be48e514eccfbe5f8d2c2366c049cc4e3d Mon Sep 17 00:00:00 2001 From: Ross Bulat Date: Wed, 22 Nov 2023 16:22:28 +0700 Subject: [PATCH 32/48] Deprecate `RewardDestination::Controller` (#2380) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deprecates `RewardDestination::Controller` variant. - [x] `RewardDestination::Controller` annotated with `#[deprecated]`. - [x] `Controller` variant is now handled the same way as `Stash` in `payout_stakers`. - [x] `set_payee` errors if `RewardDestination::Controller` is provided. - [x] Added `update_payee` call to lazily migrate `RewardDestination::Controller` `Payee` storage entries to `RewardDestination::Account(controller)` . - [x] `payout_stakers_dead_controller` has been removed from benches & weights - was not used. - [x] Tests no longer use `RewardDestination::Controller`. --------- Co-authored-by: command-bot <> Co-authored-by: Gonçalo Pestana Co-authored-by: georgepisaltu <52418509+georgepisaltu@users.noreply.github.com> --- .../westend/src/weights/pallet_staking.rs | 238 ++++----- substrate/frame/staking/src/benchmarking.rs | 54 +- substrate/frame/staking/src/lib.rs | 6 +- substrate/frame/staking/src/mock.rs | 2 +- substrate/frame/staking/src/pallet/impls.rs | 13 +- substrate/frame/staking/src/pallet/mod.rs | 45 +- substrate/frame/staking/src/tests.rs | 203 ++++---- substrate/frame/staking/src/weights.rs | 468 ++++++++---------- 8 files changed, 475 insertions(+), 554 deletions(-) diff --git a/polkadot/runtime/westend/src/weights/pallet_staking.rs b/polkadot/runtime/westend/src/weights/pallet_staking.rs index 3c4542c6d6fe..87b603621e8d 100644 --- a/polkadot/runtime/westend/src/weights/pallet_staking.rs +++ b/polkadot/runtime/westend/src/weights/pallet_staking.rs @@ -17,7 +17,7 @@ //! Autogenerated weights for `pallet_staking` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-10-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-11-21, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` //! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("westend-dev")`, DB CACHE: 1024 @@ -62,8 +62,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `894` // Estimated: `4764` - // Minimum execution time: 39_950_000 picoseconds. - Weight::from_parts(41_107_000, 0) + // Minimum execution time: 38_052_000 picoseconds. + Weight::from_parts(39_303_000, 0) .saturating_add(Weight::from_parts(0, 4764)) .saturating_add(T::DbWeight::get().reads(3)) .saturating_add(T::DbWeight::get().writes(4)) @@ -84,8 +84,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1921` // Estimated: `8877` - // Minimum execution time: 83_828_000 picoseconds. - Weight::from_parts(85_733_000, 0) + // Minimum execution time: 81_690_000 picoseconds. + Weight::from_parts(83_889_000, 0) .saturating_add(Weight::from_parts(0, 8877)) .saturating_add(T::DbWeight::get().reads(9)) .saturating_add(T::DbWeight::get().writes(7)) @@ -112,8 +112,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `2128` // Estimated: `8877` - // Minimum execution time: 89_002_000 picoseconds. - Weight::from_parts(91_556_000, 0) + // Minimum execution time: 84_409_000 picoseconds. + Weight::from_parts(87_330_000, 0) .saturating_add(Weight::from_parts(0, 8877)) .saturating_add(T::DbWeight::get().reads(12)) .saturating_add(T::DbWeight::get().writes(7)) @@ -133,11 +133,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1075` // Estimated: `4764` - // Minimum execution time: 40_839_000 picoseconds. - Weight::from_parts(42_122_428, 0) + // Minimum execution time: 39_770_000 picoseconds. + Weight::from_parts(40_828_632, 0) .saturating_add(Weight::from_parts(0, 4764)) - // Standard Error: 884 - .saturating_add(Weight::from_parts(46_036, 0).saturating_mul(s.into())) + // Standard Error: 824 + .saturating_add(Weight::from_parts(51_107, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(5)) .saturating_add(T::DbWeight::get().writes(2)) } @@ -174,11 +174,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `2127 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 84_244_000 picoseconds. - Weight::from_parts(91_199_964, 0) + // Minimum execution time: 82_500_000 picoseconds. + Weight::from_parts(90_099_121, 0) .saturating_add(Weight::from_parts(0, 6248)) - // Standard Error: 3_381 - .saturating_add(Weight::from_parts(1_327_289, 0).saturating_mul(s.into())) + // Standard Error: 3_280 + .saturating_add(Weight::from_parts(1_273_212, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(13)) .saturating_add(T::DbWeight::get().writes(11)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -210,8 +210,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1301` // Estimated: `4556` - // Minimum execution time: 49_693_000 picoseconds. - Weight::from_parts(50_814_000, 0) + // Minimum execution time: 48_236_000 picoseconds. + Weight::from_parts(49_518_000, 0) .saturating_add(Weight::from_parts(0, 4556)) .saturating_add(T::DbWeight::get().reads(11)) .saturating_add(T::DbWeight::get().writes(5)) @@ -225,11 +225,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1243 + k * (569 ±0)` // Estimated: `4556 + k * (3033 ±0)` - // Minimum execution time: 29_140_000 picoseconds. - Weight::from_parts(28_309_627, 0) + // Minimum execution time: 28_280_000 picoseconds. + Weight::from_parts(29_182_740, 0) .saturating_add(Weight::from_parts(0, 4556)) - // Standard Error: 5_780 - .saturating_add(Weight::from_parts(6_509_869, 0).saturating_mul(k.into())) + // Standard Error: 6_102 + .saturating_add(Weight::from_parts(6_412_107, 0).saturating_mul(k.into())) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(k.into()))) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(k.into()))) @@ -262,11 +262,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1797 + n * (102 ±0)` // Estimated: `6248 + n * (2520 ±0)` - // Minimum execution time: 61_377_000 picoseconds. - Weight::from_parts(58_805_232, 0) + // Minimum execution time: 59_846_000 picoseconds. + Weight::from_parts(58_029_857, 0) .saturating_add(Weight::from_parts(0, 6248)) - // Standard Error: 14_197 - .saturating_add(Weight::from_parts(4_090_197, 0).saturating_mul(n.into())) + // Standard Error: 15_967 + .saturating_add(Weight::from_parts(3_898_764, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(12)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes(6)) @@ -290,8 +290,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1581` // Estimated: `6248` - // Minimum execution time: 52_736_000 picoseconds. - Weight::from_parts(54_573_000, 0) + // Minimum execution time: 51_223_000 picoseconds. + Weight::from_parts(52_310_000, 0) .saturating_add(Weight::from_parts(0, 6248)) .saturating_add(T::DbWeight::get().reads(8)) .saturating_add(T::DbWeight::get().writes(6)) @@ -306,12 +306,28 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `865` // Estimated: `4556` - // Minimum execution time: 16_496_000 picoseconds. - Weight::from_parts(17_045_000, 0) + // Minimum execution time: 15_762_000 picoseconds. + Weight::from_parts(16_381_000, 0) .saturating_add(Weight::from_parts(0, 4556)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) } + /// Storage: `Staking::Ledger` (r:1 w:0) + /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) + /// Storage: `Staking::Payee` (r:1 w:1) + /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) + /// Storage: `Staking::Bonded` (r:1 w:0) + /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) + fn update_payee() -> Weight { + // Proof Size summary in bytes: + // Measured: `932` + // Estimated: `4556` + // Minimum execution time: 21_904_000 picoseconds. + Weight::from_parts(22_373_000, 0) + .saturating_add(Weight::from_parts(0, 4556)) + .saturating_add(T::DbWeight::get().reads(3)) + .saturating_add(T::DbWeight::get().writes(1)) + } /// Storage: `Staking::Bonded` (r:1 w:1) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:1 w:2) @@ -320,8 +336,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `865` // Estimated: `4556` - // Minimum execution time: 19_339_000 picoseconds. - Weight::from_parts(20_187_000, 0) + // Minimum execution time: 18_869_000 picoseconds. + Weight::from_parts(19_422_000, 0) .saturating_add(Weight::from_parts(0, 4556)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(3)) @@ -332,8 +348,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_340_000 picoseconds. - Weight::from_parts(2_551_000, 0) + // Minimum execution time: 2_205_000 picoseconds. + Weight::from_parts(2_320_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -343,8 +359,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 7_483_000 picoseconds. - Weight::from_parts(8_101_000, 0) + // Minimum execution time: 7_179_000 picoseconds. + Weight::from_parts(7_843_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -354,8 +370,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 7_773_000 picoseconds. - Weight::from_parts(8_610_000, 0) + // Minimum execution time: 7_206_000 picoseconds. + Weight::from_parts(7_829_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -365,8 +381,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 7_577_000 picoseconds. - Weight::from_parts(7_937_000, 0) + // Minimum execution time: 7_414_000 picoseconds. + Weight::from_parts(7_770_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } @@ -377,11 +393,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_522_000 picoseconds. - Weight::from_parts(2_735_307, 0) + // Minimum execution time: 2_256_000 picoseconds. + Weight::from_parts(2_645_840, 0) .saturating_add(Weight::from_parts(0, 0)) - // Standard Error: 38 - .saturating_add(Weight::from_parts(10_553, 0).saturating_mul(v.into())) + // Standard Error: 37 + .saturating_add(Weight::from_parts(10_207, 0).saturating_mul(v.into())) .saturating_add(T::DbWeight::get().writes(1)) } /// Storage: `Staking::SlashingSpans` (r:1 w:1) @@ -417,11 +433,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `2127 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 82_547_000 picoseconds. - Weight::from_parts(89_373_781, 0) + // Minimum execution time: 81_032_000 picoseconds. + Weight::from_parts(88_297_596, 0) .saturating_add(Weight::from_parts(0, 6248)) - // Standard Error: 3_589 - .saturating_add(Weight::from_parts(1_258_878, 0).saturating_mul(s.into())) + // Standard Error: 3_070 + .saturating_add(Weight::from_parts(1_207_207, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(13)) .saturating_add(T::DbWeight::get().writes(12)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -434,56 +450,14 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `66639` // Estimated: `70104` - // Minimum execution time: 134_619_000 picoseconds. - Weight::from_parts(1_194_949_665, 0) + // Minimum execution time: 131_456_000 picoseconds. + Weight::from_parts(935_254_517, 0) .saturating_add(Weight::from_parts(0, 70104)) - // Standard Error: 76_719 - .saturating_add(Weight::from_parts(6_455_953, 0).saturating_mul(s.into())) + // Standard Error: 57_806 + .saturating_add(Weight::from_parts(4_823_189, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(1)) .saturating_add(T::DbWeight::get().writes(1)) } - /// Storage: `Staking::CurrentEra` (r:1 w:0) - /// Proof: `Staking::CurrentEra` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasStakersOverview` (r:1 w:0) - /// Proof: `Staking::ErasStakersOverview` (`max_values`: None, `max_size`: Some(92), added: 2567, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasValidatorReward` (r:1 w:0) - /// Proof: `Staking::ErasValidatorReward` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) - /// Storage: `Staking::Bonded` (r:65 w:0) - /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) - /// Storage: `Staking::Ledger` (r:1 w:1) - /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) - /// Storage: `Balances::Locks` (r:1 w:1) - /// Proof: `Balances::Locks` (`max_values`: None, `max_size`: Some(1299), added: 3774, mode: `MaxEncodedLen`) - /// Storage: `Balances::Freezes` (r:1 w:0) - /// Proof: `Balances::Freezes` (`max_values`: None, `max_size`: Some(67), added: 2542, mode: `MaxEncodedLen`) - /// Storage: `System::Account` (r:66 w:66) - /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) - /// Storage: `Staking::ClaimedRewards` (r:1 w:1) - /// Proof: `Staking::ClaimedRewards` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasStakersPaged` (r:1 w:0) - /// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasRewardPoints` (r:1 w:0) - /// Proof: `Staking::ErasRewardPoints` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasValidatorPrefs` (r:1 w:0) - /// Proof: `Staking::ErasValidatorPrefs` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`) - /// Storage: `Staking::Payee` (r:65 w:0) - /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) - /// The range of component `n` is `[0, 64]`. - fn payout_stakers_dead_controller(n: u32, ) -> Weight { - // Proof Size summary in bytes: - // Measured: `6895 + n * (156 ±0)` - // Estimated: `9802 + n * (2603 ±0)` - // Minimum execution time: 114_338_000 picoseconds. - Weight::from_parts(138_518_124, 0) - .saturating_add(Weight::from_parts(0, 9802)) - // Standard Error: 53_621 - .saturating_add(Weight::from_parts(25_676_781, 0).saturating_mul(n.into())) - .saturating_add(T::DbWeight::get().reads(14)) - .saturating_add(T::DbWeight::get().reads((3_u64).saturating_mul(n.into()))) - .saturating_add(T::DbWeight::get().writes(5)) - .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) - .saturating_add(Weight::from_parts(0, 2603).saturating_mul(n.into())) - } /// Storage: `Staking::Bonded` (r:65 w:0) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:65 w:65) @@ -516,12 +490,12 @@ impl pallet_staking::WeightInfo for WeightInfo { fn payout_stakers_alive_staked(n: u32, ) -> Weight { // Proof Size summary in bytes: // Measured: `8249 + n * (396 ±0)` - // Estimated: `10779 + n * (3774 ±3)` - // Minimum execution time: 132_719_000 picoseconds. - Weight::from_parts(170_505_880, 0) + // Estimated: `10779 + n * (3774 ±0)` + // Minimum execution time: 129_233_000 picoseconds. + Weight::from_parts(165_096_042, 0) .saturating_add(Weight::from_parts(0, 10779)) - // Standard Error: 32_527 - .saturating_add(Weight::from_parts(42_453_136, 0).saturating_mul(n.into())) + // Standard Error: 29_598 + .saturating_add(Weight::from_parts(40_716_425, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(14)) .saturating_add(T::DbWeight::get().reads((6_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes(4)) @@ -545,11 +519,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1922 + l * (5 ±0)` // Estimated: `8877` - // Minimum execution time: 78_438_000 picoseconds. - Weight::from_parts(81_774_734, 0) + // Minimum execution time: 77_223_000 picoseconds. + Weight::from_parts(80_026_259, 0) .saturating_add(Weight::from_parts(0, 8877)) - // Standard Error: 3_706 - .saturating_add(Weight::from_parts(51_358, 0).saturating_mul(l.into())) + // Standard Error: 4_493 + .saturating_add(Weight::from_parts(52_909, 0).saturating_mul(l.into())) .saturating_add(T::DbWeight::get().reads(9)) .saturating_add(T::DbWeight::get().writes(7)) } @@ -584,11 +558,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `2127 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 92_129_000 picoseconds. - Weight::from_parts(94_137_611, 0) + // Minimum execution time: 89_871_000 picoseconds. + Weight::from_parts(92_313_331, 0) .saturating_add(Weight::from_parts(0, 6248)) - // Standard Error: 4_141 - .saturating_add(Weight::from_parts(1_283_823, 0).saturating_mul(s.into())) + // Standard Error: 3_321 + .saturating_add(Weight::from_parts(1_243_347, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(12)) .saturating_add(T::DbWeight::get().writes(11)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -634,13 +608,13 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0 + n * (716 ±0) + v * (3594 ±0)` // Estimated: `456136 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 527_896_000 picoseconds. - Weight::from_parts(533_325_000, 0) + // Minimum execution time: 518_819_000 picoseconds. + Weight::from_parts(522_108_000, 0) .saturating_add(Weight::from_parts(0, 456136)) - // Standard Error: 2_064_813 - .saturating_add(Weight::from_parts(68_484_503, 0).saturating_mul(v.into())) - // Standard Error: 205_747 - .saturating_add(Weight::from_parts(18_833_735, 0).saturating_mul(n.into())) + // Standard Error: 1_987_848 + .saturating_add(Weight::from_parts(64_855_377, 0).saturating_mul(v.into())) + // Standard Error: 198_078 + .saturating_add(Weight::from_parts(18_343_485, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(184)) .saturating_add(T::DbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(T::DbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -671,13 +645,13 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `3108 + n * (907 ±0) + v * (391 ±0)` // Estimated: `456136 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 35_302_472_000 picoseconds. - Weight::from_parts(35_651_169_000, 0) + // Minimum execution time: 34_976_277_000 picoseconds. + Weight::from_parts(35_245_501_000, 0) .saturating_add(Weight::from_parts(0, 456136)) - // Standard Error: 412_098 - .saturating_add(Weight::from_parts(5_172_265, 0).saturating_mul(v.into())) - // Standard Error: 412_098 - .saturating_add(Weight::from_parts(4_142_772, 0).saturating_mul(n.into())) + // Standard Error: 386_461 + .saturating_add(Weight::from_parts(5_145_210, 0).saturating_mul(v.into())) + // Standard Error: 386_461 + .saturating_add(Weight::from_parts(3_762_623, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(179)) .saturating_add(T::DbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(T::DbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -694,11 +668,11 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `946 + v * (50 ±0)` // Estimated: `3510 + v * (2520 ±0)` - // Minimum execution time: 2_522_650_000 picoseconds. - Weight::from_parts(97_022_833, 0) + // Minimum execution time: 2_577_411_000 picoseconds. + Weight::from_parts(86_073_486, 0) .saturating_add(Weight::from_parts(0, 3510)) - // Standard Error: 6_751 - .saturating_add(Weight::from_parts(4_990_018, 0).saturating_mul(v.into())) + // Standard Error: 8_363 + .saturating_add(Weight::from_parts(5_074_828, 0).saturating_mul(v.into())) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(v.into()))) .saturating_add(Weight::from_parts(0, 2520).saturating_mul(v.into())) @@ -719,8 +693,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_833_000 picoseconds. - Weight::from_parts(4_108_000, 0) + // Minimum execution time: 3_539_000 picoseconds. + Weight::from_parts(3_903_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(6)) } @@ -740,8 +714,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_520_000 picoseconds. - Weight::from_parts(3_686_000, 0) + // Minimum execution time: 3_244_000 picoseconds. + Weight::from_parts(3_450_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(6)) } @@ -769,8 +743,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `1704` // Estimated: `6248` - // Minimum execution time: 63_983_000 picoseconds. - Weight::from_parts(66_140_000, 0) + // Minimum execution time: 62_606_000 picoseconds. + Weight::from_parts(64_678_000, 0) .saturating_add(Weight::from_parts(0, 6248)) .saturating_add(T::DbWeight::get().reads(11)) .saturating_add(T::DbWeight::get().writes(6)) @@ -783,8 +757,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `658` // Estimated: `3510` - // Minimum execution time: 11_830_000 picoseconds. - Weight::from_parts(12_210_000, 0) + // Minimum execution time: 11_490_000 picoseconds. + Weight::from_parts(11_867_000, 0) .saturating_add(Weight::from_parts(0, 3510)) .saturating_add(T::DbWeight::get().reads(2)) .saturating_add(T::DbWeight::get().writes(1)) @@ -795,8 +769,8 @@ impl pallet_staking::WeightInfo for WeightInfo { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_364_000 picoseconds. - Weight::from_parts(2_555_000, 0) + // Minimum execution time: 2_125_000 picoseconds. + Weight::from_parts(2_337_000, 0) .saturating_add(Weight::from_parts(0, 0)) .saturating_add(T::DbWeight::get().writes(1)) } diff --git a/substrate/frame/staking/src/benchmarking.rs b/substrate/frame/staking/src/benchmarking.rs index 05c6bc397097..6f60c4909f4d 100644 --- a/substrate/frame/staking/src/benchmarking.rs +++ b/substrate/frame/staking/src/benchmarking.rs @@ -464,16 +464,28 @@ benchmarks! { } set_payee { - let (stash, controller) = create_stash_controller::(USER_SEED, 100, Default::default())?; + let (stash, controller) = create_stash_controller::(USER_SEED, 100, RewardDestination::Staked)?; assert_eq!(Payee::::get(&stash), RewardDestination::Staked); whitelist_account!(controller); - }: _(RawOrigin::Signed(controller), RewardDestination::Controller) + }: _(RawOrigin::Signed(controller.clone()), RewardDestination::Account(controller.clone())) verify { - assert_eq!(Payee::::get(&stash), RewardDestination::Controller); + assert_eq!(Payee::::get(&stash), RewardDestination::Account(controller)); + } + + update_payee { + let (stash, controller) = create_stash_controller::(USER_SEED, 100, RewardDestination::Staked)?; + Payee::::insert(&stash, { + #[allow(deprecated)] + RewardDestination::Controller + }); + whitelist_account!(controller); + }: _(RawOrigin::Signed(controller.clone()), controller.clone()) + verify { + assert_eq!(Payee::::get(&stash), RewardDestination::Account(controller)); } set_controller { - let (stash, ctlr) = create_unique_stash_controller::(9000, 100, Default::default(), false)?; + let (stash, ctlr) = create_unique_stash_controller::(9000, 100, RewardDestination::Staked, false)?; // ensure `ctlr` is the currently stored controller. assert!(!Ledger::::contains_key(&stash)); assert!(Ledger::::contains_key(&ctlr)); @@ -551,40 +563,6 @@ benchmarks! { assert_eq!(UnappliedSlashes::::get(&era).len(), (MAX_SLASHES - s) as usize); } - payout_stakers_dead_controller { - let n in 0 .. T::MaxExposurePageSize::get() as u32; - let (validator, nominators) = create_validator_with_nominators::( - n, - T::MaxExposurePageSize::get() as u32, - true, - true, - RewardDestination::Controller, - )?; - - let current_era = CurrentEra::::get().unwrap(); - // set the commission for this particular era as well. - >::insert(current_era, validator.clone(), >::validators(&validator)); - - let caller = whitelisted_caller(); - let validator_controller = >::get(&validator).unwrap(); - let balance_before = T::Currency::free_balance(&validator_controller); - for (_, controller) in &nominators { - let balance = T::Currency::free_balance(controller); - ensure!(balance.is_zero(), "Controller has balance, but should be dead."); - } - }: payout_stakers_by_page(RawOrigin::Signed(caller), validator, current_era, 0) - verify { - let balance_after = T::Currency::free_balance(&validator_controller); - ensure!( - balance_before < balance_after, - "Balance of validator controller should have increased after payout.", - ); - for (_, controller) in &nominators { - let balance = T::Currency::free_balance(controller); - ensure!(!balance.is_zero(), "Payout not given to controller."); - } - } - payout_stakers_alive_staked { let n in 0 .. T::MaxExposurePageSize::get() as u32; let (validator, nominators) = create_validator_with_nominators::( diff --git a/substrate/frame/staking/src/lib.rs b/substrate/frame/staking/src/lib.rs index 9e4697e845b6..2cfee38ab4fa 100644 --- a/substrate/frame/staking/src/lib.rs +++ b/substrate/frame/staking/src/lib.rs @@ -240,9 +240,9 @@ //! [`Payee`] storage item (see //! [`set_payee`](Call::set_payee)), to be one of the following: //! -//! - Controller account, (obviously) not increasing the staked value. //! - Stash account, not increasing the staked value. //! - Stash account, also increasing the staked value. +//! - Any other account, sent as free balance. //! //! ### Additional Fund Management Operations //! @@ -404,7 +404,9 @@ pub enum RewardDestination { Staked, /// Pay into the stash account, not increasing the amount at stake. Stash, - /// Pay into the controller account. + #[deprecated( + note = "`Controller` will be removed after January 2024. Use `Account(controller)` instead. This variant now behaves the same as `Stash` variant." + )] Controller, /// Pay into a specified account. Account(AccountId), diff --git a/substrate/frame/staking/src/mock.rs b/substrate/frame/staking/src/mock.rs index d2afd8f26e24..364d125792b8 100644 --- a/substrate/frame/staking/src/mock.rs +++ b/substrate/frame/staking/src/mock.rs @@ -594,7 +594,7 @@ pub(crate) fn current_era() -> EraIndex { pub(crate) fn bond(who: AccountId, val: Balance) { let _ = Balances::make_free_balance_be(&who, val); - assert_ok!(Staking::bond(RuntimeOrigin::signed(who), val, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(who), val, RewardDestination::Stash)); } pub(crate) fn bond_validator(who: AccountId, val: Balance) { diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index 40f30735258f..6ea8e4c9d3b2 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -338,9 +338,8 @@ impl Pallet { let dest = Self::payee(StakingAccount::Stash(stash.clone())); let maybe_imbalance = match dest { - RewardDestination::Controller => Self::bonded(stash) - .map(|controller| T::Currency::deposit_creating(&controller, amount)), - RewardDestination::Stash => T::Currency::deposit_into_existing(stash, amount).ok(), + RewardDestination::Stash => + T::Currency::deposit_into_existing(stash, amount).ok(), RewardDestination::Staked => Self::ledger(Stash(stash.clone())) .and_then(|mut ledger| { ledger.active += amount; @@ -357,6 +356,14 @@ impl Pallet { RewardDestination::Account(dest_account) => Some(T::Currency::deposit_creating(&dest_account, amount)), RewardDestination::None => None, + #[allow(deprecated)] + RewardDestination::Controller => Self::bonded(stash) + .map(|controller| { + defensive!("Paying out controller as reward destination which is deprecated and should be migrated"); + // This should never happen once payees with a `Controller` variant have been migrated. + // But if it does, just pay the controller account. + T::Currency::deposit_creating(&controller, amount) + }), }; maybe_imbalance .map(|imbalance| (imbalance, Self::payee(StakingAccount::Stash(stash.clone())))) diff --git a/substrate/frame/staking/src/pallet/mod.rs b/substrate/frame/staking/src/pallet/mod.rs index 18ad3e4a6cf1..a68e9c90da9e 100644 --- a/substrate/frame/staking/src/pallet/mod.rs +++ b/substrate/frame/staking/src/pallet/mod.rs @@ -842,6 +842,8 @@ pub mod pallet { CommissionTooLow, /// Some bound is not met. BoundNotMet, + /// Used when attempting to use deprecated controller account logic. + ControllerDeprecated, } #[pallet::hooks] @@ -1283,10 +1285,19 @@ pub mod pallet { payee: RewardDestination, ) -> DispatchResult { let controller = ensure_signed(origin)?; - let ledger = Self::ledger(Controller(controller))?; + let ledger = Self::ledger(Controller(controller.clone()))?; + + ensure!( + (payee != { + #[allow(deprecated)] + RewardDestination::Controller + }), + Error::::ControllerDeprecated + ); + let _ = ledger .set_payee(payee) - .defensive_proof("ledger was retrieved from storage, thus its bonded; qed."); + .defensive_proof("ledger was retrieved from storage, thus its bonded; qed.")?; Ok(()) } @@ -1872,6 +1883,36 @@ pub mod pallet { ensure_signed(origin)?; Self::do_payout_stakers_by_page(validator_stash, era, page) } + + /// Migrates an account's `RewardDestination::Controller` to + /// `RewardDestination::Account(controller)`. + /// + /// Effects will be felt instantly (as soon as this function is completed successfully). + /// + /// This will waive the transaction fee if the `payee` is successfully migrated. + #[pallet::call_index(27)] + #[pallet::weight(T::WeightInfo::update_payee())] + pub fn update_payee( + origin: OriginFor, + controller: T::AccountId, + ) -> DispatchResultWithPostInfo { + let _ = ensure_signed(origin)?; + let ledger = Self::ledger(StakingAccount::Controller(controller.clone()))?; + + ensure!( + (Payee::::get(&ledger.stash) == { + #[allow(deprecated)] + RewardDestination::Controller + }), + Error::::NotController + ); + + let _ = ledger + .set_payee(RewardDestination::Account(controller)) + .defensive_proof("ledger should have been previously retrieved from storage.")?; + + Ok(Pays::No.into()) + } } } diff --git a/substrate/frame/staking/src/tests.rs b/substrate/frame/staking/src/tests.rs index bac2530b19bb..7d967609f520 100644 --- a/substrate/frame/staking/src/tests.rs +++ b/substrate/frame/staking/src/tests.rs @@ -298,9 +298,9 @@ fn rewards_should_work() { let init_balance_101 = Balances::total_balance(&101); // Set payees - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(21, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); + Payee::::insert(11, RewardDestination::Account(11)); + Payee::::insert(21, RewardDestination::Account(21)); + Payee::::insert(101, RewardDestination::Account(101)); Pallet::::reward_by_ids(vec![(11, 50)]); Pallet::::reward_by_ids(vec![(11, 50)]); @@ -417,7 +417,7 @@ fn staking_should_work() { // --- Block 2: start_session(2); // add a new candidate for being a validator. account 3 controlled by 4. - assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 1500, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 1500, RewardDestination::Account(3))); assert_ok!(Staking::validate(RuntimeOrigin::signed(3), ValidatorPrefs::default())); assert_ok!(Session::set_keys( RuntimeOrigin::signed(3), @@ -585,22 +585,10 @@ fn nominating_and_rewards_should_work() { assert_ok!(Staking::validate(RuntimeOrigin::signed(31), Default::default())); // Set payee to controller. - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(11), - RewardDestination::Controller - )); - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(21), - RewardDestination::Controller - )); - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(31), - RewardDestination::Controller - )); - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(41), - RewardDestination::Controller - )); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(21), RewardDestination::Stash)); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(31), RewardDestination::Stash)); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(41), RewardDestination::Stash)); // give the man some money let initial_balance = 1000; @@ -612,14 +600,14 @@ fn nominating_and_rewards_should_work() { assert_ok!(Staking::bond( RuntimeOrigin::signed(1), 1000, - RewardDestination::Controller + RewardDestination::Account(1) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(1), vec![11, 21, 31])); assert_ok!(Staking::bond( RuntimeOrigin::signed(3), 1000, - RewardDestination::Controller + RewardDestination::Account(3) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(3), vec![11, 21, 41])); @@ -1116,8 +1104,8 @@ fn reward_destination_works() { // (era 1, page 0) is claimed assert_eq!(Staking::claimed_rewards(1, &11), vec![0]); - // Change RewardDestination to Controller - >::insert(&11, RewardDestination::Controller); + // Change RewardDestination to Account + >::insert(&11, RewardDestination::Account(11)); // Check controller balance assert_eq!(Balances::free_balance(11), 23150); @@ -1129,8 +1117,8 @@ fn reward_destination_works() { mock::start_active_era(3); mock::make_all_reward_payment(2); - // Check that RewardDestination is Controller - assert_eq!(Staking::payee(11.into()), RewardDestination::Controller); + // Check that RewardDestination is Account(11) + assert_eq!(Staking::payee(11.into()), RewardDestination::Account(11)); // Check that reward went to the controller account assert_eq!(Balances::free_balance(11), recorded_stash_balance + total_payout_2); // Check that amount at stake is NOT increased @@ -1159,9 +1147,9 @@ fn validator_payment_prefs_work() { let commission = Perbill::from_percent(40); >::insert(&11, ValidatorPrefs { commission, ..Default::default() }); - // Reward controller so staked ratio doesn't change. - >::insert(&11, RewardDestination::Controller); - >::insert(&101, RewardDestination::Controller); + // Reward stash so staked ratio doesn't change. + >::insert(&11, RewardDestination::Stash); + >::insert(&101, RewardDestination::Stash); mock::start_active_era(1); mock::make_all_reward_payment(0); @@ -1250,8 +1238,8 @@ fn bond_extra_and_withdraw_unbonded_works() { // * it can unbond a portion of its funds from the stash account. // * Once the unbonding period is done, it can actually take the funds out of the stash. ExtBuilder::default().nominate(false).build_and_execute(|| { - // Set payee to controller. avoids confusion - assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Controller)); + // Set payee to stash. + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); // Give account 11 some large free balance greater than total let _ = Balances::make_free_balance_be(&11, 1000000); @@ -1461,8 +1449,8 @@ fn rebond_works() { // * it can unbond a portion of its funds from the stash account. // * it can re-bond a portion of the funds scheduled to unlock. ExtBuilder::default().nominate(false).build_and_execute(|| { - // Set payee to controller. avoids confusion - assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Controller)); + // Set payee to stash. + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); // Give account 11 some large free balance greater than total let _ = Balances::make_free_balance_be(&11, 1000000); @@ -1587,8 +1575,8 @@ fn rebond_works() { fn rebond_is_fifo() { // Rebond should proceed by reversing the most recent bond operations. ExtBuilder::default().nominate(false).build_and_execute(|| { - // Set payee to controller. avoids confusion - assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Controller)); + // Set payee to stash. + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); // Give account 11 some large free balance greater than total let _ = Balances::make_free_balance_be(&11, 1000000); @@ -1683,8 +1671,8 @@ fn rebond_emits_right_value_in_event() { // When a user calls rebond with more than can be rebonded, things succeed, // and the rebond event emits the actual value rebonded. ExtBuilder::default().nominate(false).build_and_execute(|| { - // Set payee to controller. avoids confusion - assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Controller)); + // Set payee to stash. + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); // Give account 11 some large free balance greater than total let _ = Balances::make_free_balance_be(&11, 1000000); @@ -1836,10 +1824,7 @@ fn switching_roles() { ExtBuilder::default().nominate(false).build_and_execute(|| { // Reset reward destination for i in &[11, 21] { - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(*i), - RewardDestination::Controller - )); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(*i), RewardDestination::Stash)); } assert_eq_uvec!(validator_controllers(), vec![21, 11]); @@ -1850,14 +1835,14 @@ fn switching_roles() { } // add 2 nominators - assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 2000, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 2000, RewardDestination::Account(1))); assert_ok!(Staking::nominate(RuntimeOrigin::signed(1), vec![11, 5])); - assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 500, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 500, RewardDestination::Account(3))); assert_ok!(Staking::nominate(RuntimeOrigin::signed(3), vec![21, 1])); // add a new validator candidate - assert_ok!(Staking::bond(RuntimeOrigin::signed(5), 1000, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(5), 1000, RewardDestination::Account(5))); assert_ok!(Staking::validate(RuntimeOrigin::signed(5), ValidatorPrefs::default())); assert_ok!(Session::set_keys( RuntimeOrigin::signed(5), @@ -1928,11 +1913,11 @@ fn bond_with_no_staked_value() { .build_and_execute(|| { // Can't bond with 1 assert_noop!( - Staking::bond(RuntimeOrigin::signed(1), 1, RewardDestination::Controller), + Staking::bond(RuntimeOrigin::signed(1), 1, RewardDestination::Account(1)), Error::::InsufficientBond, ); // bonded with absolute minimum value possible. - assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 5, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 5, RewardDestination::Account(1))); assert_eq!(Balances::locks(&1)[0].amount, 5); // unbonding even 1 will cause all to be unbonded. @@ -1974,15 +1959,12 @@ fn bond_with_little_staked_value_bounded() { .build_and_execute(|| { // setup assert_ok!(Staking::chill(RuntimeOrigin::signed(31))); - assert_ok!(Staking::set_payee( - RuntimeOrigin::signed(11), - RewardDestination::Controller - )); + assert_ok!(Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Stash)); let init_balance_1 = Balances::free_balance(&1); let init_balance_11 = Balances::free_balance(&11); // Stingy validator. - assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 1, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(1), 1, RewardDestination::Account(1))); assert_ok!(Staking::validate(RuntimeOrigin::signed(1), ValidatorPrefs::default())); assert_ok!(Session::set_keys( RuntimeOrigin::signed(1), @@ -2061,14 +2043,14 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider() { assert_ok!(Staking::bond( RuntimeOrigin::signed(1), 1000, - RewardDestination::Controller + RewardDestination::Account(1) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(1), vec![11, 11, 11, 21, 31])); assert_ok!(Staking::bond( RuntimeOrigin::signed(3), 1000, - RewardDestination::Controller + RewardDestination::Account(3) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(3), vec![21, 31])); @@ -2114,14 +2096,14 @@ fn bond_with_duplicate_vote_should_be_ignored_by_election_provider_elected() { assert_ok!(Staking::bond( RuntimeOrigin::signed(1), 1000, - RewardDestination::Controller + RewardDestination::Account(1) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(1), vec![11, 11, 11, 21])); assert_ok!(Staking::bond( RuntimeOrigin::signed(3), 1000, - RewardDestination::Controller + RewardDestination::Account(3) )); assert_ok!(Staking::nominate(RuntimeOrigin::signed(3), vec![21])); @@ -3530,8 +3512,8 @@ fn claim_reward_at_the_last_era_and_no_double_claim_and_invalid_claim() { let part_for_101 = Perbill::from_rational::(125, 1125); // Check state - Payee::::insert(11, RewardDestination::Controller); - Payee::::insert(101, RewardDestination::Controller); + Payee::::insert(11, RewardDestination::Account(11)); + Payee::::insert(101, RewardDestination::Account(101)); Pallet::::reward_by_ids(vec![(11, 1)]); // Compute total payout now for whole duration as other parameter won't change @@ -3820,8 +3802,8 @@ fn test_multi_page_payout_stakers_by_page() { staking_events_since_last_call().as_slice(), &[ .., - Event::Rewarded { stash: 1063, dest: RewardDestination::Controller, amount: 111 }, - Event::Rewarded { stash: 1064, dest: RewardDestination::Controller, amount: 111 }, + Event::Rewarded { stash: 1063, dest: RewardDestination::Stash, amount: 111 }, + Event::Rewarded { stash: 1064, dest: RewardDestination::Stash, amount: 111 }, ] )); @@ -3843,8 +3825,8 @@ fn test_multi_page_payout_stakers_by_page() { events.as_slice(), &[ Event::PayoutStarted { era_index: 1, validator_stash: 11 }, - Event::Rewarded { stash: 1065, dest: RewardDestination::Controller, amount: 111 }, - Event::Rewarded { stash: 1066, dest: RewardDestination::Controller, amount: 111 }, + Event::Rewarded { stash: 1065, dest: RewardDestination::Stash, amount: 111 }, + Event::Rewarded { stash: 1066, dest: RewardDestination::Stash, amount: 111 }, .. ] )); @@ -4685,40 +4667,6 @@ fn offences_weight_calculated_correctly() { }); } -#[test] -fn payout_creates_controller() { - ExtBuilder::default().has_stakers(false).build_and_execute(|| { - let balance = 1000; - // Create a validator: - bond_validator(11, balance); - - // create a stash/controller pair and nominate - let (stash, controller) = testing_utils::create_unique_stash_controller::( - 0, - 100, - RewardDestination::Controller, - false, - ) - .unwrap(); - - assert_ok!(Staking::nominate(RuntimeOrigin::signed(controller), vec![11])); - - // kill controller - assert_ok!(Balances::transfer_allow_death(RuntimeOrigin::signed(controller), stash, 100)); - assert_eq!(Balances::free_balance(controller), 0); - - mock::start_active_era(1); - Staking::reward_by_ids(vec![(11, 1)]); - // compute and ensure the reward amount is greater than zero. - let _ = current_total_payout_for_duration(reward_time_per_era()); - mock::start_active_era(2); - assert_ok!(Staking::payout_stakers_by_page(RuntimeOrigin::signed(1337), 11, 1, 0)); - - // Controller is created - assert!(Balances::free_balance(controller) > 0); - }) -} - #[test] fn payout_to_any_account_works() { ExtBuilder::default().has_stakers(false).build_and_execute(|| { @@ -5462,7 +5410,7 @@ fn min_bond_checks_work() { .min_validator_bond(1_500) .build_and_execute(|| { // 500 is not enough for any role - assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 500, RewardDestination::Controller)); + assert_ok!(Staking::bond(RuntimeOrigin::signed(3), 500, RewardDestination::Stash)); assert_noop!( Staking::nominate(RuntimeOrigin::signed(3), vec![1]), Error::::InsufficientBond @@ -5524,19 +5472,11 @@ fn chill_other_works() { Balances::make_free_balance_be(&c, 100_000); // Nominator - assert_ok!(Staking::bond( - RuntimeOrigin::signed(a), - 1000, - RewardDestination::Controller - )); + assert_ok!(Staking::bond(RuntimeOrigin::signed(a), 1000, RewardDestination::Stash)); assert_ok!(Staking::nominate(RuntimeOrigin::signed(a), vec![1])); // Validator - assert_ok!(Staking::bond( - RuntimeOrigin::signed(b), - 1500, - RewardDestination::Controller - )); + assert_ok!(Staking::bond(RuntimeOrigin::signed(b), 1500, RewardDestination::Stash)); assert_ok!(Staking::validate(RuntimeOrigin::signed(b), ValidatorPrefs::default())); } @@ -5683,7 +5623,7 @@ fn capped_stakers_works() { let (_, controller) = testing_utils::create_stash_controller::( i + 10_000_000, 100, - RewardDestination::Controller, + RewardDestination::Stash, ) .unwrap(); assert_ok!(Staking::validate( @@ -5694,12 +5634,9 @@ fn capped_stakers_works() { } // but no more - let (_, last_validator) = testing_utils::create_stash_controller::( - 1337, - 100, - RewardDestination::Controller, - ) - .unwrap(); + let (_, last_validator) = + testing_utils::create_stash_controller::(1337, 100, RewardDestination::Stash) + .unwrap(); assert_noop!( Staking::validate(RuntimeOrigin::signed(last_validator), ValidatorPrefs::default()), @@ -5712,7 +5649,7 @@ fn capped_stakers_works() { let (_, controller) = testing_utils::create_stash_controller::( i + 20_000_000, 100, - RewardDestination::Controller, + RewardDestination::Stash, ) .unwrap(); assert_ok!(Staking::nominate(RuntimeOrigin::signed(controller), vec![1])); @@ -5723,7 +5660,7 @@ fn capped_stakers_works() { let (_, last_nominator) = testing_utils::create_stash_controller::( 30_000_000, 100, - RewardDestination::Controller, + RewardDestination::Stash, ) .unwrap(); assert_noop!( @@ -6787,7 +6724,7 @@ mod ledger { assert_ok!(Staking::bond( RuntimeOrigin::signed(10), 100, - RewardDestination::Controller + RewardDestination::Account(10) )); assert_eq!(>::get(&10), Some(10)); @@ -6896,4 +6833,38 @@ mod ledger { >::remove(42); // ensures try-state checks pass. }) } + + #[test] + #[allow(deprecated)] + fn set_payee_errors_on_controller_destination() { + ExtBuilder::default().build_and_execute(|| { + Payee::::insert(11, RewardDestination::Staked); + assert_noop!( + Staking::set_payee(RuntimeOrigin::signed(11), RewardDestination::Controller), + Error::::ControllerDeprecated + ); + assert_eq!(Payee::::get(&11), RewardDestination::Staked); + }) + } + + #[test] + #[allow(deprecated)] + fn update_payee_migration_works() { + ExtBuilder::default().build_and_execute(|| { + // migrate a `Controller` variant to `Account` variant. + Payee::::insert(11, RewardDestination::Controller); + assert_eq!(Payee::::get(&11), RewardDestination::Controller); + assert_ok!(Staking::update_payee(RuntimeOrigin::signed(11), 11)); + assert_eq!(Payee::::get(&11), RewardDestination::Account(11)); + + // Do not migrate a variant if not `Controller`. + Payee::::insert(21, RewardDestination::Stash); + assert_eq!(Payee::::get(&21), RewardDestination::Stash); + assert_noop!( + Staking::update_payee(RuntimeOrigin::signed(11), 21), + Error::::NotController + ); + assert_eq!(Payee::::get(&21), RewardDestination::Stash); + }) + } } diff --git a/substrate/frame/staking/src/weights.rs b/substrate/frame/staking/src/weights.rs index ad6dbccde9f8..ad9ca75a0328 100644 --- a/substrate/frame/staking/src/weights.rs +++ b/substrate/frame/staking/src/weights.rs @@ -18,7 +18,7 @@ //! Autogenerated weights for `pallet_staking` //! //! THIS FILE WAS AUTO-GENERATED USING THE SUBSTRATE BENCHMARK CLI VERSION 4.0.0-dev -//! DATE: 2023-10-31, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` +//! DATE: 2023-11-17, STEPS: `50`, REPEAT: `20`, LOW RANGE: `[]`, HIGH RANGE: `[]` //! WORST CASE MAP SIZE: `1000000` //! HOSTNAME: `runner-yprdrvc7-project-674-concurrent-0`, CPU: `Intel(R) Xeon(R) CPU @ 2.60GHz` //! WASM-EXECUTION: `Compiled`, CHAIN: `Some("dev")`, DB CACHE: `1024` @@ -59,6 +59,7 @@ pub trait WeightInfo { fn nominate(n: u32, ) -> Weight; fn chill() -> Weight; fn set_payee() -> Weight; + fn update_payee() -> Weight; fn set_controller() -> Weight; fn set_validator_count() -> Weight; fn force_no_eras() -> Weight; @@ -67,7 +68,6 @@ pub trait WeightInfo { fn set_invulnerables(v: u32, ) -> Weight; fn force_unstake(s: u32, ) -> Weight; fn cancel_deferred_slash(s: u32, ) -> Weight; - fn payout_stakers_dead_controller(n: u32, ) -> Weight; fn payout_stakers_alive_staked(n: u32, ) -> Weight; fn rebond(l: u32, ) -> Weight; fn reap_stash(s: u32, ) -> Weight; @@ -98,8 +98,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `927` // Estimated: `4764` - // Minimum execution time: 42_811_000 picoseconds. - Weight::from_parts(44_465_000, 4764) + // Minimum execution time: 43_427_000 picoseconds. + Weight::from_parts(45_221_000, 4764) .saturating_add(T::DbWeight::get().reads(3_u64)) .saturating_add(T::DbWeight::get().writes(4_u64)) } @@ -119,8 +119,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1990` // Estimated: `8877` - // Minimum execution time: 87_628_000 picoseconds. - Weight::from_parts(90_020_000, 8877) + // Minimum execution time: 87_100_000 picoseconds. + Weight::from_parts(90_599_000, 8877) .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(7_u64)) } @@ -146,8 +146,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `2195` // Estimated: `8877` - // Minimum execution time: 91_655_000 picoseconds. - Weight::from_parts(94_146_000, 8877) + // Minimum execution time: 91_488_000 picoseconds. + Weight::from_parts(94_126_000, 8877) .saturating_add(T::DbWeight::get().reads(12_u64)) .saturating_add(T::DbWeight::get().writes(7_u64)) } @@ -166,10 +166,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1115` // Estimated: `4764` - // Minimum execution time: 42_953_000 picoseconds. - Weight::from_parts(44_648_505, 4764) - // Standard Error: 937 - .saturating_add(Weight::from_parts(51_090, 0).saturating_mul(s.into())) + // Minimum execution time: 42_713_000 picoseconds. + Weight::from_parts(44_530_499, 4764) + // Standard Error: 1_067 + .saturating_add(Weight::from_parts(51_411, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(5_u64)) .saturating_add(T::DbWeight::get().writes(2_u64)) } @@ -206,10 +206,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 89_218_000 picoseconds. - Weight::from_parts(97_761_884, 6248) - // Standard Error: 3_888 - .saturating_add(Weight::from_parts(1_346_441, 0).saturating_mul(s.into())) + // Minimum execution time: 88_700_000 picoseconds. + Weight::from_parts(98_329_624, 6248) + // Standard Error: 4_477 + .saturating_add(Weight::from_parts(1_347_380, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(13_u64)) .saturating_add(T::DbWeight::get().writes(11_u64)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -241,8 +241,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1372` // Estimated: `4556` - // Minimum execution time: 51_200_000 picoseconds. - Weight::from_parts(53_403_000, 4556) + // Minimum execution time: 51_877_000 picoseconds. + Weight::from_parts(53_503_000, 4556) .saturating_add(T::DbWeight::get().reads(11_u64)) .saturating_add(T::DbWeight::get().writes(5_u64)) } @@ -255,10 +255,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1280 + k * (569 ±0)` // Estimated: `4556 + k * (3033 ±0)` - // Minimum execution time: 28_963_000 picoseconds. - Weight::from_parts(29_884_371, 4556) - // Standard Error: 9_063 - .saturating_add(Weight::from_parts(6_532_967, 0).saturating_mul(k.into())) + // Minimum execution time: 29_604_000 picoseconds. + Weight::from_parts(31_726_296, 4556) + // Standard Error: 6_350 + .saturating_add(Weight::from_parts(6_416_846, 0).saturating_mul(k.into())) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(k.into()))) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(k.into()))) @@ -291,10 +291,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1866 + n * (102 ±0)` // Estimated: `6248 + n * (2520 ±0)` - // Minimum execution time: 64_644_000 picoseconds. - Weight::from_parts(62_855_016, 6248) - // Standard Error: 17_528 - .saturating_add(Weight::from_parts(3_993_850, 0).saturating_mul(n.into())) + // Minimum execution time: 64_276_000 picoseconds. + Weight::from_parts(62_615_844, 6248) + // Standard Error: 14_914 + .saturating_add(Weight::from_parts(4_097_019, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(12_u64)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes(6_u64)) @@ -318,8 +318,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1650` // Estimated: `6248` - // Minimum execution time: 54_505_000 picoseconds. - Weight::from_parts(56_026_000, 6248) + // Minimum execution time: 55_064_000 picoseconds. + Weight::from_parts(56_566_000, 6248) .saturating_add(T::DbWeight::get().reads(8_u64)) .saturating_add(T::DbWeight::get().writes(6_u64)) } @@ -333,11 +333,26 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `902` // Estimated: `4556` - // Minimum execution time: 16_639_000 picoseconds. - Weight::from_parts(17_342_000, 4556) + // Minimum execution time: 16_626_000 picoseconds. + Weight::from_parts(17_519_000, 4556) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } + /// Storage: `Staking::Ledger` (r:1 w:0) + /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) + /// Storage: `Staking::Payee` (r:1 w:1) + /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) + /// Storage: `Staking::Bonded` (r:1 w:0) + /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) + fn update_payee() -> Weight { + // Proof Size summary in bytes: + // Measured: `969` + // Estimated: `4556` + // Minimum execution time: 20_545_000 picoseconds. + Weight::from_parts(21_395_000, 4556) + .saturating_add(T::DbWeight::get().reads(3_u64)) + .saturating_add(T::DbWeight::get().writes(1_u64)) + } /// Storage: `Staking::Bonded` (r:1 w:1) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:1 w:2) @@ -346,8 +361,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `902` // Estimated: `4556` - // Minimum execution time: 20_334_000 picoseconds. - Weight::from_parts(21_067_000, 4556) + // Minimum execution time: 20_179_000 picoseconds. + Weight::from_parts(20_728_000, 4556) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(3_u64)) } @@ -357,8 +372,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_680_000 picoseconds. - Weight::from_parts(2_774_000, 0) + // Minimum execution time: 2_594_000 picoseconds. + Weight::from_parts(2_777_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -367,8 +382,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_613_000 picoseconds. - Weight::from_parts(8_922_000, 0) + // Minimum execution time: 8_302_000 picoseconds. + Weight::from_parts(8_741_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -377,8 +392,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_657_000 picoseconds. - Weight::from_parts(9_020_000, 0) + // Minimum execution time: 8_504_000 picoseconds. + Weight::from_parts(8_774_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -387,8 +402,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_600_000 picoseconds. - Weight::from_parts(9_157_000, 0) + // Minimum execution time: 8_474_000 picoseconds. + Weight::from_parts(8_740_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Staking::Invulnerables` (r:0 w:1) @@ -398,10 +413,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_792_000 picoseconds. - Weight::from_parts(3_293_694, 0) - // Standard Error: 31 - .saturating_add(Weight::from_parts(10_668, 0).saturating_mul(v.into())) + // Minimum execution time: 2_732_000 picoseconds. + Weight::from_parts(3_360_048, 0) + // Standard Error: 34 + .saturating_add(Weight::from_parts(9_964, 0).saturating_mul(v.into())) .saturating_add(T::DbWeight::get().writes(1_u64)) } /// Storage: `Staking::SlashingSpans` (r:1 w:1) @@ -437,10 +452,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 86_537_000 picoseconds. - Weight::from_parts(95_127_637, 6248) - // Standard Error: 3_902 - .saturating_add(Weight::from_parts(1_336_182, 0).saturating_mul(s.into())) + // Minimum execution time: 87_495_000 picoseconds. + Weight::from_parts(95_710_388, 6248) + // Standard Error: 5_849 + .saturating_add(Weight::from_parts(1_339_335, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(13_u64)) .saturating_add(T::DbWeight::get().writes(12_u64)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -453,54 +468,13 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `66672` // Estimated: `70137` - // Minimum execution time: 100_777_000 picoseconds. - Weight::from_parts(896_540_406, 70137) - // Standard Error: 57_788 - .saturating_add(Weight::from_parts(4_870_910, 0).saturating_mul(s.into())) + // Minimum execution time: 98_255_000 picoseconds. + Weight::from_parts(892_807_378, 70137) + // Standard Error: 57_735 + .saturating_add(Weight::from_parts(4_876_449, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(1_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } - /// Storage: `Staking::CurrentEra` (r:1 w:0) - /// Proof: `Staking::CurrentEra` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasStakersOverview` (r:1 w:0) - /// Proof: `Staking::ErasStakersOverview` (`max_values`: None, `max_size`: Some(92), added: 2567, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasValidatorReward` (r:1 w:0) - /// Proof: `Staking::ErasValidatorReward` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) - /// Storage: `Staking::Bonded` (r:257 w:0) - /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) - /// Storage: `Staking::Ledger` (r:1 w:1) - /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) - /// Storage: `Balances::Locks` (r:1 w:1) - /// Proof: `Balances::Locks` (`max_values`: None, `max_size`: Some(1299), added: 3774, mode: `MaxEncodedLen`) - /// Storage: `Balances::Freezes` (r:1 w:0) - /// Proof: `Balances::Freezes` (`max_values`: None, `max_size`: Some(67), added: 2542, mode: `MaxEncodedLen`) - /// Storage: `System::Account` (r:258 w:258) - /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) - /// Storage: `Staking::ClaimedRewards` (r:1 w:1) - /// Proof: `Staking::ClaimedRewards` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasStakersPaged` (r:1 w:0) - /// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasRewardPoints` (r:1 w:0) - /// Proof: `Staking::ErasRewardPoints` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasValidatorPrefs` (r:1 w:0) - /// Proof: `Staking::ErasValidatorPrefs` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`) - /// Storage: `Staking::Payee` (r:257 w:0) - /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) - /// The range of component `n` is `[0, 256]`. - fn payout_stakers_dead_controller(n: u32, ) -> Weight { - // Proof Size summary in bytes: - // Measured: `21644 + n * (155 ±0)` - // Estimated: `21412 + n * (2603 ±0)` - // Minimum execution time: 133_129_000 picoseconds. - Weight::from_parts(190_983_630, 21412) - // Standard Error: 17_497 - .saturating_add(Weight::from_parts(24_723_153, 0).saturating_mul(n.into())) - .saturating_add(T::DbWeight::get().reads(14_u64)) - .saturating_add(T::DbWeight::get().reads((3_u64).saturating_mul(n.into()))) - .saturating_add(T::DbWeight::get().writes(5_u64)) - .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(n.into()))) - .saturating_add(Weight::from_parts(0, 2603).saturating_mul(n.into())) - } /// Storage: `Staking::Bonded` (r:257 w:0) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:257 w:257) @@ -534,10 +508,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `33297 + n * (377 ±0)` // Estimated: `30944 + n * (3774 ±3)` - // Minimum execution time: 149_773_000 picoseconds. - Weight::from_parts(151_527_124, 30944) - // Standard Error: 24_152 - .saturating_add(Weight::from_parts(46_124_074, 0).saturating_mul(n.into())) + // Minimum execution time: 144_258_000 picoseconds. + Weight::from_parts(175_256_796, 30944) + // Standard Error: 24_339 + .saturating_add(Weight::from_parts(46_011_700, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(14_u64)) .saturating_add(T::DbWeight::get().reads((6_u64).saturating_mul(n.into()))) .saturating_add(T::DbWeight::get().writes(4_u64)) @@ -561,10 +535,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1991 + l * (7 ±0)` // Estimated: `8877` - // Minimum execution time: 81_618_000 picoseconds. - Weight::from_parts(85_245_630, 8877) - // Standard Error: 5_049 - .saturating_add(Weight::from_parts(39_811, 0).saturating_mul(l.into())) + // Minimum execution time: 83_532_000 picoseconds. + Weight::from_parts(86_757_943, 8877) + // Standard Error: 6_379 + .saturating_add(Weight::from_parts(57_566, 0).saturating_mul(l.into())) .saturating_add(T::DbWeight::get().reads(9_u64)) .saturating_add(T::DbWeight::get().writes(7_u64)) } @@ -599,10 +573,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 95_395_000 picoseconds. - Weight::from_parts(100_459_234, 6248) - // Standard Error: 3_781 - .saturating_add(Weight::from_parts(1_333_607, 0).saturating_mul(s.into())) + // Minimum execution time: 96_776_000 picoseconds. + Weight::from_parts(100_950_027, 6248) + // Standard Error: 4_719 + .saturating_add(Weight::from_parts(1_432_091, 0).saturating_mul(s.into())) .saturating_add(T::DbWeight::get().reads(12_u64)) .saturating_add(T::DbWeight::get().writes(11_u64)) .saturating_add(T::DbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -648,12 +622,12 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0 + n * (720 ±0) + v * (3598 ±0)` // Estimated: `512390 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 571_337_000 picoseconds. - Weight::from_parts(578_857_000, 512390) - // Standard Error: 2_090_511 - .saturating_add(Weight::from_parts(68_626_083, 0).saturating_mul(v.into())) - // Standard Error: 208_307 - .saturating_add(Weight::from_parts(18_645_374, 0).saturating_mul(n.into())) + // Minimum execution time: 577_699_000 picoseconds. + Weight::from_parts(582_827_000, 512390) + // Standard Error: 2_000_851 + .saturating_add(Weight::from_parts(67_316_744, 0).saturating_mul(v.into())) + // Standard Error: 199_373 + .saturating_add(Weight::from_parts(18_503_387, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(206_u64)) .saturating_add(T::DbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(T::DbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -684,12 +658,12 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `3175 + n * (911 ±0) + v * (395 ±0)` // Estimated: `512390 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 34_590_734_000 picoseconds. - Weight::from_parts(35_238_091_000, 512390) - // Standard Error: 427_974 - .saturating_add(Weight::from_parts(5_084_196, 0).saturating_mul(v.into())) - // Standard Error: 427_974 - .saturating_add(Weight::from_parts(4_503_420, 0).saturating_mul(n.into())) + // Minimum execution time: 34_048_778_000 picoseconds. + Weight::from_parts(34_397_777_000, 512390) + // Standard Error: 346_115 + .saturating_add(Weight::from_parts(3_704_941, 0).saturating_mul(v.into())) + // Standard Error: 346_115 + .saturating_add(Weight::from_parts(4_064_819, 0).saturating_mul(n.into())) .saturating_add(T::DbWeight::get().reads(201_u64)) .saturating_add(T::DbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(T::DbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -706,10 +680,10 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `979 + v * (50 ±0)` // Estimated: `3510 + v * (2520 ±0)` - // Minimum execution time: 2_509_588_000 picoseconds. - Weight::from_parts(89_050_539, 3510) - // Standard Error: 11_803 - .saturating_add(Weight::from_parts(5_031_416, 0).saturating_mul(v.into())) + // Minimum execution time: 2_473_149_000 picoseconds. + Weight::from_parts(84_721_859, 3510) + // Standard Error: 8_690 + .saturating_add(Weight::from_parts(4_870_439, 0).saturating_mul(v.into())) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().reads((1_u64).saturating_mul(v.into()))) .saturating_add(Weight::from_parts(0, 2520).saturating_mul(v.into())) @@ -730,8 +704,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 5_347_000 picoseconds. - Weight::from_parts(5_562_000, 0) + // Minimum execution time: 5_312_000 picoseconds. + Weight::from_parts(5_897_000, 0) .saturating_add(T::DbWeight::get().writes(6_u64)) } /// Storage: `Staking::MinCommission` (r:0 w:1) @@ -750,8 +724,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 4_725_000 picoseconds. - Weight::from_parts(5_075_000, 0) + // Minimum execution time: 4_676_000 picoseconds. + Weight::from_parts(4_913_000, 0) .saturating_add(T::DbWeight::get().writes(6_u64)) } /// Storage: `Staking::Ledger` (r:1 w:0) @@ -778,8 +752,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `1773` // Estimated: `6248` - // Minimum execution time: 67_204_000 picoseconds. - Weight::from_parts(69_197_000, 6248) + // Minimum execution time: 67_286_000 picoseconds. + Weight::from_parts(69_081_000, 6248) .saturating_add(T::DbWeight::get().reads(11_u64)) .saturating_add(T::DbWeight::get().writes(6_u64)) } @@ -791,8 +765,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `691` // Estimated: `3510` - // Minimum execution time: 12_497_000 picoseconds. - Weight::from_parts(12_943_000, 3510) + // Minimum execution time: 12_749_000 picoseconds. + Weight::from_parts(13_275_000, 3510) .saturating_add(T::DbWeight::get().reads(2_u64)) .saturating_add(T::DbWeight::get().writes(1_u64)) } @@ -802,8 +776,8 @@ impl WeightInfo for SubstrateWeight { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_245_000 picoseconds. - Weight::from_parts(3_352_000, 0) + // Minimum execution time: 3_155_000 picoseconds. + Weight::from_parts(3_319_000, 0) .saturating_add(T::DbWeight::get().writes(1_u64)) } } @@ -824,8 +798,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `927` // Estimated: `4764` - // Minimum execution time: 42_811_000 picoseconds. - Weight::from_parts(44_465_000, 4764) + // Minimum execution time: 43_427_000 picoseconds. + Weight::from_parts(45_221_000, 4764) .saturating_add(RocksDbWeight::get().reads(3_u64)) .saturating_add(RocksDbWeight::get().writes(4_u64)) } @@ -845,8 +819,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1990` // Estimated: `8877` - // Minimum execution time: 87_628_000 picoseconds. - Weight::from_parts(90_020_000, 8877) + // Minimum execution time: 87_100_000 picoseconds. + Weight::from_parts(90_599_000, 8877) .saturating_add(RocksDbWeight::get().reads(9_u64)) .saturating_add(RocksDbWeight::get().writes(7_u64)) } @@ -872,8 +846,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `2195` // Estimated: `8877` - // Minimum execution time: 91_655_000 picoseconds. - Weight::from_parts(94_146_000, 8877) + // Minimum execution time: 91_488_000 picoseconds. + Weight::from_parts(94_126_000, 8877) .saturating_add(RocksDbWeight::get().reads(12_u64)) .saturating_add(RocksDbWeight::get().writes(7_u64)) } @@ -892,10 +866,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1115` // Estimated: `4764` - // Minimum execution time: 42_953_000 picoseconds. - Weight::from_parts(44_648_505, 4764) - // Standard Error: 937 - .saturating_add(Weight::from_parts(51_090, 0).saturating_mul(s.into())) + // Minimum execution time: 42_713_000 picoseconds. + Weight::from_parts(44_530_499, 4764) + // Standard Error: 1_067 + .saturating_add(Weight::from_parts(51_411, 0).saturating_mul(s.into())) .saturating_add(RocksDbWeight::get().reads(5_u64)) .saturating_add(RocksDbWeight::get().writes(2_u64)) } @@ -932,10 +906,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 89_218_000 picoseconds. - Weight::from_parts(97_761_884, 6248) - // Standard Error: 3_888 - .saturating_add(Weight::from_parts(1_346_441, 0).saturating_mul(s.into())) + // Minimum execution time: 88_700_000 picoseconds. + Weight::from_parts(98_329_624, 6248) + // Standard Error: 4_477 + .saturating_add(Weight::from_parts(1_347_380, 0).saturating_mul(s.into())) .saturating_add(RocksDbWeight::get().reads(13_u64)) .saturating_add(RocksDbWeight::get().writes(11_u64)) .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -967,8 +941,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1372` // Estimated: `4556` - // Minimum execution time: 51_200_000 picoseconds. - Weight::from_parts(53_403_000, 4556) + // Minimum execution time: 51_877_000 picoseconds. + Weight::from_parts(53_503_000, 4556) .saturating_add(RocksDbWeight::get().reads(11_u64)) .saturating_add(RocksDbWeight::get().writes(5_u64)) } @@ -981,10 +955,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1280 + k * (569 ±0)` // Estimated: `4556 + k * (3033 ±0)` - // Minimum execution time: 28_963_000 picoseconds. - Weight::from_parts(29_884_371, 4556) - // Standard Error: 9_063 - .saturating_add(Weight::from_parts(6_532_967, 0).saturating_mul(k.into())) + // Minimum execution time: 29_604_000 picoseconds. + Weight::from_parts(31_726_296, 4556) + // Standard Error: 6_350 + .saturating_add(Weight::from_parts(6_416_846, 0).saturating_mul(k.into())) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(k.into()))) .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(k.into()))) @@ -1017,10 +991,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1866 + n * (102 ±0)` // Estimated: `6248 + n * (2520 ±0)` - // Minimum execution time: 64_644_000 picoseconds. - Weight::from_parts(62_855_016, 6248) - // Standard Error: 17_528 - .saturating_add(Weight::from_parts(3_993_850, 0).saturating_mul(n.into())) + // Minimum execution time: 64_276_000 picoseconds. + Weight::from_parts(62_615_844, 6248) + // Standard Error: 14_914 + .saturating_add(Weight::from_parts(4_097_019, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(12_u64)) .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(n.into()))) .saturating_add(RocksDbWeight::get().writes(6_u64)) @@ -1044,8 +1018,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1650` // Estimated: `6248` - // Minimum execution time: 54_505_000 picoseconds. - Weight::from_parts(56_026_000, 6248) + // Minimum execution time: 55_064_000 picoseconds. + Weight::from_parts(56_566_000, 6248) .saturating_add(RocksDbWeight::get().reads(8_u64)) .saturating_add(RocksDbWeight::get().writes(6_u64)) } @@ -1059,11 +1033,26 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `902` // Estimated: `4556` - // Minimum execution time: 16_639_000 picoseconds. - Weight::from_parts(17_342_000, 4556) + // Minimum execution time: 16_626_000 picoseconds. + Weight::from_parts(17_519_000, 4556) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } + /// Storage: `Staking::Ledger` (r:1 w:0) + /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) + /// Storage: `Staking::Payee` (r:1 w:1) + /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) + /// Storage: `Staking::Bonded` (r:1 w:0) + /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) + fn update_payee() -> Weight { + // Proof Size summary in bytes: + // Measured: `969` + // Estimated: `4556` + // Minimum execution time: 20_545_000 picoseconds. + Weight::from_parts(21_395_000, 4556) + .saturating_add(RocksDbWeight::get().reads(3_u64)) + .saturating_add(RocksDbWeight::get().writes(1_u64)) + } /// Storage: `Staking::Bonded` (r:1 w:1) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:1 w:2) @@ -1072,8 +1061,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `902` // Estimated: `4556` - // Minimum execution time: 20_334_000 picoseconds. - Weight::from_parts(21_067_000, 4556) + // Minimum execution time: 20_179_000 picoseconds. + Weight::from_parts(20_728_000, 4556) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(3_u64)) } @@ -1083,8 +1072,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_680_000 picoseconds. - Weight::from_parts(2_774_000, 0) + // Minimum execution time: 2_594_000 picoseconds. + Weight::from_parts(2_777_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -1093,8 +1082,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_613_000 picoseconds. - Weight::from_parts(8_922_000, 0) + // Minimum execution time: 8_302_000 picoseconds. + Weight::from_parts(8_741_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -1103,8 +1092,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_657_000 picoseconds. - Weight::from_parts(9_020_000, 0) + // Minimum execution time: 8_504_000 picoseconds. + Weight::from_parts(8_774_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Staking::ForceEra` (r:0 w:1) @@ -1113,8 +1102,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 8_600_000 picoseconds. - Weight::from_parts(9_157_000, 0) + // Minimum execution time: 8_474_000 picoseconds. + Weight::from_parts(8_740_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Staking::Invulnerables` (r:0 w:1) @@ -1124,10 +1113,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 2_792_000 picoseconds. - Weight::from_parts(3_293_694, 0) - // Standard Error: 31 - .saturating_add(Weight::from_parts(10_668, 0).saturating_mul(v.into())) + // Minimum execution time: 2_732_000 picoseconds. + Weight::from_parts(3_360_048, 0) + // Standard Error: 34 + .saturating_add(Weight::from_parts(9_964, 0).saturating_mul(v.into())) .saturating_add(RocksDbWeight::get().writes(1_u64)) } /// Storage: `Staking::SlashingSpans` (r:1 w:1) @@ -1163,10 +1152,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 86_537_000 picoseconds. - Weight::from_parts(95_127_637, 6248) - // Standard Error: 3_902 - .saturating_add(Weight::from_parts(1_336_182, 0).saturating_mul(s.into())) + // Minimum execution time: 87_495_000 picoseconds. + Weight::from_parts(95_710_388, 6248) + // Standard Error: 5_849 + .saturating_add(Weight::from_parts(1_339_335, 0).saturating_mul(s.into())) .saturating_add(RocksDbWeight::get().reads(13_u64)) .saturating_add(RocksDbWeight::get().writes(12_u64)) .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -1179,54 +1168,13 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `66672` // Estimated: `70137` - // Minimum execution time: 100_777_000 picoseconds. - Weight::from_parts(896_540_406, 70137) - // Standard Error: 57_788 - .saturating_add(Weight::from_parts(4_870_910, 0).saturating_mul(s.into())) + // Minimum execution time: 98_255_000 picoseconds. + Weight::from_parts(892_807_378, 70137) + // Standard Error: 57_735 + .saturating_add(Weight::from_parts(4_876_449, 0).saturating_mul(s.into())) .saturating_add(RocksDbWeight::get().reads(1_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } - /// Storage: `Staking::CurrentEra` (r:1 w:0) - /// Proof: `Staking::CurrentEra` (`max_values`: Some(1), `max_size`: Some(4), added: 499, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasStakersOverview` (r:1 w:0) - /// Proof: `Staking::ErasStakersOverview` (`max_values`: None, `max_size`: Some(92), added: 2567, mode: `MaxEncodedLen`) - /// Storage: `Staking::ErasValidatorReward` (r:1 w:0) - /// Proof: `Staking::ErasValidatorReward` (`max_values`: None, `max_size`: Some(28), added: 2503, mode: `MaxEncodedLen`) - /// Storage: `Staking::Bonded` (r:257 w:0) - /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) - /// Storage: `Staking::Ledger` (r:1 w:1) - /// Proof: `Staking::Ledger` (`max_values`: None, `max_size`: Some(1091), added: 3566, mode: `MaxEncodedLen`) - /// Storage: `Balances::Locks` (r:1 w:1) - /// Proof: `Balances::Locks` (`max_values`: None, `max_size`: Some(1299), added: 3774, mode: `MaxEncodedLen`) - /// Storage: `Balances::Freezes` (r:1 w:0) - /// Proof: `Balances::Freezes` (`max_values`: None, `max_size`: Some(67), added: 2542, mode: `MaxEncodedLen`) - /// Storage: `System::Account` (r:258 w:258) - /// Proof: `System::Account` (`max_values`: None, `max_size`: Some(128), added: 2603, mode: `MaxEncodedLen`) - /// Storage: `Staking::ClaimedRewards` (r:1 w:1) - /// Proof: `Staking::ClaimedRewards` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasStakersPaged` (r:1 w:0) - /// Proof: `Staking::ErasStakersPaged` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasRewardPoints` (r:1 w:0) - /// Proof: `Staking::ErasRewardPoints` (`max_values`: None, `max_size`: None, mode: `Measured`) - /// Storage: `Staking::ErasValidatorPrefs` (r:1 w:0) - /// Proof: `Staking::ErasValidatorPrefs` (`max_values`: None, `max_size`: Some(57), added: 2532, mode: `MaxEncodedLen`) - /// Storage: `Staking::Payee` (r:257 w:0) - /// Proof: `Staking::Payee` (`max_values`: None, `max_size`: Some(73), added: 2548, mode: `MaxEncodedLen`) - /// The range of component `n` is `[0, 256]`. - fn payout_stakers_dead_controller(n: u32, ) -> Weight { - // Proof Size summary in bytes: - // Measured: `21644 + n * (155 ±0)` - // Estimated: `21412 + n * (2603 ±0)` - // Minimum execution time: 133_129_000 picoseconds. - Weight::from_parts(190_983_630, 21412) - // Standard Error: 17_497 - .saturating_add(Weight::from_parts(24_723_153, 0).saturating_mul(n.into())) - .saturating_add(RocksDbWeight::get().reads(14_u64)) - .saturating_add(RocksDbWeight::get().reads((3_u64).saturating_mul(n.into()))) - .saturating_add(RocksDbWeight::get().writes(5_u64)) - .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(n.into()))) - .saturating_add(Weight::from_parts(0, 2603).saturating_mul(n.into())) - } /// Storage: `Staking::Bonded` (r:257 w:0) /// Proof: `Staking::Bonded` (`max_values`: None, `max_size`: Some(72), added: 2547, mode: `MaxEncodedLen`) /// Storage: `Staking::Ledger` (r:257 w:257) @@ -1260,10 +1208,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `33297 + n * (377 ±0)` // Estimated: `30944 + n * (3774 ±3)` - // Minimum execution time: 149_773_000 picoseconds. - Weight::from_parts(151_527_124, 30944) - // Standard Error: 24_152 - .saturating_add(Weight::from_parts(46_124_074, 0).saturating_mul(n.into())) + // Minimum execution time: 144_258_000 picoseconds. + Weight::from_parts(175_256_796, 30944) + // Standard Error: 24_339 + .saturating_add(Weight::from_parts(46_011_700, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(14_u64)) .saturating_add(RocksDbWeight::get().reads((6_u64).saturating_mul(n.into()))) .saturating_add(RocksDbWeight::get().writes(4_u64)) @@ -1287,10 +1235,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1991 + l * (7 ±0)` // Estimated: `8877` - // Minimum execution time: 81_618_000 picoseconds. - Weight::from_parts(85_245_630, 8877) - // Standard Error: 5_049 - .saturating_add(Weight::from_parts(39_811, 0).saturating_mul(l.into())) + // Minimum execution time: 83_532_000 picoseconds. + Weight::from_parts(86_757_943, 8877) + // Standard Error: 6_379 + .saturating_add(Weight::from_parts(57_566, 0).saturating_mul(l.into())) .saturating_add(RocksDbWeight::get().reads(9_u64)) .saturating_add(RocksDbWeight::get().writes(7_u64)) } @@ -1325,10 +1273,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `2196 + s * (4 ±0)` // Estimated: `6248 + s * (4 ±0)` - // Minimum execution time: 95_395_000 picoseconds. - Weight::from_parts(100_459_234, 6248) - // Standard Error: 3_781 - .saturating_add(Weight::from_parts(1_333_607, 0).saturating_mul(s.into())) + // Minimum execution time: 96_776_000 picoseconds. + Weight::from_parts(100_950_027, 6248) + // Standard Error: 4_719 + .saturating_add(Weight::from_parts(1_432_091, 0).saturating_mul(s.into())) .saturating_add(RocksDbWeight::get().reads(12_u64)) .saturating_add(RocksDbWeight::get().writes(11_u64)) .saturating_add(RocksDbWeight::get().writes((1_u64).saturating_mul(s.into()))) @@ -1374,12 +1322,12 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0 + n * (720 ±0) + v * (3598 ±0)` // Estimated: `512390 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 571_337_000 picoseconds. - Weight::from_parts(578_857_000, 512390) - // Standard Error: 2_090_511 - .saturating_add(Weight::from_parts(68_626_083, 0).saturating_mul(v.into())) - // Standard Error: 208_307 - .saturating_add(Weight::from_parts(18_645_374, 0).saturating_mul(n.into())) + // Minimum execution time: 577_699_000 picoseconds. + Weight::from_parts(582_827_000, 512390) + // Standard Error: 2_000_851 + .saturating_add(Weight::from_parts(67_316_744, 0).saturating_mul(v.into())) + // Standard Error: 199_373 + .saturating_add(Weight::from_parts(18_503_387, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(206_u64)) .saturating_add(RocksDbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(RocksDbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -1410,12 +1358,12 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `3175 + n * (911 ±0) + v * (395 ±0)` // Estimated: `512390 + n * (3566 ±0) + v * (3566 ±0)` - // Minimum execution time: 34_590_734_000 picoseconds. - Weight::from_parts(35_238_091_000, 512390) - // Standard Error: 427_974 - .saturating_add(Weight::from_parts(5_084_196, 0).saturating_mul(v.into())) - // Standard Error: 427_974 - .saturating_add(Weight::from_parts(4_503_420, 0).saturating_mul(n.into())) + // Minimum execution time: 34_048_778_000 picoseconds. + Weight::from_parts(34_397_777_000, 512390) + // Standard Error: 346_115 + .saturating_add(Weight::from_parts(3_704_941, 0).saturating_mul(v.into())) + // Standard Error: 346_115 + .saturating_add(Weight::from_parts(4_064_819, 0).saturating_mul(n.into())) .saturating_add(RocksDbWeight::get().reads(201_u64)) .saturating_add(RocksDbWeight::get().reads((5_u64).saturating_mul(v.into()))) .saturating_add(RocksDbWeight::get().reads((4_u64).saturating_mul(n.into()))) @@ -1432,10 +1380,10 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `979 + v * (50 ±0)` // Estimated: `3510 + v * (2520 ±0)` - // Minimum execution time: 2_509_588_000 picoseconds. - Weight::from_parts(89_050_539, 3510) - // Standard Error: 11_803 - .saturating_add(Weight::from_parts(5_031_416, 0).saturating_mul(v.into())) + // Minimum execution time: 2_473_149_000 picoseconds. + Weight::from_parts(84_721_859, 3510) + // Standard Error: 8_690 + .saturating_add(Weight::from_parts(4_870_439, 0).saturating_mul(v.into())) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().reads((1_u64).saturating_mul(v.into()))) .saturating_add(Weight::from_parts(0, 2520).saturating_mul(v.into())) @@ -1456,8 +1404,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 5_347_000 picoseconds. - Weight::from_parts(5_562_000, 0) + // Minimum execution time: 5_312_000 picoseconds. + Weight::from_parts(5_897_000, 0) .saturating_add(RocksDbWeight::get().writes(6_u64)) } /// Storage: `Staking::MinCommission` (r:0 w:1) @@ -1476,8 +1424,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 4_725_000 picoseconds. - Weight::from_parts(5_075_000, 0) + // Minimum execution time: 4_676_000 picoseconds. + Weight::from_parts(4_913_000, 0) .saturating_add(RocksDbWeight::get().writes(6_u64)) } /// Storage: `Staking::Ledger` (r:1 w:0) @@ -1504,8 +1452,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `1773` // Estimated: `6248` - // Minimum execution time: 67_204_000 picoseconds. - Weight::from_parts(69_197_000, 6248) + // Minimum execution time: 67_286_000 picoseconds. + Weight::from_parts(69_081_000, 6248) .saturating_add(RocksDbWeight::get().reads(11_u64)) .saturating_add(RocksDbWeight::get().writes(6_u64)) } @@ -1517,8 +1465,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `691` // Estimated: `3510` - // Minimum execution time: 12_497_000 picoseconds. - Weight::from_parts(12_943_000, 3510) + // Minimum execution time: 12_749_000 picoseconds. + Weight::from_parts(13_275_000, 3510) .saturating_add(RocksDbWeight::get().reads(2_u64)) .saturating_add(RocksDbWeight::get().writes(1_u64)) } @@ -1528,8 +1476,8 @@ impl WeightInfo for () { // Proof Size summary in bytes: // Measured: `0` // Estimated: `0` - // Minimum execution time: 3_245_000 picoseconds. - Weight::from_parts(3_352_000, 0) + // Minimum execution time: 3_155_000 picoseconds. + Weight::from_parts(3_319_000, 0) .saturating_add(RocksDbWeight::get().writes(1_u64)) } } From a9ba1e540ff524dda0dec8bb0b7da204e043adae Mon Sep 17 00:00:00 2001 From: Alexander Samusev <41779041+alvicsam@users.noreply.github.com> Date: Wed, 22 Nov 2023 10:29:56 +0100 Subject: [PATCH 33/48] Revert docker images tag naming to (#2434) Docker images from paritypr are also used in testnets. PR reverts docker tags naming to a more convenient. cc @PierreBesson --- .gitlab-ci.yml | 2 +- .gitlab/pipeline/publish.yml | 201 +---------------------------------- 2 files changed, 5 insertions(+), 198 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e698aba81d55..f7e54961fa98 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,7 +30,7 @@ variables: RUSTY_CACHIER_COMPRESSION_METHOD: zstd NEXTEST_FAILURE_OUTPUT: immediate-final NEXTEST_SUCCESS_OUTPUT: final - DOCKER_IMAGES_VERSION: "${CI_COMMIT_SHA}" + DOCKER_IMAGES_VERSION: "${CI_COMMIT_REF_NAME}-${CI_COMMIT_SHORT_SHA}" default: retry: diff --git a/.gitlab/pipeline/publish.yml b/.gitlab/pipeline/publish.yml index 3cc2002cc1c5..92ebc9eea1fa 100644 --- a/.gitlab/pipeline/publish.yml +++ b/.gitlab/pipeline/publish.yml @@ -63,16 +63,16 @@ publish-rustdoc: after_script: - rm -rf .git/ ./* -# cumulus - +# note: images are used not only in zombienet but also in rococo, wococo and versi .build-push-image: image: $BUILDAH_IMAGE variables: DOCKERFILE: "" # docker/path-to.Dockerfile IMAGE_NAME: "" # docker.io/paritypr/image_name script: - # Exit if the job is not running in a merge queue - # - if [[ $CI_COMMIT_REF_NAME != *"gh-readonly-queue"* ]]; then echo "I will run only in a merge queue"; exit 0; fi + # Dockertag should differ in a merge queue + # TODO: test this + # - if [[ $CI_COMMIT_REF_NAME == *"gh-readonly-queue"* ]]; export DOCKER_IMAGES_VERSION="${CI_COMMIT_SHORT_SHA}"; fi - $BUILDAH_COMMAND build --format=docker --build-arg VCS_REF="${CI_COMMIT_SHA}" @@ -112,59 +112,6 @@ build-push-image-test-parachain: variables: DOCKERFILE: "docker/dockerfiles/test-parachain_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/test-parachain" -# publish-s3: -# stage: publish -# extends: -# - .kubernetes-env -# - .publish-refs -# image: paritytech/awscli:latest -# needs: -# - job: build-linux-stable-cumulus -# artifacts: true -# variables: -# GIT_STRATEGY: none -# BUCKET: "releases.parity.io" -# PREFIX: "cumulus/${ARCH}-${DOCKER_OS}" -# script: -# - echo "___Publishing a binary with debug assertions!___" -# - echo "___VERSION = $(cat ./artifacts/VERSION) ___" -# - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ -# - echo "___Updating objects in latest path___" -# - aws s3 sync s3://${BUCKET}/${PREFIX}/$(cat ./artifacts/VERSION)/ s3://${BUCKET}/${PREFIX}/latest/ -# after_script: -# - aws s3 ls s3://${BUCKET}/${PREFIX}/latest/ -# --recursive --human-readable --summarize - -# publish-benchmarks-assets-s3: &publish-benchmarks -# stage: publish -# extends: -# - .kubernetes-env -# - .benchmarks-refs -# image: paritytech/awscli:latest -# needs: -# - job: benchmarks-assets -# artifacts: true -# variables: -# GIT_STRATEGY: none -# BUCKET: "releases.parity.io" -# PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-assets" -# script: -# - echo "___Publishing benchmark results___" -# - aws s3 sync ./artifacts/ s3://${BUCKET}/${PREFIX}/ -# after_script: -# - aws s3 ls s3://${BUCKET}/${PREFIX}/ --recursive --human-readable --summarize - -# publish-benchmarks-collectives-s3: -# <<: *publish-benchmarks -# variables: -# GIT_STRATEGY: none -# BUCKET: "releases.parity.io" -# PREFIX: "cumulus/$CI_COMMIT_REF_NAME/benchmarks-collectives" -# needs: -# - job: benchmarks-collectives -# artifacts: true - -### Polkadot build-push-image-polkadot-debug: stage: publish @@ -217,143 +164,3 @@ build-push-image-substrate-pr: variables: DOCKERFILE: "docker/dockerfiles/substrate_injected.Dockerfile" IMAGE_NAME: "docker.io/paritypr/substrate" -# old way - -# .build-push-image-polkadot: -# before_script: -# # - test -s ./artifacts/VERSION || exit 1 -# # - test -s ./artifacts/EXTRATAG || exit 1 -# - VERSION="$(cat ./artifacts/VERSION)" -# - EXTRATAG="$(cat ./artifacts/EXTRATAG)" -# - echo "Polkadot version = ${VERSION} (EXTRATAG = ${EXTRATAG})" -# script: -# # - test "$DOCKER_USER" -a "$DOCKER_PASS" || -# # ( echo "no docker credentials provided"; exit 1 ) -# - cd ./artifacts -# - $BUILDAH_COMMAND build -# --format=docker -# --build-arg VCS_REF="${CI_COMMIT_SHA}" -# --build-arg BUILD_DATE="$(date -u '+%Y-%m-%dT%H:%M:%SZ')" -# --build-arg IMAGE_NAME="${IMAGE_NAME}" -# --tag "$IMAGE_NAME:$VERSION" -# --tag "$IMAGE_NAME:$EXTRATAG" -# --file ${DOCKERFILE} . -# # The job will success only on the protected branch -# # - echo "$DOCKER_PASS" | -# # buildah login --username "$DOCKER_USER" --password-stdin docker.io -# # - $BUILDAH_COMMAND info -# # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$VERSION" -# # - $BUILDAH_COMMAND push --format=v2s2 "$IMAGE_NAME:$EXTRATAG" -# after_script: -# - buildah logout --all - -# publish-polkadot-debug-image: -# stage: publish -# image: ${BUILDAH_IMAGE} -# extends: -# - .kubernetes-env -# - .build-push-image-polkadot -# rules: -# - if: $CI_PIPELINE_SOURCE == "web" -# - if: $CI_PIPELINE_SOURCE == "schedule" -# - if: $CI_COMMIT_REF_NAME == "master" -# - if: $CI_COMMIT_REF_NAME =~ /^[0-9]+$/ # PRs -# - if: $CI_COMMIT_REF_NAME =~ /^v[0-9]+\.[0-9]+.*$/ # i.e. v1.0, v2.1rc1 -# variables: -# GIT_STRATEGY: none -# DOCKER_USER: ${PARITYPR_USER} -# DOCKER_PASS: ${PARITYPR_PASS} -# # docker/dockerfiles/polkadot/polkadot_injected_debug.Dockerfile -# DOCKERFILE: polkadot_injected_debug.Dockerfile -# IMAGE_NAME: docker.io/paritypr/polkadot-debug -# needs: -# - job: build-linux-stable -# artifacts: true -# after_script: -# # pass artifacts to the zombienet-tests job -# # https://docs.gitlab.com/ee/ci/multi_project_pipelines.html#with-variable-inheritance -# - echo "PARACHAINS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/parachains.env -# - echo "PARACHAINS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/parachains.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/parachains.env -# expire_in: 1 days - -# publish-test-collators-image: -# # service image for zombienet -# stage: publish -# extends: -# - .kubernetes-env -# - .build-push-image-polkadot -# - .zombienet-refs -# variables: -# CI_IMAGE: ${BUILDAH_IMAGE} -# GIT_STRATEGY: none -# DOCKER_USER: ${PARITYPR_USER} -# DOCKER_PASS: ${PARITYPR_PASS} -# # docker/dockerfiles/collator_injected.Dockerfile -# DOCKERFILE: collator_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/colander -# needs: -# - job: build-test-collators -# artifacts: true -# after_script: -# - buildah logout --all -# # pass artifacts to the zombienet-tests job -# - echo "COLLATOR_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/collator.env -# - echo "COLLATOR_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/collator.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/collator.env - -# publish-malus-image: -# # service image for Simnet -# stage: publish -# extends: -# - .kubernetes-env -# - .build-push-image-polkadot -# - .zombienet-refs -# variables: -# CI_IMAGE: ${BUILDAH_IMAGE} -# GIT_STRATEGY: none -# DOCKER_USER: ${PARITYPR_USER} -# DOCKER_PASS: ${PARITYPR_PASS} -# # docker/dockerfiles/malus_injected.Dockerfile -# DOCKERFILE: malus_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/malus -# needs: -# - job: build-malus -# artifacts: true -# after_script: -# - buildah logout "$IMAGE_NAME" -# # pass artifacts to the zombienet-tests job -# - echo "MALUS_IMAGE_NAME=${IMAGE_NAME}" > ./artifacts/malus.env -# - echo "MALUS_IMAGE_TAG=$(cat ./artifacts/EXTRATAG)" >> ./artifacts/malus.env -# artifacts: -# reports: -# # this artifact is used in zombienet-tests job -# dotenv: ./artifacts/malus.env - -# substrate - -# publish-substrate-image-pr: -# # service image for zombienet -# stage: publish -# extends: -# - .kubernetes-env -# - .build-push-image-polkadot -# - .zombienet-refs -# variables: -# CI_IMAGE: ${BUILDAH_IMAGE} -# GIT_STRATEGY: none -# DOCKER_USER: ${PARITYPR_USER} -# DOCKER_PASS: ${PARITYPR_PASS} -# DOCKERFILE: substrate_injected.Dockerfile -# IMAGE_NAME: docker.io/paritypr/substrate -# needs: -# - job: build-linux-substrate -# artifacts: true -# after_script: -# - buildah logout "$IMAGE_NAME" From 98f9e2ea9d5d034e85de7a87f6ba941827bc1a13 Mon Sep 17 00:00:00 2001 From: gupnik <17176722+gupnik@users.noreply.github.com> Date: Wed, 22 Nov 2023 17:32:49 +0530 Subject: [PATCH 34/48] Fixes import path in benchmark macro (#2437) Fully-qualified path was not being used in benchmark macro for one of the cases. This PR fixes this and removes the unnecessary import in a couple of files, which I believe was done to fix this issue. --- .../bin/node-template/pallets/template/src/benchmarking.rs | 1 - substrate/frame/asset-rate/src/benchmarking.rs | 1 - substrate/frame/support/procedural/src/benchmark.rs | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/substrate/bin/node-template/pallets/template/src/benchmarking.rs b/substrate/bin/node-template/pallets/template/src/benchmarking.rs index 6c3cae6066b4..5a262417629c 100644 --- a/substrate/bin/node-template/pallets/template/src/benchmarking.rs +++ b/substrate/bin/node-template/pallets/template/src/benchmarking.rs @@ -1,7 +1,6 @@ //! Benchmarking setup for pallet-template #![cfg(feature = "runtime-benchmarks")] use super::*; -use sp_std::vec; #[allow(unused)] use crate::Pallet as Template; diff --git a/substrate/frame/asset-rate/src/benchmarking.rs b/substrate/frame/asset-rate/src/benchmarking.rs index 21d53a89e397..6fcc7c7fadb2 100644 --- a/substrate/frame/asset-rate/src/benchmarking.rs +++ b/substrate/frame/asset-rate/src/benchmarking.rs @@ -25,7 +25,6 @@ use frame_benchmarking::v2::*; use frame_support::assert_ok; use frame_system::RawOrigin; use sp_core::crypto::FromEntropy; -use sp_std::vec; /// Trait describing the factory function for the `AssetKind` parameter. pub trait AssetKindFactory { diff --git a/substrate/frame/support/procedural/src/benchmark.rs b/substrate/frame/support/procedural/src/benchmark.rs index fb55e8c9f662..6ded82d91aa5 100644 --- a/substrate/frame/support/procedural/src/benchmark.rs +++ b/substrate/frame/support/procedural/src/benchmark.rs @@ -517,7 +517,7 @@ pub fn benchmarks( components, // TODO: Not supported by V2 syntax as of yet. // https://github.com/paritytech/substrate/issues/13132 - pov_modes: vec![], + pov_modes: #krate::__private::vec![], } }).collect::<#krate::__private::Vec<_>>() } From 408af9b32d95acbbac5e18bee66fd1b74230a699 Mon Sep 17 00:00:00 2001 From: Marcin S Date: Wed, 22 Nov 2023 15:45:52 +0100 Subject: [PATCH 35/48] PVF: Fix unshare "no such file or directory" error (#2426) --- .gitlab/pipeline/test.yml | 2 +- Cargo.lock | 1 + polkadot/node/core/pvf/Cargo.toml | 1 + polkadot/node/core/pvf/common/src/lib.rs | 2 +- polkadot/node/core/pvf/src/artifacts.rs | 5 +- .../node/core/pvf/src/execute/worker_intf.rs | 4 +- polkadot/node/core/pvf/src/host.rs | 9 +- .../node/core/pvf/src/prepare/worker_intf.rs | 4 +- polkadot/node/core/pvf/src/security.rs | 19 ++-- polkadot/node/core/pvf/src/worker_intf.rs | 105 ++++++++---------- polkadot/node/core/pvf/tests/it/main.rs | 32 ++++++ .../functional/0001-parachains-pvf.zndsl | 4 +- 12 files changed, 105 insertions(+), 83 deletions(-) diff --git a/.gitlab/pipeline/test.yml b/.gitlab/pipeline/test.yml index 4ed3ec19c48a..79ef9070dba9 100644 --- a/.gitlab/pipeline/test.yml +++ b/.gitlab/pipeline/test.yml @@ -29,7 +29,7 @@ test-linux-stable: --locked \ --release \ --no-fail-fast \ - --features try-runtime,experimental \ + --features try-runtime,experimental,ci-only-tests \ --partition count:${CI_NODE_INDEX}/${CI_NODE_TOTAL} # Upload tests results to Elasticsearch - echo "Upload test results to Elasticsearch" diff --git a/Cargo.lock b/Cargo.lock index 903e9463cf6c..bfe24e528ebe 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12591,6 +12591,7 @@ dependencies = [ "rand 0.8.5", "rococo-runtime", "rusty-fork", + "sc-sysinfo", "slotmap", "sp-core", "sp-maybe-compressed-blob", diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index 27da484fe4f1..05509a5f8536 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -54,6 +54,7 @@ halt = { package = "test-parachain-halt", path = "../../../parachain/test-parach [target.'cfg(target_os = "linux")'.dev-dependencies] procfs = "0.16.0" rusty-fork = "0.3.0" +sc-sysinfo = { path = "../../../../substrate/client/sysinfo" } [[bench]] name = "host_prepare_rococo_runtime" diff --git a/polkadot/node/core/pvf/common/src/lib.rs b/polkadot/node/core/pvf/common/src/lib.rs index 278fa3fe821c..c041c60aaf9c 100644 --- a/polkadot/node/core/pvf/common/src/lib.rs +++ b/polkadot/node/core/pvf/common/src/lib.rs @@ -47,7 +47,7 @@ pub mod tests { } /// Status of security features on the current system. -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, PartialEq, Eq)] pub struct SecurityStatus { /// Whether the landlock features we use are fully available on this system. pub can_enable_landlock: bool, diff --git a/polkadot/node/core/pvf/src/artifacts.rs b/polkadot/node/core/pvf/src/artifacts.rs index 53085eade3cb..79b53467b4e3 100644 --- a/polkadot/node/core/pvf/src/artifacts.rs +++ b/polkadot/node/core/pvf/src/artifacts.rs @@ -499,8 +499,7 @@ mod tests { #[tokio::test] async fn remove_stale_cache_on_startup() { - let cache_dir = crate::worker_intf::tmppath("test-cache").await.unwrap(); - fs::create_dir_all(&cache_dir).unwrap(); + let cache_dir = tempfile::Builder::new().prefix("test-cache-").tempdir().unwrap(); // invalid prefix create_rand_artifact(&cache_dir, ""); @@ -529,7 +528,7 @@ mod tests { assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 7); - let artifacts = Artifacts::new_and_prune(&cache_dir).await; + let artifacts = Artifacts::new_and_prune(cache_dir.path()).await; assert_eq!(fs::read_dir(&cache_dir).unwrap().count(), 1); assert_eq!(artifacts.len(), 1); diff --git a/polkadot/node/core/pvf/src/execute/worker_intf.rs b/polkadot/node/core/pvf/src/execute/worker_intf.rs index 4faaf13e62f7..fc6cd8fc7256 100644 --- a/polkadot/node/core/pvf/src/execute/worker_intf.rs +++ b/polkadot/node/core/pvf/src/execute/worker_intf.rs @@ -278,7 +278,7 @@ where // Cheaply create a hard link to the artifact. The artifact is always at a known location in the // worker cache, and the child can't access any other artifacts or gain any information from the // original filename. - let link_path = worker_dir::execute_artifact(&worker_dir.path); + let link_path = worker_dir::execute_artifact(worker_dir.path()); if let Err(err) = tokio::fs::hard_link(artifact_path, link_path).await { gum::warn!( target: LOG_TARGET, @@ -292,7 +292,7 @@ where } } - let worker_dir_path = worker_dir.path.clone(); + let worker_dir_path = worker_dir.path().to_owned(); let outcome = f(worker_dir).await; // Try to clear the worker dir. diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 6049f51834e3..be8f7aee7784 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -24,7 +24,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts}, execute::{self, PendingExecutionRequest}, metrics::Metrics, - prepare, security, Priority, ValidationError, LOG_TARGET, + prepare, security, Priority, SecurityStatus, ValidationError, LOG_TARGET, }; use always_assert::never; use futures::{ @@ -70,6 +70,8 @@ pub(crate) type PrecheckResultSender = oneshot::Sender; #[derive(Clone)] pub struct ValidationHost { to_host_tx: mpsc::Sender, + /// Available security features, detected by the host during startup. + pub security_status: SecurityStatus, } impl ValidationHost { @@ -216,7 +218,7 @@ pub async fn start( let (to_host_tx, to_host_rx) = mpsc::channel(10); - let validation_host = ValidationHost { to_host_tx }; + let validation_host = ValidationHost { to_host_tx, security_status: security_status.clone() }; let (to_prepare_pool, from_prepare_pool, run_prepare_pool) = prepare::start_pool( metrics.clone(), @@ -978,7 +980,8 @@ pub(crate) mod tests { fn host_handle(&mut self) -> ValidationHost { let to_host_tx = self.to_host_tx.take().unwrap(); - ValidationHost { to_host_tx } + let security_status = Default::default(); + ValidationHost { to_host_tx, security_status } } async fn poll_and_recv_result(&mut self, result_rx: oneshot::Receiver) -> T diff --git a/polkadot/node/core/pvf/src/prepare/worker_intf.rs b/polkadot/node/core/pvf/src/prepare/worker_intf.rs index 318aea7295de..f978005068c8 100644 --- a/polkadot/node/core/pvf/src/prepare/worker_intf.rs +++ b/polkadot/node/core/pvf/src/prepare/worker_intf.rs @@ -318,7 +318,7 @@ where { // Create the tmp file here so that the child doesn't need any file creation rights. This will // be cleared at the end of this function. - let tmp_file = worker_dir::prepare_tmp_artifact(&worker_dir.path); + let tmp_file = worker_dir::prepare_tmp_artifact(worker_dir.path()); if let Err(err) = tokio::fs::File::create(&tmp_file).await { gum::warn!( target: LOG_TARGET, @@ -333,7 +333,7 @@ where } }; - let worker_dir_path = worker_dir.path.clone(); + let worker_dir_path = worker_dir.path().to_owned(); let outcome = f(tmp_file, stream, worker_dir).await; // Try to clear the worker dir. diff --git a/polkadot/node/core/pvf/src/security.rs b/polkadot/node/core/pvf/src/security.rs index 8c06c68392f4..ef8714f58c81 100644 --- a/polkadot/node/core/pvf/src/security.rs +++ b/polkadot/node/core/pvf/src/security.rs @@ -28,7 +28,7 @@ const SECURE_MODE_ANNOUNCEMENT: &'static str = /// Run checks for supported security features. /// -/// # Return +/// # Returns /// /// Returns the set of security features that we were able to enable. If an error occurs while /// enabling a security feature we set the corresponding status to `false`. @@ -158,18 +158,15 @@ async fn check_can_unshare_user_namespace_and_change_root( ) -> SecureModeResult { cfg_if::cfg_if! { if #[cfg(target_os = "linux")] { - let cache_dir_tempdir = - crate::worker_intf::tmppath_in("check-can-unshare", cache_path) - .await - .map_err( - |err| - SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( - format!("could not create a temporary directory in {:?}: {}", cache_path, err) - ) - )?; + let cache_dir_tempdir = tempfile::Builder::new() + .prefix("check-can-unshare-") + .tempdir_in(cache_path) + .map_err(|err| SecureModeError::CannotUnshareUserNamespaceAndChangeRoot( + format!("could not create a temporary directory in {:?}: {}", cache_path, err) + ))?; match tokio::process::Command::new(prepare_worker_program_path) .arg("--check-can-unshare-user-namespace-and-change-root") - .arg(cache_dir_tempdir) + .arg(cache_dir_tempdir.path()) .output() .await { diff --git a/polkadot/node/core/pvf/src/worker_intf.rs b/polkadot/node/core/pvf/src/worker_intf.rs index 5e589b9abcee..9d6907c10929 100644 --- a/polkadot/node/core/pvf/src/worker_intf.rs +++ b/polkadot/node/core/pvf/src/worker_intf.rs @@ -71,6 +71,7 @@ pub async fn spawn_with_program_path( with_transient_socket_path(debug_id, |socket_path| { let socket_path = socket_path.to_owned(); + let worker_dir_path = worker_dir.path().to_owned(); async move { let listener = UnixListener::bind(&socket_path).map_err(|err| { @@ -91,7 +92,7 @@ pub async fn spawn_with_program_path( &program_path, &extra_args, &socket_path, - &worker_dir.path, + &worker_dir_path, security_status, ) .map_err(|err| { @@ -100,7 +101,7 @@ pub async fn spawn_with_program_path( %debug_id, ?program_path, ?extra_args, - ?worker_dir.path, + ?worker_dir_path, ?socket_path, "cannot spawn a worker: {:?}", err, @@ -108,7 +109,6 @@ pub async fn spawn_with_program_path( SpawnErr::ProcessSpawn })?; - let worker_dir_path = worker_dir.path.clone(); futures::select! { accept_result = listener.accept().fuse() => { let (stream, _) = accept_result.map_err(|err| { @@ -150,7 +150,42 @@ where F: FnOnce(&Path) -> Fut, Fut: futures::Future> + 'static, { - let socket_path = tmppath(&format!("pvf-host-{}", debug_id)) + /// Returns a path under [`std::env::temp_dir`]. The path name will start with the given prefix. + /// + /// There is only a certain number of retries. If exceeded this function will give up and return + /// an error. + pub async fn tmppath(prefix: &str) -> io::Result { + fn make_tmppath(prefix: &str, dir: &Path) -> PathBuf { + use rand::distributions::Alphanumeric; + + const DESCRIMINATOR_LEN: usize = 10; + + let mut buf = Vec::with_capacity(prefix.len() + DESCRIMINATOR_LEN); + buf.extend(prefix.as_bytes()); + buf.extend(rand::thread_rng().sample_iter(&Alphanumeric).take(DESCRIMINATOR_LEN)); + + let s = std::str::from_utf8(&buf) + .expect("the string is collected from a valid utf-8 sequence; qed"); + + let mut path = dir.to_owned(); + path.push(s); + path + } + + const NUM_RETRIES: usize = 50; + + let dir = std::env::temp_dir(); + for _ in 0..NUM_RETRIES { + let tmp_path = make_tmppath(prefix, &dir); + if !tmp_path.exists() { + return Ok(tmp_path) + } + } + + Err(io::Error::new(io::ErrorKind::Other, "failed to create a temporary path")) + } + + let socket_path = tmppath(&format!("pvf-host-{}-", debug_id)) .await .map_err(|_| SpawnErr::TmpPath)?; let result = f(&socket_path).await; @@ -162,46 +197,6 @@ where result } -/// Returns a path under the given `dir`. The path name will start with the given prefix. -/// -/// There is only a certain number of retries. If exceeded this function will give up and return an -/// error. -pub async fn tmppath_in(prefix: &str, dir: &Path) -> io::Result { - fn make_tmppath(prefix: &str, dir: &Path) -> PathBuf { - use rand::distributions::Alphanumeric; - - const DESCRIMINATOR_LEN: usize = 10; - - let mut buf = Vec::with_capacity(prefix.len() + DESCRIMINATOR_LEN); - buf.extend(prefix.as_bytes()); - buf.extend(rand::thread_rng().sample_iter(&Alphanumeric).take(DESCRIMINATOR_LEN)); - - let s = std::str::from_utf8(&buf) - .expect("the string is collected from a valid utf-8 sequence; qed"); - - let mut path = dir.to_owned(); - path.push(s); - path - } - - const NUM_RETRIES: usize = 50; - - for _ in 0..NUM_RETRIES { - let tmp_path = make_tmppath(prefix, dir); - if !tmp_path.exists() { - return Ok(tmp_path) - } - } - - Err(io::Error::new(io::ErrorKind::Other, "failed to create a temporary path")) -} - -/// The same as [`tmppath_in`], but uses [`std::env::temp_dir`] as the directory. -pub async fn tmppath(prefix: &str) -> io::Result { - let temp_dir = std::env::temp_dir(); - tmppath_in(prefix, &temp_dir).await -} - /// A struct that represents an idle worker. /// /// This struct is supposed to be used as a token that is passed by move into a subroutine that @@ -224,8 +219,6 @@ pub struct IdleWorker { pub enum SpawnErr { /// Cannot obtain a temporary path location. TmpPath, - /// An FS error occurred. - Fs(String), /// Cannot bind the socket to the given path. Bind, /// An error happened during accepting a connection to the socket. @@ -419,26 +412,22 @@ pub async fn framed_recv(r: &mut (impl AsyncRead + Unpin)) -> io::Result /// ``` #[derive(Debug)] pub struct WorkerDir { - pub path: PathBuf, + tempdir: tempfile::TempDir, } impl WorkerDir { /// Creates a new, empty worker dir with a random name in the given cache dir. pub async fn new(debug_id: &'static str, cache_dir: &Path) -> Result { let prefix = format!("worker-dir-{}-", debug_id); - let path = tmppath_in(&prefix, cache_dir).await.map_err(|_| SpawnErr::TmpPath)?; - tokio::fs::create_dir(&path) - .await - .map_err(|err| SpawnErr::Fs(err.to_string()))?; - Ok(Self { path }) + let tempdir = tempfile::Builder::new() + .prefix(&prefix) + .tempdir_in(cache_dir) + .map_err(|_| SpawnErr::TmpPath)?; + Ok(Self { tempdir }) } -} -// Try to clean up the temporary worker dir at the end of the worker's lifetime. It should be wiped -// on startup, but we make a best effort not to leave it around. -impl Drop for WorkerDir { - fn drop(&mut self) { - let _ = std::fs::remove_dir_all(&self.path); + pub fn path(&self) -> &Path { + self.tempdir.path() } } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 4c81ac502dd4..b53d598cd0f7 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -18,6 +18,8 @@ use assert_matches::assert_matches; use parity_scale_codec::Encode as _; +#[cfg(all(feature = "ci-only-tests", target_os = "linux"))] +use polkadot_node_core_pvf::SecurityStatus; use polkadot_node_core_pvf::{ start, testing::build_workers_and_get_paths, Config, InvalidCandidate, Metrics, PrepareError, PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, @@ -122,6 +124,11 @@ impl TestHost { .unwrap(); result_rx.await.unwrap() } + + #[cfg(all(feature = "ci-only-tests", target_os = "linux"))] + async fn security_status(&self) -> SecurityStatus { + self.host.lock().await.security_status.clone() + } } #[tokio::test] @@ -402,3 +409,28 @@ async fn prepare_can_run_serially() { // Prepare a different wasm blob to prevent skipping work. let _stats = host.precheck_pvf(halt::wasm_binary_unwrap(), Default::default()).await.unwrap(); } + +// CI machines should be able to enable all the security features. +#[cfg(all(feature = "ci-only-tests", target_os = "linux"))] +#[tokio::test] +async fn all_security_features_work() { + // Landlock is only available starting Linux 5.13, and we may be testing on an old kernel. + let sysinfo = sc_sysinfo::gather_sysinfo(); + // The version will look something like "5.15.0-87-generic". + let version = sysinfo.linux_kernel.unwrap(); + let version_split: Vec<&str> = version.split(".").collect(); + let major: u32 = version_split[0].parse().unwrap(); + let minor: u32 = version_split[1].parse().unwrap(); + let can_enable_landlock = if major >= 6 { true } else { minor >= 13 }; + + let host = TestHost::new().await; + + assert_eq!( + host.security_status().await, + SecurityStatus { + can_enable_landlock, + can_enable_seccomp: true, + can_unshare_user_namespace_and_change_root: true, + } + ); +} diff --git a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl index 46bb8bcdf72b..135999a092a7 100644 --- a/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl +++ b/polkadot/zombienet_tests/functional/0001-parachains-pvf.zndsl @@ -54,8 +54,8 @@ one: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets [" two: reports histogram polkadot_pvf_preparation_time has 0 samples in buckets ["20", "30", "60", "120", "+Inf"] within 10 seconds # Check execution time. -# There are two different timeout conditions: BACKING_EXECUTION_TIMEOUT(2s) and -# APPROVAL_EXECUTION_TIMEOUT(6s). Currently these are not differentiated by metrics +# There are two different timeout conditions: DEFAULT_BACKING_EXECUTION_TIMEOUT(2s) and +# DEFAULT_APPROVAL_EXECUTION_TIMEOUT(12s). Currently these are not differentiated by metrics # because the metrics are defined in `polkadot-node-core-pvf` which is a level below # the relevant subsystems. # That being said, we will take the simplifying assumption of testing only the From 0d6dcb3f48559eff24b3a4e2233fc31e145717d2 Mon Sep 17 00:00:00 2001 From: James Wilson Date: Wed, 22 Nov 2023 17:27:53 +0000 Subject: [PATCH 36/48] Make TypeInfo for SkipCheckIfFeeless transparent, too (#2449) The `SkipCheckifFeeless::IDENTIFIER` became transparent (ie was whatever the inner signed ext was). This PR just makes the `TypeInfo` transparent too, so that libraries that use said info to decode the data (ie subxt) can behave identically whether or not the `SkipCheckifFeeless` wrapper is used or not. --------- Co-authored-by: Oliver Tale-Yazdi --- .../skip-feeless-payment/src/lib.rs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs index 55afaaf93d78..6c34c26ce923 100644 --- a/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs +++ b/substrate/frame/transaction-payment/skip-feeless-payment/src/lib.rs @@ -40,7 +40,7 @@ use frame_support::{ dispatch::{CheckIfFeeless, DispatchResult}, traits::{IsType, OriginTrait}, }; -use scale_info::TypeInfo; +use scale_info::{StaticTypeInfo, TypeInfo}; use sp_runtime::{ traits::{DispatchInfoOf, PostDispatchInfoOf, SignedExtension}, transaction_validity::TransactionValidityError, @@ -75,10 +75,17 @@ pub mod pallet { } /// A [`SignedExtension`] that skips the wrapped extension if the dispatchable is feeless. -#[derive(Encode, Decode, Clone, Eq, PartialEq, TypeInfo)] -#[scale_info(skip_type_params(T))] +#[derive(Encode, Decode, Clone, Eq, PartialEq)] pub struct SkipCheckIfFeeless(pub S, sp_std::marker::PhantomData); +// Make this extension "invisible" from the outside (ie metadata type information) +impl TypeInfo for SkipCheckIfFeeless { + type Identity = S; + fn type_info() -> scale_info::Type { + S::type_info() + } +} + impl sp_std::fmt::Debug for SkipCheckIfFeeless { #[cfg(feature = "std")] fn fmt(&self, f: &mut sp_std::fmt::Formatter) -> sp_std::fmt::Result { From 0956357b602af1bb10f9bd6fb3998b856bd07592 Mon Sep 17 00:00:00 2001 From: girazoki Date: Wed, 22 Nov 2023 18:29:06 +0100 Subject: [PATCH 37/48] work with additional key values (#1757) Add the possibility to inject additional key-values in the sproof-builder that generates the relay root that gets stored in parachain-system. Rationale: pallets that verify additional storage items (not those verified by parachain-system) from the relay should be able to proof against the relay root that gets stored in parachain-system. This PR allows to create provide additional nibles that can later be used for verifiability in other pallets --- cumulus/primitives/parachain-inherent/src/mock.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cumulus/primitives/parachain-inherent/src/mock.rs b/cumulus/primitives/parachain-inherent/src/mock.rs index 5168b46a14d0..e40cb49acddd 100644 --- a/cumulus/primitives/parachain-inherent/src/mock.rs +++ b/cumulus/primitives/parachain-inherent/src/mock.rs @@ -61,6 +61,8 @@ pub struct MockValidationDataInherentDataProvider { pub raw_downward_messages: Vec>, // Inbound Horizontal messages sorted by channel pub raw_horizontal_messages: Vec<(ParaId, Vec)>, + // Additional key-value pairs that should be injected. + pub additional_key_values: Option, Vec)>>, } pub trait GenerateRandomness { @@ -210,6 +212,10 @@ impl> InherentDataProvider sproof_builder.randomness = self.relay_randomness_config.generate_randomness(self.current_para_block.into()); + if let Some(key_values) = &self.additional_key_values { + sproof_builder.additional_key_values = key_values.clone() + } + let (relay_parent_storage_root, proof) = sproof_builder.into_state_root_and_proof(); inherent_data.put_data( From 49874882cf3eadde01951ee48916c9811d6a8c7e Mon Sep 17 00:00:00 2001 From: Michal Kucharczyk <1728078+michalkucharczyk@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:24:11 +0100 Subject: [PATCH 38/48] polkadot-node-subsystems: `ChainApiBackend` added + `polkadot-debug` image version fixed (#2411) The out-dated version (bad tag) of [polkadot image](https://github.com/paritytech/polkadot-sdk/blob/ede4a362622dfa69eb167eaa876246b1289f4b41/.gitlab/pipeline/zombienet/cumulus.yml#L31) ([docker info](https://hub.docker.com/layers/paritypr/polkadot-debug/master/images/sha256:adb1658052cf671b50c90d5cece5c7a131efa1a95978249bd5cb85a5ad654f7a?context=explore)) was used. This PR fixes this. See also: https://github.com/paritytech/polkadot-sdk/pull/2411#issuecomment-1822632724 Also adds an abstraction that allows asynchronous backends to be passed to `ChainApiSubsystem` --------- Co-authored-by: Sebastian Kunert --- .gitlab/pipeline/zombienet/cumulus.yml | 2 +- Cargo.lock | 6 ++ .../relay-chain-minimal-node/Cargo.toml | 4 + .../src/blockchain_rpc_client.rs | 86 ++++++++++++++++++- .../src/collator_overseer.rs | 3 +- polkadot/node/core/chain-api/Cargo.toml | 5 +- polkadot/node/core/chain-api/src/lib.rs | 79 +++++++++-------- polkadot/node/core/chain-api/src/tests.rs | 21 +++-- polkadot/node/overseer/src/lib.rs | 4 +- polkadot/node/subsystem-types/Cargo.toml | 1 + polkadot/node/subsystem-types/src/lib.rs | 2 +- .../subsystem-types/src/runtime_client.rs | 57 +++++++++++- 12 files changed, 214 insertions(+), 56 deletions(-) diff --git a/.gitlab/pipeline/zombienet/cumulus.yml b/.gitlab/pipeline/zombienet/cumulus.yml index 3cac67c2966e..3e4df000b7f7 100644 --- a/.gitlab/pipeline/zombienet/cumulus.yml +++ b/.gitlab/pipeline/zombienet/cumulus.yml @@ -28,7 +28,7 @@ - job: build-push-image-test-parachain artifacts: true variables: - POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:master" + POLKADOT_IMAGE: "docker.io/paritypr/polkadot-debug:${DOCKER_IMAGES_VERSION}" GH_DIR: "https://github.com/paritytech/cumulus/tree/${CI_COMMIT_SHORT_SHA}/zombienet/tests" LOCAL_DIR: "/builds/parity/mirrors/polkadot-sdk/cumulus/zombienet/tests" COL_IMAGE: "docker.io/paritypr/test-parachain:${DOCKER_IMAGES_VERSION}" diff --git a/Cargo.lock b/Cargo.lock index bfe24e528ebe..26da3b4f8252 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4183,6 +4183,7 @@ dependencies = [ "polkadot-core-primitives", "polkadot-network-bridge", "polkadot-node-collation-generation", + "polkadot-node-core-chain-api", "polkadot-node-core-prospective-parachains", "polkadot-node-core-runtime-api", "polkadot-node-network-protocol", @@ -4190,16 +4191,19 @@ dependencies = [ "polkadot-overseer", "polkadot-primitives", "sc-authority-discovery", + "sc-client-api", "sc-network", "sc-network-common", "sc-service", "sc-tracing", "sc-utils", "sp-api", + "sp-blockchain", "sp-consensus", "sp-consensus-babe", "sp-runtime", "substrate-prometheus-endpoint", + "tokio", "tracing", ] @@ -12443,6 +12447,7 @@ dependencies = [ "polkadot-node-primitives", "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", + "polkadot-node-subsystem-types", "polkadot-primitives", "sc-client-api", "sc-consensus-babe", @@ -12851,6 +12856,7 @@ dependencies = [ "smallvec", "sp-api", "sp-authority-discovery", + "sp-blockchain", "sp-consensus-babe", "substrate-prometheus-endpoint", "thiserror", diff --git a/cumulus/client/relay-chain-minimal-node/Cargo.toml b/cumulus/client/relay-chain-minimal-node/Cargo.toml index 53173fb41189..ce76fc5cd6d2 100644 --- a/cumulus/client/relay-chain-minimal-node/Cargo.toml +++ b/cumulus/client/relay-chain-minimal-node/Cargo.toml @@ -19,6 +19,7 @@ polkadot-collator-protocol = { path = "../../../polkadot/node/network/collator-p polkadot-network-bridge = { path = "../../../polkadot/node/network/bridge" } polkadot-node-collation-generation = { path = "../../../polkadot/node/collation-generation" } polkadot-node-core-runtime-api = { path = "../../../polkadot/node/core/runtime-api" } +polkadot-node-core-chain-api = { path = "../../../polkadot/node/core/chain-api" } polkadot-node-core-prospective-parachains = { path = "../../../polkadot/node/core/prospective-parachains" } # substrate deps @@ -26,6 +27,7 @@ sc-authority-discovery = { path = "../../../substrate/client/authority-discovery sc-network = { path = "../../../substrate/client/network" } sc-network-common = { path = "../../../substrate/client/network/common" } sc-service = { path = "../../../substrate/client/service" } +sc-client-api = { path = "../../../substrate/client/api" } substrate-prometheus-endpoint = { path = "../../../substrate/utils/prometheus" } sc-tracing = { path = "../../../substrate/client/tracing" } sc-utils = { path = "../../../substrate/client/utils" } @@ -33,6 +35,8 @@ sp-api = { path = "../../../substrate/primitives/api" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } sp-consensus = { path = "../../../substrate/primitives/consensus/common" } sp-runtime = { path = "../../../substrate/primitives/runtime" } +sp-blockchain = { path = "../../../substrate/primitives/blockchain" } +tokio = { version = "1.32.0", features = ["macros"] } # cumulus deps cumulus-relay-chain-interface = { path = "../relay-chain-interface" } diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index a473b3bced02..c40ca5c858ba 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -20,14 +20,16 @@ use cumulus_relay_chain_interface::{RelayChainError, RelayChainResult}; use cumulus_relay_chain_rpc_interface::RelayChainRpcClient; use futures::{Stream, StreamExt}; use polkadot_core_primitives::{Block, BlockNumber, Hash, Header}; -use polkadot_overseer::RuntimeApiSubsystemClient; +use polkadot_overseer::{ChainApiBackend, RuntimeApiSubsystemClient}; use polkadot_primitives::{ async_backing::{AsyncBackingParams, BackingState}, slashing, vstaging::NodeFeatures, }; use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError}; -use sp_api::{ApiError, RuntimeApiInfo}; +use sc_client_api::AuxStore; +use sp_api::{ApiError, BlockT, HeaderT, NumberFor, RuntimeApiInfo}; +use sp_blockchain::Info; #[derive(Clone)] pub struct BlockChainRpcClient { @@ -54,6 +56,64 @@ impl BlockChainRpcClient { } } +#[async_trait::async_trait] +impl ChainApiBackend for BlockChainRpcClient { + async fn header( + &self, + hash: ::Hash, + ) -> sp_blockchain::Result::Header>> { + Ok(self.rpc_client.chain_get_header(Some(hash)).await?) + } + + async fn info(&self) -> sp_blockchain::Result> { + let (best_header_opt, genesis_hash, finalized_head) = futures::try_join!( + self.rpc_client.chain_get_header(None), + self.rpc_client.chain_get_head(Some(0)), + self.rpc_client.chain_get_finalized_head() + )?; + let best_header = best_header_opt.ok_or_else(|| { + RelayChainError::GenericError( + "Unable to retrieve best header from relay chain.".to_string(), + ) + })?; + + let finalized_header = + self.rpc_client.chain_get_header(Some(finalized_head)).await?.ok_or_else(|| { + RelayChainError::GenericError( + "Unable to retrieve finalized header from relay chain.".to_string(), + ) + })?; + Ok(Info { + best_hash: best_header.hash(), + best_number: best_header.number, + genesis_hash, + finalized_hash: finalized_head, + finalized_number: finalized_header.number, + finalized_state: Some((finalized_header.hash(), finalized_header.number)), + number_leaves: 1, + block_gap: None, + }) + } + + async fn number( + &self, + hash: ::Hash, + ) -> sp_blockchain::Result::Header as HeaderT>::Number>> { + Ok(self + .rpc_client + .chain_get_header(Some(hash)) + .await? + .map(|maybe_header| maybe_header.number)) + } + + async fn hash( + &self, + number: NumberFor, + ) -> sp_blockchain::Result::Hash>> { + Ok(self.rpc_client.chain_get_block_hash(number.into()).await?) + } +} + #[async_trait::async_trait] impl RuntimeApiSubsystemClient for BlockChainRpcClient { async fn validators( @@ -403,3 +463,25 @@ impl BlockChainRpcClient { Ok(self.rpc_client.get_finalized_heads_stream()?.boxed()) } } + +// Implementation required by ChainApiSubsystem +// but never called in our case. +impl AuxStore for BlockChainRpcClient { + fn insert_aux< + 'a, + 'b: 'a, + 'c: 'a, + I: IntoIterator, + D: IntoIterator, + >( + &self, + _insert: I, + _delete: D, + ) -> sp_blockchain::Result<()> { + unimplemented!("Not supported on the RPC collator") + } + + fn get_aux(&self, _key: &[u8]) -> sp_blockchain::Result>> { + unimplemented!("Not supported on the RPC collator") + } +} diff --git a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs index 945344f85e97..a785a9f6f79c 100644 --- a/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs +++ b/cumulus/client/relay-chain-minimal-node/src/collator_overseer.rs @@ -24,6 +24,7 @@ use polkadot_network_bridge::{ NetworkBridgeTx as NetworkBridgeTxSubsystem, }; use polkadot_node_collation_generation::CollationGenerationSubsystem; +use polkadot_node_core_chain_api::ChainApiSubsystem; use polkadot_node_core_prospective_parachains::ProspectiveParachainsSubsystem; use polkadot_node_core_runtime_api::RuntimeApiSubsystem; use polkadot_node_network_protocol::{ @@ -112,7 +113,7 @@ fn build_overseer( .candidate_backing(DummySubsystem) .candidate_validation(DummySubsystem) .pvf_checker(DummySubsystem) - .chain_api(DummySubsystem) + .chain_api(ChainApiSubsystem::new(runtime_client.clone(), Metrics::register(registry)?)) .collation_generation(CollationGenerationSubsystem::new(Metrics::register(registry)?)) .collator_protocol({ let side = ProtocolSide::Collator { diff --git a/polkadot/node/core/chain-api/Cargo.toml b/polkadot/node/core/chain-api/Cargo.toml index 154fa20e75d0..fa824e78ffee 100644 --- a/polkadot/node/core/chain-api/Cargo.toml +++ b/polkadot/node/core/chain-api/Cargo.toml @@ -9,10 +9,9 @@ description = "The Chain API subsystem provides access to chain related utility [dependencies] futures = "0.3.21" gum = { package = "tracing-gum", path = "../../gum" } -sp-blockchain = { path = "../../../../substrate/primitives/blockchain" } -polkadot-primitives = { path = "../../../primitives" } polkadot-node-metrics = { path = "../../metrics" } polkadot-node-subsystem = { path = "../../subsystem" } +polkadot-node-subsystem-types = { path = "../../subsystem-types" } sc-client-api = { path = "../../../../substrate/client/api" } sc-consensus-babe = { path = "../../../../substrate/client/consensus/babe" } @@ -21,5 +20,7 @@ futures = { version = "0.3.21", features = ["thread-pool"] } maplit = "1.0.2" parity-scale-codec = "3.6.1" polkadot-node-primitives = { path = "../../primitives" } +polkadot-primitives = { path = "../../../primitives" } polkadot-node-subsystem-test-helpers = { path = "../../subsystem-test-helpers" } sp-core = { path = "../../../../substrate/primitives/core" } +sp-blockchain = { path = "../../../../substrate/primitives/blockchain" } diff --git a/polkadot/node/core/chain-api/src/lib.rs b/polkadot/node/core/chain-api/src/lib.rs index 9b25481d7186..7fd5166310fe 100644 --- a/polkadot/node/core/chain-api/src/lib.rs +++ b/polkadot/node/core/chain-api/src/lib.rs @@ -35,13 +35,13 @@ use std::sync::Arc; use futures::prelude::*; use sc_client_api::AuxStore; -use sp_blockchain::HeaderBackend; +use futures::stream::StreamExt; use polkadot_node_subsystem::{ messages::ChainApiMessage, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, }; -use polkadot_primitives::Block; +use polkadot_node_subsystem_types::ChainApiBackend; mod metrics; use self::metrics::Metrics; @@ -67,7 +67,7 @@ impl ChainApiSubsystem { #[overseer::subsystem(ChainApi, error = SubsystemError, prefix = self::overseer)] impl ChainApiSubsystem where - Client: HeaderBackend + AuxStore + 'static, + Client: ChainApiBackend + AuxStore + 'static, { fn start(self, ctx: Context) -> SpawnedSubsystem { let future = run::(ctx, self) @@ -83,7 +83,7 @@ async fn run( subsystem: ChainApiSubsystem, ) -> SubsystemResult<()> where - Client: HeaderBackend + AuxStore, + Client: ChainApiBackend + AuxStore, { loop { match ctx.recv().await? { @@ -93,13 +93,15 @@ where FromOrchestra::Communication { msg } => match msg { ChainApiMessage::BlockNumber(hash, response_channel) => { let _timer = subsystem.metrics.time_block_number(); - let result = subsystem.client.number(hash).map_err(|e| e.to_string().into()); + let result = + subsystem.client.number(hash).await.map_err(|e| e.to_string().into()); subsystem.metrics.on_request(result.is_ok()); let _ = response_channel.send(result); }, ChainApiMessage::BlockHeader(hash, response_channel) => { let _timer = subsystem.metrics.time_block_header(); - let result = subsystem.client.header(hash).map_err(|e| e.to_string().into()); + let result = + subsystem.client.header(hash).await.map_err(|e| e.to_string().into()); subsystem.metrics.on_request(result.is_ok()); let _ = response_channel.send(result); }, @@ -113,46 +115,51 @@ where ChainApiMessage::FinalizedBlockHash(number, response_channel) => { let _timer = subsystem.metrics.time_finalized_block_hash(); // Note: we don't verify it's finalized - let result = subsystem.client.hash(number).map_err(|e| e.to_string().into()); + let result = + subsystem.client.hash(number).await.map_err(|e| e.to_string().into()); subsystem.metrics.on_request(result.is_ok()); let _ = response_channel.send(result); }, ChainApiMessage::FinalizedBlockNumber(response_channel) => { let _timer = subsystem.metrics.time_finalized_block_number(); - let result = subsystem.client.info().finalized_number; - // always succeeds - subsystem.metrics.on_request(true); - let _ = response_channel.send(Ok(result)); + let result = subsystem + .client + .info() + .await + .map_err(|e| e.to_string().into()) + .map(|info| info.finalized_number); + subsystem.metrics.on_request(result.is_ok()); + let _ = response_channel.send(result); }, ChainApiMessage::Ancestors { hash, k, response_channel } => { let _timer = subsystem.metrics.time_ancestors(); gum::trace!(target: LOG_TARGET, hash=%hash, k=k, "ChainApiMessage::Ancestors"); - let mut hash = hash; - - let next_parent = core::iter::from_fn(|| { - let maybe_header = subsystem.client.header(hash); - match maybe_header { - // propagate the error - Err(e) => { - let e = e.to_string().into(); - Some(Err(e)) - }, - // fewer than `k` ancestors are available - Ok(None) => None, - Ok(Some(header)) => { - // stop at the genesis header. - if header.number == 0 { - None - } else { - hash = header.parent_hash; - Some(Ok(hash)) - } - }, - } - }); - - let result = next_parent.take(k).collect::, _>>(); + let next_parent_stream = futures::stream::unfold( + (hash, subsystem.client.clone()), + |(hash, client)| async move { + let maybe_header = client.header(hash).await; + match maybe_header { + // propagate the error + Err(e) => { + let e = e.to_string().into(); + Some((Err(e), (hash, client))) + }, + // fewer than `k` ancestors are available + Ok(None) => None, + Ok(Some(header)) => { + // stop at the genesis header. + if header.number == 0 { + None + } else { + Some((Ok(header.parent_hash), (header.parent_hash, client))) + } + }, + } + }, + ); + + let result = next_parent_stream.take(k).try_collect().await; subsystem.metrics.on_request(result.is_ok()); let _ = response_channel.send(result); }, diff --git a/polkadot/node/core/chain-api/src/tests.rs b/polkadot/node/core/chain-api/src/tests.rs index 331a4f9ba820..eae8f6fa4ac5 100644 --- a/polkadot/node/core/chain-api/src/tests.rs +++ b/polkadot/node/core/chain-api/src/tests.rs @@ -22,7 +22,8 @@ use std::collections::BTreeMap; use polkadot_node_primitives::BlockWeight; use polkadot_node_subsystem_test_helpers::{make_subsystem_context, TestSubsystemContextHandle}; -use polkadot_primitives::{BlockNumber, Hash, Header}; +use polkadot_node_subsystem_types::ChainApiBackend; +use polkadot_primitives::{Block, BlockNumber, Hash, Header}; use sp_blockchain::Info as BlockInfo; use sp_core::testing::TaskExecutor; @@ -110,7 +111,7 @@ fn last_key_value(map: &BTreeMap) -> (K, V) { map.iter().last().map(|(k, v)| (k.clone(), v.clone())).unwrap() } -impl HeaderBackend for TestClient { +impl sp_blockchain::HeaderBackend for TestClient { fn info(&self) -> BlockInfo { let genesis_hash = self.blocks.iter().next().map(|(h, _)| *h).unwrap(); let (best_hash, best_number) = last_key_value(&self.blocks); @@ -191,8 +192,8 @@ fn request_block_number() { async move { let zero = Hash::zero(); let test_cases = [ - (TWO, client.number(TWO).unwrap()), - (zero, client.number(zero).unwrap()), // not here + (TWO, client.number(TWO).await.unwrap()), + (zero, client.number(zero).await.unwrap()), // not here ]; for (hash, expected) in &test_cases { let (tx, rx) = oneshot::channel(); @@ -217,8 +218,10 @@ fn request_block_header() { test_harness(|client, mut sender| { async move { const NOT_HERE: Hash = Hash::repeat_byte(0x5); - let test_cases = - [(TWO, client.header(TWO).unwrap()), (NOT_HERE, client.header(NOT_HERE).unwrap())]; + let test_cases = [ + (TWO, client.header(TWO).await.unwrap()), + (NOT_HERE, client.header(NOT_HERE).await.unwrap()), + ]; for (hash, expected) in &test_cases { let (tx, rx) = oneshot::channel(); @@ -270,8 +273,8 @@ fn request_finalized_hash() { test_harness(|client, mut sender| { async move { let test_cases = [ - (1, client.hash(1).unwrap()), // not here - (2, client.hash(2).unwrap()), + (1, client.hash(1).await.unwrap()), // not here + (2, client.hash(2).await.unwrap()), ]; for (number, expected) in &test_cases { let (tx, rx) = oneshot::channel(); @@ -297,7 +300,7 @@ fn request_last_finalized_number() { async move { let (tx, rx) = oneshot::channel(); - let expected = client.info().finalized_number; + let expected = client.info().await.unwrap().finalized_number; sender .send(FromOrchestra::Communication { msg: ChainApiMessage::FinalizedBlockNumber(tx), diff --git a/polkadot/node/overseer/src/lib.rs b/polkadot/node/overseer/src/lib.rs index 5207bb830d8c..da99546a44f7 100644 --- a/polkadot/node/overseer/src/lib.rs +++ b/polkadot/node/overseer/src/lib.rs @@ -87,8 +87,8 @@ use polkadot_node_subsystem_types::messages::{ pub use polkadot_node_subsystem_types::{ errors::{SubsystemError, SubsystemResult}, - jaeger, ActivatedLeaf, ActiveLeavesUpdate, OverseerSignal, RuntimeApiSubsystemClient, - UnpinHandle, + jaeger, ActivatedLeaf, ActiveLeavesUpdate, ChainApiBackend, OverseerSignal, + RuntimeApiSubsystemClient, UnpinHandle, }; pub mod metrics; diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index 9fd3775da591..8e345cf222c6 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -17,6 +17,7 @@ polkadot-node-jaeger = { path = "../jaeger" } orchestra = { version = "0.3.3", default-features = false, features=["futures_channel"] } sc-network = { path = "../../../substrate/client/network" } sp-api = { path = "../../../substrate/primitives/api" } +sp-blockchain = { path = "../../../substrate/primitives/blockchain" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } sp-authority-discovery = { path = "../../../substrate/primitives/authority-discovery" } sc-client-api = { path = "../../../substrate/client/api" } diff --git a/polkadot/node/subsystem-types/src/lib.rs b/polkadot/node/subsystem-types/src/lib.rs index e3d6e4decf20..cd39aa03e567 100644 --- a/polkadot/node/subsystem-types/src/lib.rs +++ b/polkadot/node/subsystem-types/src/lib.rs @@ -40,7 +40,7 @@ pub mod errors; pub mod messages; mod runtime_client; -pub use runtime_client::{DefaultSubsystemClient, RuntimeApiSubsystemClient}; +pub use runtime_client::{ChainApiBackend, DefaultSubsystemClient, RuntimeApiSubsystemClient}; pub use jaeger::*; pub use polkadot_node_jaeger as jaeger; diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 8369fd215f4f..36e3365cf08c 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -18,17 +18,70 @@ use async_trait::async_trait; use polkadot_primitives::{ async_backing, runtime_api::ParachainHost, slashing, vstaging, Block, BlockNumber, CandidateCommitments, CandidateEvent, CandidateHash, CommittedCandidateReceipt, CoreState, - DisputeState, ExecutorParams, GroupRotationInfo, Hash, Id, InboundDownwardMessage, + DisputeState, ExecutorParams, GroupRotationInfo, Hash, Header, Id, InboundDownwardMessage, InboundHrmpMessage, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, ScrapedOnChainVotes, SessionIndex, SessionInfo, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, }; +use sc_client_api::HeaderBackend; use sc_transaction_pool_api::OffchainTransactionPoolFactory; -use sp_api::{ApiError, ApiExt, ProvideRuntimeApi}; +use sp_api::{ApiError, ApiExt, HeaderT, NumberFor, ProvideRuntimeApi}; use sp_authority_discovery::AuthorityDiscoveryApi; +use sp_blockchain::Info; use sp_consensus_babe::{BabeApi, Epoch}; use std::{collections::BTreeMap, sync::Arc}; +/// Offers header utilities. +/// +/// This is a async wrapper trait for ['HeaderBackend'] to be used with the +/// `ChainApiSubsystem`. +// This trait was introduced to suit the needs of collators. Depending on their operating mode, they +// might not have a client of the relay chain that can supply a synchronous HeaderBackend +// implementation. +#[async_trait] +pub trait ChainApiBackend: Send + Sync { + /// Get block header. Returns `None` if block is not found. + async fn header(&self, hash: Hash) -> sp_blockchain::Result>; + /// Get blockchain info. + async fn info(&self) -> sp_blockchain::Result>; + /// Get block number by hash. Returns `None` if the header is not in the chain. + async fn number( + &self, + hash: Hash, + ) -> sp_blockchain::Result::Number>>; + /// Get block hash by number. Returns `None` if the header is not in the chain. + async fn hash(&self, number: NumberFor) -> sp_blockchain::Result>; +} + +#[async_trait] +impl ChainApiBackend for T +where + T: HeaderBackend, +{ + /// Get block header. Returns `None` if block is not found. + async fn header(&self, hash: Hash) -> sp_blockchain::Result> { + HeaderBackend::header(self, hash) + } + + /// Get blockchain info. + async fn info(&self) -> sp_blockchain::Result> { + Ok(HeaderBackend::info(self)) + } + + /// Get block number by hash. Returns `None` if the header is not in the chain. + async fn number( + &self, + hash: Hash, + ) -> sp_blockchain::Result::Number>> { + HeaderBackend::number(self, hash) + } + + /// Get block hash by number. Returns `None` if the header is not in the chain. + async fn hash(&self, number: NumberFor) -> sp_blockchain::Result> { + HeaderBackend::hash(self, number) + } +} + /// Exposes all runtime calls that are used by the runtime API subsystem. #[async_trait] pub trait RuntimeApiSubsystemClient { From ec189333845fffebd45bf023b0797d6a4f121d8d Mon Sep 17 00:00:00 2001 From: Joshy Orndorff Date: Wed, 22 Nov 2023 17:21:20 -0500 Subject: [PATCH 39/48] Remove `#[macro_use]` annotation from `mod service` in all nodes. (#2456) This PR removes `#[macro_use]` from the service module in each of the Substrate nodes in the repo. * Parachain Template * Polkadot Parachain * Minimal Node * Node Template * Kitchen Sink Node IDK why this annotation was present, maybe from when we had the `new_partial!` macro? --------- Co-authored-by: Joshy Orndorff --- cumulus/parachain-template/node/src/main.rs | 3 +-- cumulus/polkadot-parachain/src/main.rs | 3 +-- substrate/bin/minimal/node/src/main.rs | 3 +-- substrate/bin/node-template/node/src/main.rs | 5 ++--- substrate/bin/node/cli/src/lib.rs | 6 ++---- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/cumulus/parachain-template/node/src/main.rs b/cumulus/parachain-template/node/src/main.rs index ba9f28b354f1..12738a6793c0 100644 --- a/cumulus/parachain-template/node/src/main.rs +++ b/cumulus/parachain-template/node/src/main.rs @@ -3,11 +3,10 @@ #![warn(missing_docs)] mod chain_spec; -#[macro_use] -mod service; mod cli; mod command; mod rpc; +mod service; fn main() -> sc_cli::Result<()> { command::run() diff --git a/cumulus/polkadot-parachain/src/main.rs b/cumulus/polkadot-parachain/src/main.rs index e40af8128f71..26d7dae4b8a0 100644 --- a/cumulus/polkadot-parachain/src/main.rs +++ b/cumulus/polkadot-parachain/src/main.rs @@ -20,11 +20,10 @@ #![warn(unused_extern_crates)] mod chain_spec; -#[macro_use] -mod service; mod cli; mod command; mod rpc; +mod service; fn main() -> sc_cli::Result<()> { command::run() diff --git a/substrate/bin/minimal/node/src/main.rs b/substrate/bin/minimal/node/src/main.rs index 900651fd1fdb..3cf7d98311ea 100644 --- a/substrate/bin/minimal/node/src/main.rs +++ b/substrate/bin/minimal/node/src/main.rs @@ -19,11 +19,10 @@ #![warn(missing_docs)] mod chain_spec; -#[macro_use] -mod service; mod cli; mod command; mod rpc; +mod service; fn main() -> sc_cli::Result<()> { command::run() diff --git a/substrate/bin/node-template/node/src/main.rs b/substrate/bin/node-template/node/src/main.rs index 426cbabb6fbf..8918dd43a011 100644 --- a/substrate/bin/node-template/node/src/main.rs +++ b/substrate/bin/node-template/node/src/main.rs @@ -1,13 +1,12 @@ //! Substrate Node Template CLI library. #![warn(missing_docs)] -mod chain_spec; -#[macro_use] -mod service; mod benchmarking; +mod chain_spec; mod cli; mod command; mod rpc; +mod service; fn main() -> sc_cli::Result<()> { command::run() diff --git a/substrate/bin/node/cli/src/lib.rs b/substrate/bin/node/cli/src/lib.rs index 2fe238ef316e..0ff544932a9a 100644 --- a/substrate/bin/node/cli/src/lib.rs +++ b/substrate/bin/node/cli/src/lib.rs @@ -30,16 +30,14 @@ #![warn(missing_docs)] -pub mod chain_spec; - -#[macro_use] -pub mod service; #[cfg(feature = "cli")] mod benchmarking; +pub mod chain_spec; #[cfg(feature = "cli")] mod cli; #[cfg(feature = "cli")] mod command; +pub mod service; #[cfg(feature = "cli")] pub use cli::*; From 2d09e83d0703ca6bf6aba773e80ea14576887ac7 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 23 Nov 2023 15:18:24 +0400 Subject: [PATCH 40/48] Add `on-chain-release-build` feature for Collectives Westend (#2463) Collectives Westend is missing an `on-chain-release-build` feature flag. --- .../collectives-westend/Cargo.toml | 108 ++++++++++-------- 1 file changed, 60 insertions(+), 48 deletions(-) diff --git a/cumulus/parachains/runtimes/collectives/collectives-westend/Cargo.toml b/cumulus/parachains/runtimes/collectives/collectives-westend/Cargo.toml index 94f2de33e908..79d6c697b5f9 100644 --- a/cumulus/parachains/runtimes/collectives/collectives-westend/Cargo.toml +++ b/cumulus/parachains/runtimes/collectives/collectives-westend/Cargo.toml @@ -7,69 +7,76 @@ license = "Apache-2.0" description = "Westend Collectives Parachain Runtime" [dependencies] -codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = ["derive", "max-encoded-len"] } +codec = { package = "parity-scale-codec", version = "3.0.0", default-features = false, features = [ + "derive", + "max-encoded-len", +] } hex-literal = { version = "0.4.1" } log = { version = "0.4.20", default-features = false } -scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } +scale-info = { version = "2.10.0", default-features = false, features = [ + "derive", +] } smallvec = "1.11.0" # Substrate -frame-benchmarking = { path = "../../../../../substrate/frame/benchmarking", default-features = false, optional = true} -frame-executive = { path = "../../../../../substrate/frame/executive", default-features = false} -frame-support = { path = "../../../../../substrate/frame/support", default-features = false} -frame-system = { path = "../../../../../substrate/frame/system", default-features = false} -frame-system-benchmarking = { path = "../../../../../substrate/frame/system/benchmarking", default-features = false, optional = true} -frame-system-rpc-runtime-api = { path = "../../../../../substrate/frame/system/rpc/runtime-api", default-features = false} -frame-try-runtime = { path = "../../../../../substrate/frame/try-runtime", default-features = false, optional = true} -pallet-alliance = { path = "../../../../../substrate/frame/alliance", default-features = false} -pallet-aura = { path = "../../../../../substrate/frame/aura", default-features = false} -pallet-authorship = { path = "../../../../../substrate/frame/authorship", default-features = false} -pallet-balances = { path = "../../../../../substrate/frame/balances", default-features = false} -pallet-collective = { path = "../../../../../substrate/frame/collective", default-features = false} -pallet-multisig = { path = "../../../../../substrate/frame/multisig", default-features = false} +frame-benchmarking = { path = "../../../../../substrate/frame/benchmarking", default-features = false, optional = true } +frame-executive = { path = "../../../../../substrate/frame/executive", default-features = false } +frame-support = { path = "../../../../../substrate/frame/support", default-features = false } +frame-system = { path = "../../../../../substrate/frame/system", default-features = false } +frame-system-benchmarking = { path = "../../../../../substrate/frame/system/benchmarking", default-features = false, optional = true } +frame-system-rpc-runtime-api = { path = "../../../../../substrate/frame/system/rpc/runtime-api", default-features = false } +frame-try-runtime = { path = "../../../../../substrate/frame/try-runtime", default-features = false, optional = true } +pallet-alliance = { path = "../../../../../substrate/frame/alliance", default-features = false } +pallet-aura = { path = "../../../../../substrate/frame/aura", default-features = false } +pallet-authorship = { path = "../../../../../substrate/frame/authorship", default-features = false } +pallet-balances = { path = "../../../../../substrate/frame/balances", default-features = false } +pallet-collective = { path = "../../../../../substrate/frame/collective", default-features = false } +pallet-multisig = { path = "../../../../../substrate/frame/multisig", default-features = false } pallet-preimage = { path = "../../../../../substrate/frame/preimage", default-features = false } -pallet-proxy = { path = "../../../../../substrate/frame/proxy", default-features = false} +pallet-proxy = { path = "../../../../../substrate/frame/proxy", default-features = false } pallet-scheduler = { path = "../../../../../substrate/frame/scheduler", default-features = false } -pallet-session = { path = "../../../../../substrate/frame/session", default-features = false} -pallet-timestamp = { path = "../../../../../substrate/frame/timestamp", default-features = false} -pallet-transaction-payment = { path = "../../../../../substrate/frame/transaction-payment", default-features = false} -pallet-transaction-payment-rpc-runtime-api = { path = "../../../../../substrate/frame/transaction-payment/rpc/runtime-api", default-features = false} -pallet-utility = { path = "../../../../../substrate/frame/utility", default-features = false} -pallet-referenda = { path = "../../../../../substrate/frame/referenda", default-features = false} -pallet-ranked-collective = { path = "../../../../../substrate/frame/ranked-collective", default-features = false} -pallet-core-fellowship = { path = "../../../../../substrate/frame/core-fellowship", default-features = false} -pallet-salary = { path = "../../../../../substrate/frame/salary", default-features = false} -sp-api = { path = "../../../../../substrate/primitives/api", default-features = false} +pallet-session = { path = "../../../../../substrate/frame/session", default-features = false } +pallet-timestamp = { path = "../../../../../substrate/frame/timestamp", default-features = false } +pallet-transaction-payment = { path = "../../../../../substrate/frame/transaction-payment", default-features = false } +pallet-transaction-payment-rpc-runtime-api = { path = "../../../../../substrate/frame/transaction-payment/rpc/runtime-api", default-features = false } +pallet-utility = { path = "../../../../../substrate/frame/utility", default-features = false } +pallet-referenda = { path = "../../../../../substrate/frame/referenda", default-features = false } +pallet-ranked-collective = { path = "../../../../../substrate/frame/ranked-collective", default-features = false } +pallet-core-fellowship = { path = "../../../../../substrate/frame/core-fellowship", default-features = false } +pallet-salary = { path = "../../../../../substrate/frame/salary", default-features = false } +sp-api = { path = "../../../../../substrate/primitives/api", default-features = false } sp-arithmetic = { path = "../../../../../substrate/primitives/arithmetic", default-features = false } -sp-block-builder = { path = "../../../../../substrate/primitives/block-builder", default-features = false} -sp-consensus-aura = { path = "../../../../../substrate/primitives/consensus/aura", default-features = false} -sp-core = { path = "../../../../../substrate/primitives/core", default-features = false} +sp-block-builder = { path = "../../../../../substrate/primitives/block-builder", default-features = false } +sp-consensus-aura = { path = "../../../../../substrate/primitives/consensus/aura", default-features = false } +sp-core = { path = "../../../../../substrate/primitives/core", default-features = false } sp-genesis-builder = { path = "../../../../../substrate/primitives/genesis-builder", default-features = false } -sp-inherents = { path = "../../../../../substrate/primitives/inherents", default-features = false} -sp-offchain = { path = "../../../../../substrate/primitives/offchain", default-features = false} -sp-runtime = { path = "../../../../../substrate/primitives/runtime", default-features = false} -sp-session = { path = "../../../../../substrate/primitives/session", default-features = false} -sp-std = { path = "../../../../../substrate/primitives/std", default-features = false} -sp-storage = { path = "../../../../../substrate/primitives/storage", default-features = false} -sp-transaction-pool = { path = "../../../../../substrate/primitives/transaction-pool", default-features = false} -sp-version = { path = "../../../../../substrate/primitives/version", default-features = false} +sp-inherents = { path = "../../../../../substrate/primitives/inherents", default-features = false } +sp-offchain = { path = "../../../../../substrate/primitives/offchain", default-features = false } +sp-runtime = { path = "../../../../../substrate/primitives/runtime", default-features = false } +sp-session = { path = "../../../../../substrate/primitives/session", default-features = false } +sp-std = { path = "../../../../../substrate/primitives/std", default-features = false } +sp-storage = { path = "../../../../../substrate/primitives/storage", default-features = false } +sp-transaction-pool = { path = "../../../../../substrate/primitives/transaction-pool", default-features = false } +sp-version = { path = "../../../../../substrate/primitives/version", default-features = false } # Polkadot -pallet-xcm = { path = "../../../../../polkadot/xcm/pallet-xcm", default-features = false} -polkadot-core-primitives = { path = "../../../../../polkadot/core-primitives", default-features = false} -polkadot-parachain-primitives = { path = "../../../../../polkadot/parachain", default-features = false} -polkadot-runtime-common = { path = "../../../../../polkadot/runtime/common", default-features = false} -xcm = { package = "staging-xcm", path = "../../../../../polkadot/xcm", default-features = false} -xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot/xcm/xcm-builder", default-features = false} -xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false} -westend-runtime-constants = { path = "../../../../../polkadot/runtime/westend/constants", default-features = false} +pallet-xcm = { path = "../../../../../polkadot/xcm/pallet-xcm", default-features = false } +polkadot-core-primitives = { path = "../../../../../polkadot/core-primitives", default-features = false } +polkadot-parachain-primitives = { path = "../../../../../polkadot/parachain", default-features = false } +polkadot-runtime-common = { path = "../../../../../polkadot/runtime/common", default-features = false } +xcm = { package = "staging-xcm", path = "../../../../../polkadot/xcm", default-features = false } +xcm-builder = { package = "staging-xcm-builder", path = "../../../../../polkadot/xcm/xcm-builder", default-features = false } +xcm-executor = { package = "staging-xcm-executor", path = "../../../../../polkadot/xcm/xcm-executor", default-features = false } +westend-runtime-constants = { path = "../../../../../polkadot/runtime/westend/constants", default-features = false } # Cumulus cumulus-pallet-aura-ext = { path = "../../../../pallets/aura-ext", default-features = false } pallet-message-queue = { path = "../../../../../substrate/frame/message-queue", default-features = false } cumulus-pallet-dmp-queue = { path = "../../../../pallets/dmp-queue", default-features = false } -cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system", default-features = false, features = ["parameterized-consensus-hook",] } -cumulus-pallet-session-benchmarking = { path = "../../../../pallets/session-benchmarking", default-features = false} +cumulus-pallet-parachain-system = { path = "../../../../pallets/parachain-system", default-features = false, features = [ + "parameterized-consensus-hook", +] } +cumulus-pallet-session-benchmarking = { path = "../../../../pallets/session-benchmarking", default-features = false } cumulus-pallet-xcm = { path = "../../../../pallets/xcm", default-features = false } cumulus-pallet-xcmp-queue = { path = "../../../../pallets/xcmp-queue", default-features = false } cumulus-primitives-core = { path = "../../../../primitives/core", default-features = false } @@ -84,7 +91,7 @@ testnets-common = { path = "../../../testnets-common", default-features = false substrate-wasm-builder = { path = "../../../../../substrate/utils/wasm-builder", optional = true } [dev-dependencies] -sp-io = { path = "../../../../../substrate/primitives/io", features = [ "std" ]} +sp-io = { path = "../../../../../substrate/primitives/io", features = ["std"] } [features] default = [ "std" ] @@ -228,3 +235,8 @@ std = [ ] experimental = [ "pallet-aura/experimental" ] + +# A feature that should be enabled when the runtime should be built for on-chain +# deployment. This will disable stuff that shouldn't be part of the on-chain wasm +# to make it smaller like logging for example. +on-chain-release-build = [ "sp-api/disable-logging" ] From 12062f6a3a75a36e6e61836ab15c5926d9773583 Mon Sep 17 00:00:00 2001 From: Liam Aharon Date: Thu, 23 Nov 2023 17:52:06 +0400 Subject: [PATCH 41/48] CI: Disable runtime upgrade spec name check on Westend Asset Hub and fix Staking pallet migration (#2447) Westend Asset Hub currently failing because the spec name is being changed next runtime upgrade (https://gitlab.parity.io/parity/mirrors/polkadot-sdk/-/jobs/4413125). This also fixes an idempotency issue with a staking pallet migration. Similar issues will be caught automatically now that we've also updated to try-runtime-cli v0.5.0 which checks for idempotency issues. This also enables try-state checks running in the CI. --- .gitlab/pipeline/check.yml | 13 +++++++------ substrate/frame/staking/src/migrations.rs | 12 +----------- 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.gitlab/pipeline/check.yml b/.gitlab/pipeline/check.yml index 429491fb1742..b2f2efc5e667 100644 --- a/.gitlab/pipeline/check.yml +++ b/.gitlab/pipeline/check.yml @@ -101,16 +101,16 @@ check-rust-feature-propagation: export RUST_LOG=remote-ext=debug,runtime=debug echo "---------- Downloading try-runtime CLI ----------" - curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.3.3/try-runtime-x86_64-unknown-linux-musl -o try-runtime + curl -sL https://github.com/paritytech/try-runtime-cli/releases/download/v0.5.0/try-runtime-x86_64-unknown-linux-musl -o try-runtime chmod +x ./try-runtime echo "---------- Building ${PACKAGE} runtime ----------" time cargo build --release --locked -p "$PACKAGE" --features try-runtime echo "---------- Executing on-runtime-upgrade for ${NETWORK} ----------" - time ./try-runtime \ + time ./try-runtime ${COMMAND_EXTRA_ARGS} \ --runtime ./target/release/wbuild/"$PACKAGE"/"$WASM" \ - on-runtime-upgrade --checks=pre-and-post ${EXTRA_ARGS} live --uri ${URI} + on-runtime-upgrade --disable-spec-version-check --checks=all ${SUBCOMMAND_EXTRA_ARGS} live --uri ${URI} # Check runtime migrations for Parity managed relay chains check-runtime-migration-westend: @@ -124,7 +124,7 @@ check-runtime-migration-westend: PACKAGE: "westend-runtime" WASM: "westend_runtime.compact.compressed.wasm" URI: "wss://westend-try-runtime-node.parity-chains.parity.io:443" - EXTRA_ARGS: "--no-weight-warnings" + SUBCOMMAND_EXTRA_ARGS: "--no-weight-warnings" check-runtime-migration-rococo: stage: check @@ -137,7 +137,7 @@ check-runtime-migration-rococo: PACKAGE: "rococo-runtime" WASM: "rococo_runtime.compact.compressed.wasm" URI: "wss://rococo-try-runtime-node.parity-chains.parity.io:443" - EXTRA_ARGS: "--no-weight-warnings" + SUBCOMMAND_EXTRA_ARGS: "--no-weight-warnings" # Check runtime migrations for Parity managed asset hub chains check-runtime-migration-asset-hub-westend: @@ -151,7 +151,7 @@ check-runtime-migration-asset-hub-westend: PACKAGE: "asset-hub-westend-runtime" WASM: "asset_hub_westend_runtime.compact.compressed.wasm" URI: "wss://westend-asset-hub-rpc.polkadot.io:443" - + check-runtime-migration-asset-hub-rococo: stage: check extends: @@ -214,6 +214,7 @@ check-runtime-migration-collectives-westend: PACKAGE: "collectives-westend-runtime" WASM: "collectives_westend_runtime.compact.compressed.wasm" URI: "wss://westend-collectives-rpc.polkadot.io:443" + COMMAND_EXTRA_ARGS: "--disable-spec-name-check" find-fail-ci-phrase: stage: check diff --git a/substrate/frame/staking/src/migrations.rs b/substrate/frame/staking/src/migrations.rs index 84b00254126f..311e9667cebc 100644 --- a/substrate/frame/staking/src/migrations.rs +++ b/substrate/frame/staking/src/migrations.rs @@ -81,20 +81,10 @@ pub mod v14 { } } - #[cfg(feature = "try-runtime")] - fn pre_upgrade() -> Result, TryRuntimeError> { - frame_support::ensure!( - Pallet::::on_chain_storage_version() == 13, - "Required v13 before upgrading to v14." - ); - - Ok(Default::default()) - } - #[cfg(feature = "try-runtime")] fn post_upgrade(_state: Vec) -> Result<(), TryRuntimeError> { frame_support::ensure!( - Pallet::::on_chain_storage_version() == 14, + Pallet::::on_chain_storage_version() >= 14, "v14 not applied" ); Ok(()) From 21f1811c6600d8a7fe043592ff34dcb79284d583 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20K=C3=B6cher?= Date: Thu, 23 Nov 2023 14:52:46 +0100 Subject: [PATCH 42/48] sp-api: Move macro related re-exports to `__private` (#2446) This moves the macro related re-exports to `__private` to make it more obvious for downstream users that they are using an internal api. --------- Co-authored-by: command-bot <> --- Cargo.lock | 10 ++- .../relay-chain-rpc-interface/Cargo.toml | 1 + .../src/rpc_client.rs | 2 +- cumulus/pallets/xcmp-queue/src/tests.rs | 2 +- polkadot/node/service/src/lib.rs | 2 +- substrate/client/api/src/call_executor.rs | 4 +- substrate/client/api/src/in_mem.rs | 3 +- substrate/client/consensus/beefy/src/lib.rs | 4 +- .../client/consensus/beefy/src/worker.rs | 7 +- substrate/client/executor/src/wasm_runtime.rs | 15 ++-- .../merkle-mountain-range/rpc/src/lib.rs | 4 +- substrate/client/rpc-spec-v2/Cargo.toml | 2 + .../client/rpc-spec-v2/src/archive/archive.rs | 4 +- .../src/chain_head/chain_head_storage.rs | 2 +- .../rpc-spec-v2/src/chain_head/test_utils.rs | 7 +- .../rpc-spec-v2/src/chain_head/tests.rs | 2 +- substrate/client/tracing/Cargo.toml | 1 + substrate/client/tracing/src/block/mod.rs | 3 +- substrate/frame/beefy-mmr/src/lib.rs | 3 +- .../frame/contracts/src/storage/meter.rs | 3 +- substrate/frame/nfts/runtime-api/Cargo.toml | 7 +- substrate/frame/nfts/runtime-api/src/lib.rs | 2 +- .../support/test/tests/construct_runtime.rs | 2 +- substrate/primitives/api/Cargo.toml | 16 ++-- .../primitives/api/proc-macro/src/utils.rs | 6 +- substrate/primitives/api/src/lib.rs | 80 +++++++++++-------- .../primitives/consensus/beefy/Cargo.toml | 16 ++-- .../primitives/consensus/beefy/src/mmr.rs | 3 +- .../frame/benchmarking-cli/src/block/bench.rs | 8 +- .../benchmarking-cli/src/storage/write.rs | 2 +- 30 files changed, 124 insertions(+), 99 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 26da3b4f8252..1fe9d1b81926 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1493,8 +1493,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93f2635620bf0b9d4576eb7bb9a38a55df78bd1205d26fa994b25911a69f212f" dependencies = [ "bitcoin_hashes", - "rand 0.7.3", - "rand_core 0.5.1", + "rand 0.8.5", + "rand_core 0.6.4", "serde", "unicode-normalization", ] @@ -4237,6 +4237,7 @@ dependencies = [ "sp-runtime", "sp-state-machine", "sp-storage 13.0.0", + "sp-version", "thiserror", "tokio", "tokio-util", @@ -10423,6 +10424,7 @@ dependencies = [ "pallet-nfts", "parity-scale-codec", "sp-api", + "sp-std 8.0.0", ] [[package]] @@ -16124,6 +16126,7 @@ dependencies = [ "sp-blockchain", "sp-consensus", "sp-core", + "sp-externalities 0.19.0", "sp-maybe-compressed-blob", "sp-rpc", "sp-runtime", @@ -16357,6 +16360,7 @@ dependencies = [ "lazy_static", "libc", "log", + "parity-scale-codec", "parking_lot 0.12.1", "regex", "rustc-hash", @@ -20011,7 +20015,7 @@ checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", "digest 0.10.7", - "rand 0.7.3", + "rand 0.8.5", "static_assertions", ] diff --git a/cumulus/client/relay-chain-rpc-interface/Cargo.toml b/cumulus/client/relay-chain-rpc-interface/Cargo.toml index 93fe57145b0c..0159cade23f6 100644 --- a/cumulus/client/relay-chain-rpc-interface/Cargo.toml +++ b/cumulus/client/relay-chain-rpc-interface/Cargo.toml @@ -20,6 +20,7 @@ sp-authority-discovery = { path = "../../../substrate/primitives/authority-disco sp-state-machine = { path = "../../../substrate/primitives/state-machine" } sp-storage = { path = "../../../substrate/primitives/storage" } sp-runtime = { path = "../../../substrate/primitives/runtime" } +sp-version = { path = "../../../substrate/primitives/version" } sc-client-api = { path = "../../../substrate/client/api" } sc-rpc-api = { path = "../../../substrate/client/rpc-api" } sc-service = { path = "../../../substrate/client/service" } diff --git a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs index cc993c6ff9f6..8e0d5fae6777 100644 --- a/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs +++ b/cumulus/client/relay-chain-rpc-interface/src/rpc_client.rs @@ -46,10 +46,10 @@ use cumulus_relay_chain_interface::{RelayChainError, RelayChainResult}; use sc_client_api::StorageData; use sc_rpc_api::{state::ReadProof, system::Health}; use sc_service::TaskManager; -use sp_api::RuntimeVersion; use sp_consensus_babe::Epoch; use sp_core::sp_std::collections::btree_map::BTreeMap; use sp_storage::StorageKey; +use sp_version::RuntimeVersion; use crate::{ light_client_worker::{build_smoldot_client, LightClientRpcWorker}, diff --git a/cumulus/pallets/xcmp-queue/src/tests.rs b/cumulus/pallets/xcmp-queue/src/tests.rs index 30dba6ead340..f88dedc1ece4 100644 --- a/cumulus/pallets/xcmp-queue/src/tests.rs +++ b/cumulus/pallets/xcmp-queue/src/tests.rs @@ -155,7 +155,7 @@ fn xcm_enqueueing_broken_xcm_works() { .take(20) .collect::>(), ); - EnqueuedMessages::set(&vec![]); + EnqueuedMessages::take(); // But if we do it all in one page, then it only uses the first 10: XcmpQueue::handle_xcmp_messages( diff --git a/polkadot/node/service/src/lib.rs b/polkadot/node/service/src/lib.rs index 0ed7940b3e80..c03835b2a4b0 100644 --- a/polkadot/node/service/src/lib.rs +++ b/polkadot/node/service/src/lib.rs @@ -98,7 +98,7 @@ pub use service::{ ChainSpec, Configuration, Error as SubstrateServiceError, PruningMode, Role, RuntimeGenesis, TFullBackend, TFullCallExecutor, TFullClient, TaskManager, TransactionPoolOptions, }; -pub use sp_api::{ApiRef, ConstructRuntimeApi, Core as CoreApi, ProvideRuntimeApi, StateBackend}; +pub use sp_api::{ApiRef, ConstructRuntimeApi, Core as CoreApi, ProvideRuntimeApi}; pub use sp_runtime::{ generic, traits::{self as runtime_traits, BlakeTwo256, Block as BlockT, Header as HeaderT, NumberFor}, diff --git a/substrate/client/api/src/call_executor.rs b/substrate/client/api/src/call_executor.rs index 49b51ccc943e..d131cbcec002 100644 --- a/substrate/client/api/src/call_executor.rs +++ b/substrate/client/api/src/call_executor.rs @@ -21,12 +21,12 @@ use sc_executor::{RuntimeVersion, RuntimeVersionOf}; use sp_core::traits::CallContext; use sp_externalities::Extensions; -use sp_runtime::traits::Block as BlockT; +use sp_runtime::traits::{Block as BlockT, HashingFor}; use sp_state_machine::{OverlayedChanges, StorageProof}; use std::cell::RefCell; use crate::execution_extensions::ExecutionExtensions; -use sp_api::{HashingFor, ProofRecorder}; +use sp_api::ProofRecorder; /// Executor Provider pub trait ExecutorProvider { diff --git a/substrate/client/api/src/in_mem.rs b/substrate/client/api/src/in_mem.rs index 807bdf0e3347..b933ed1f17e0 100644 --- a/substrate/client/api/src/in_mem.rs +++ b/substrate/client/api/src/in_mem.rs @@ -812,9 +812,8 @@ pub fn check_genesis_storage(storage: &Storage) -> sp_blockchain::Result<()> { #[cfg(test)] mod tests { use crate::{in_mem::Blockchain, NewBlockState}; - use sp_api::HeaderT; use sp_blockchain::Backend; - use sp_runtime::{ConsensusEngineId, Justifications}; + use sp_runtime::{traits::Header as HeaderT, ConsensusEngineId, Justifications}; use substrate_test_runtime::{Block, Header, H256}; pub const ID1: ConsensusEngineId = *b"TST1"; diff --git a/substrate/client/consensus/beefy/src/lib.rs b/substrate/client/consensus/beefy/src/lib.rs index 89a5d51c8870..3d104f132508 100644 --- a/substrate/client/consensus/beefy/src/lib.rs +++ b/substrate/client/consensus/beefy/src/lib.rs @@ -40,7 +40,7 @@ use sc_client_api::{Backend, BlockBackend, BlockchainEvents, FinalityNotificatio use sc_consensus::BlockImport; use sc_network::{NetworkRequest, ProtocolName}; use sc_network_gossip::{GossipEngine, Network as GossipNetwork, Syncing as GossipSyncing}; -use sp_api::{HeaderT, NumberFor, ProvideRuntimeApi}; +use sp_api::ProvideRuntimeApi; use sp_blockchain::{ Backend as BlockchainBackend, Error as ClientError, HeaderBackend, Result as ClientResult, }; @@ -51,7 +51,7 @@ use sp_consensus_beefy::{ }; use sp_keystore::KeystorePtr; use sp_mmr_primitives::MmrApi; -use sp_runtime::traits::{Block, Zero}; +use sp_runtime::traits::{Block, Header as HeaderT, NumberFor, Zero}; use std::{ collections::{BTreeMap, VecDeque}, marker::PhantomData, diff --git a/substrate/client/consensus/beefy/src/worker.rs b/substrate/client/consensus/beefy/src/worker.rs index 966c74103657..0eea5647e51c 100644 --- a/substrate/client/consensus/beefy/src/worker.rs +++ b/substrate/client/consensus/beefy/src/worker.rs @@ -36,7 +36,7 @@ use log::{debug, error, info, log_enabled, trace, warn}; use sc_client_api::{Backend, FinalityNotification, FinalityNotifications, HeaderBackend}; use sc_network_gossip::GossipEngine; use sc_utils::{mpsc::TracingUnboundedReceiver, notification::NotificationReceiver}; -use sp_api::{BlockId, ProvideRuntimeApi}; +use sp_api::ProvideRuntimeApi; use sp_arithmetic::traits::{AtLeast32Bit, Saturating}; use sp_consensus::SyncOracle; use sp_consensus_beefy::{ @@ -46,7 +46,7 @@ use sp_consensus_beefy::{ VersionedFinalityProof, VoteMessage, BEEFY_ENGINE_ID, }; use sp_runtime::{ - generic::OpaqueDigestItemId, + generic::{BlockId, OpaqueDigestItemId}, traits::{Block, Header, NumberFor, Zero}, SaturatedConversion, }; @@ -1074,13 +1074,12 @@ pub(crate) mod tests { use sc_client_api::{Backend as BackendT, HeaderBackend}; use sc_network_sync::SyncingService; use sc_network_test::TestNetFactory; - use sp_api::HeaderT; use sp_blockchain::Backend as BlockchainBackendT; use sp_consensus_beefy::{ generate_equivocation_proof, known_payloads, known_payloads::MMR_ROOT_ID, mmr::MmrRootProvider, Keyring, Payload, SignedCommitment, }; - use sp_runtime::traits::One; + use sp_runtime::traits::{Header as HeaderT, One}; use substrate_test_runtime_client::{ runtime::{Block, Digest, DigestItem, Header}, Backend, diff --git a/substrate/client/executor/src/wasm_runtime.rs b/substrate/client/executor/src/wasm_runtime.rs index 6dec3abdb20c..501279a312cc 100644 --- a/substrate/client/executor/src/wasm_runtime.rs +++ b/substrate/client/executor/src/wasm_runtime.rs @@ -441,6 +441,7 @@ mod tests { use codec::Encode; use sp_api::{Core, RuntimeApiInfo}; use sp_runtime::RuntimeString; + use sp_version::{create_apis_vec, RuntimeVersion}; use sp_wasm_interface::HostFunctions; use substrate_test_runtime::Block; @@ -470,7 +471,7 @@ mod tests { authoring_version: 1, spec_version: 1, impl_version: 1, - apis: sp_api::create_apis_vec!([(>::ID, 1)]), + apis: create_apis_vec!([(>::ID, 1)]), }; let version = decode_version(&old_runtime_version.encode()).unwrap(); @@ -486,7 +487,7 @@ mod tests { authoring_version: 1, spec_version: 1, impl_version: 1, - apis: sp_api::create_apis_vec!([(>::ID, 3)]), + apis: create_apis_vec!([(>::ID, 3)]), }; decode_version(&old_runtime_version.encode()).unwrap_err(); @@ -494,13 +495,13 @@ mod tests { #[test] fn new_runtime_version_decodes() { - let old_runtime_version = sp_api::RuntimeVersion { + let old_runtime_version = RuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, spec_version: 1, impl_version: 1, - apis: sp_api::create_apis_vec!([(>::ID, 3)]), + apis: create_apis_vec!([(>::ID, 3)]), transaction_version: 3, state_version: 4, }; @@ -509,13 +510,13 @@ mod tests { assert_eq!(3, version.transaction_version); assert_eq!(0, version.state_version); - let old_runtime_version = sp_api::RuntimeVersion { + let old_runtime_version = RuntimeVersion { spec_name: "test".into(), impl_name: "test".into(), authoring_version: 1, spec_version: 1, impl_version: 1, - apis: sp_api::create_apis_vec!([(>::ID, 4)]), + apis: create_apis_vec!([(>::ID, 4)]), transaction_version: 3, state_version: 4, }; @@ -538,7 +539,7 @@ mod tests { authoring_version: 100, spec_version: 100, impl_version: 100, - apis: sp_api::create_apis_vec!([(>::ID, 4)]), + apis: create_apis_vec!([(>::ID, 4)]), transaction_version: 100, state_version: 1, }; diff --git a/substrate/client/merkle-mountain-range/rpc/src/lib.rs b/substrate/client/merkle-mountain-range/rpc/src/lib.rs index 5be82b600d91..1653749ffab9 100644 --- a/substrate/client/merkle-mountain-range/rpc/src/lib.rs +++ b/substrate/client/merkle-mountain-range/rpc/src/lib.rs @@ -30,14 +30,14 @@ use jsonrpsee::{ }; use serde::{Deserialize, Serialize}; -use sp_api::{ApiExt, NumberFor, ProvideRuntimeApi}; +use sp_api::{ApiExt, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; use sp_core::{ offchain::{storage::OffchainDb, OffchainDbExt, OffchainStorage}, Bytes, }; use sp_mmr_primitives::{Error as MmrError, Proof}; -use sp_runtime::traits::Block as BlockT; +use sp_runtime::traits::{Block as BlockT, NumberFor}; pub use sp_mmr_primitives::MmrApi as MmrRuntimeApi; diff --git a/substrate/client/rpc-spec-v2/Cargo.toml b/substrate/client/rpc-spec-v2/Cargo.toml index ca61286ddfa0..8ca4f321f711 100644 --- a/substrate/client/rpc-spec-v2/Cargo.toml +++ b/substrate/client/rpc-spec-v2/Cargo.toml @@ -37,12 +37,14 @@ tokio = { version = "1.22.0", features = ["sync"] } array-bytes = "6.1" log = "0.4.17" futures-util = { version = "0.3.19", default-features = false } + [dev-dependencies] serde_json = "1.0.108" tokio = { version = "1.22.0", features = ["macros"] } substrate-test-runtime-client = { path = "../../test-utils/runtime/client" } substrate-test-runtime = { path = "../../test-utils/runtime" } sp-consensus = { path = "../../primitives/consensus/common" } +sp-externalities = { path = "../../primitives/externalities" } sp-maybe-compressed-blob = { path = "../../primitives/maybe-compressed-blob" } sc-block-builder = { path = "../block-builder" } sc-service = { path = "../service", features = ["test-helpers"]} diff --git a/substrate/client/rpc-spec-v2/src/archive/archive.rs b/substrate/client/rpc-spec-v2/src/archive/archive.rs index bded842d8fd0..b2bcc62ce317 100644 --- a/substrate/client/rpc-spec-v2/src/archive/archive.rs +++ b/substrate/client/rpc-spec-v2/src/archive/archive.rs @@ -29,13 +29,13 @@ use jsonrpsee::core::{async_trait, RpcResult}; use sc_client_api::{ Backend, BlockBackend, BlockchainEvents, CallExecutor, ExecutorProvider, StorageProvider, }; -use sp_api::{CallApiAt, CallContext, NumberFor}; +use sp_api::{CallApiAt, CallContext}; use sp_blockchain::{ Backend as BlockChainBackend, Error as BlockChainError, HeaderBackend, HeaderMetadata, }; use sp_core::Bytes; use sp_runtime::{ - traits::{Block as BlockT, Header as HeaderT}, + traits::{Block as BlockT, Header as HeaderT, NumberFor}, SaturatedConversion, }; use std::{collections::HashSet, marker::PhantomData, sync::Arc}; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs index 6e19f59a5d68..c23489a050e5 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/chain_head_storage.rs @@ -22,8 +22,8 @@ use std::{collections::VecDeque, marker::PhantomData, sync::Arc}; use sc_client_api::{Backend, ChildInfo, StorageKey, StorageProvider}; use sc_utils::mpsc::TracingUnboundedSender; -use sp_api::BlockT; use sp_core::storage::well_known_keys; +use sp_runtime::traits::Block as BlockT; use crate::chain_head::event::OperationStorageItems; diff --git a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs index a901f3039ffe..d63a98a5cb0d 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/test_utils.rs @@ -24,14 +24,15 @@ use sc_client_api::{ StorageData, StorageEventStream, StorageKey, StorageProvider, }; use sc_utils::mpsc::{tracing_unbounded, TracingUnboundedSender}; -use sp_api::{CallApiAt, CallApiAtParams, NumberFor, RuntimeVersion}; +use sp_api::{CallApiAt, CallApiAtParams}; use sp_blockchain::{BlockStatus, CachedHeaderMetadata, HeaderBackend, HeaderMetadata, Info}; use sp_consensus::BlockOrigin; use sp_runtime::{ generic::SignedBlock, - traits::{Block as BlockT, Header as HeaderT}, + traits::{Block as BlockT, Header as HeaderT, NumberFor}, Justifications, }; +use sp_version::RuntimeVersion; use std::sync::Arc; use substrate_test_runtime::{Block, Hash, Header}; @@ -235,7 +236,7 @@ impl> CallApiAt for ChainHeadMock fn initialize_extensions( &self, at: ::Hash, - extensions: &mut sp_api::Extensions, + extensions: &mut sp_externalities::Extensions, ) -> Result<(), sp_api::ApiError> { self.client.initialize_extensions(at, extensions) } diff --git a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs index 4e6775fe2809..518b7da432bc 100644 --- a/substrate/client/rpc-spec-v2/src/chain_head/tests.rs +++ b/substrate/client/rpc-spec-v2/src/chain_head/tests.rs @@ -34,7 +34,6 @@ use jsonrpsee::{ use sc_block_builder::BlockBuilderBuilder; use sc_client_api::ChildInfo; use sc_service::client::new_in_mem; -use sp_api::BlockT; use sp_blockchain::HeaderBackend; use sp_consensus::BlockOrigin; use sp_core::{ @@ -42,6 +41,7 @@ use sp_core::{ testing::TaskExecutor, Blake2Hasher, Hasher, }; +use sp_runtime::traits::Block as BlockT; use sp_version::RuntimeVersion; use std::{ collections::{HashMap, HashSet}, diff --git a/substrate/client/tracing/Cargo.toml b/substrate/client/tracing/Cargo.toml index ffcbf0749083..796d4f1d8266 100644 --- a/substrate/client/tracing/Cargo.toml +++ b/substrate/client/tracing/Cargo.toml @@ -16,6 +16,7 @@ targets = ["x86_64-unknown-linux-gnu"] ansi_term = "0.12.1" atty = "0.2.13" chrono = "0.4.27" +codec = { package = "parity-scale-codec", version = "3.6.1" } lazy_static = "1.4.0" libc = "0.2.121" log = { version = "0.4.17" } diff --git a/substrate/client/tracing/src/block/mod.rs b/substrate/client/tracing/src/block/mod.rs index 9ebf8e55c94d..01744cd5563b 100644 --- a/substrate/client/tracing/src/block/mod.rs +++ b/substrate/client/tracing/src/block/mod.rs @@ -25,6 +25,7 @@ use std::{ time::Instant, }; +use codec::Encode; use parking_lot::Mutex; use tracing::{ dispatcher, @@ -34,7 +35,7 @@ use tracing::{ use crate::{SpanDatum, TraceEvent, Values}; use sc_client_api::BlockBackend; -use sp_api::{Core, Encode, Metadata, ProvideRuntimeApi}; +use sp_api::{Core, Metadata, ProvideRuntimeApi}; use sp_blockchain::HeaderBackend; use sp_core::hexdisplay::HexDisplay; use sp_rpc::tracing::{BlockTrace, Span, TraceBlockResponse}; diff --git a/substrate/frame/beefy-mmr/src/lib.rs b/substrate/frame/beefy-mmr/src/lib.rs index a0bf7cdcf86a..fa3caba7977d 100644 --- a/substrate/frame/beefy-mmr/src/lib.rs +++ b/substrate/frame/beefy-mmr/src/lib.rs @@ -36,6 +36,7 @@ use sp_runtime::traits::{Convert, Member}; use sp_std::prelude::*; +use codec::Decode; use pallet_mmr::{LeafDataProvider, ParentNumberAndHash}; use sp_consensus_beefy::{ mmr::{BeefyAuthoritySet, BeefyDataProvider, BeefyNextAuthoritySet, MmrLeaf, MmrLeafVersion}, @@ -226,7 +227,7 @@ sp_api::decl_runtime_apis! { /// API useful for BEEFY light clients. pub trait BeefyMmrApi where - BeefyAuthoritySet: sp_api::Decode, + BeefyAuthoritySet: Decode, { /// Return the currently active BEEFY authority set proof. fn authority_set_proof() -> BeefyAuthoritySet; diff --git a/substrate/frame/contracts/src/storage/meter.rs b/substrate/frame/contracts/src/storage/meter.rs index 9f098090bc8b..495cbd90db5a 100644 --- a/substrate/frame/contracts/src/storage/meter.rs +++ b/substrate/frame/contracts/src/storage/meter.rs @@ -33,9 +33,8 @@ use frame_support::{ }, DefaultNoBound, RuntimeDebugNoBound, }; -use sp_api::HashT; use sp_runtime::{ - traits::{Saturating, Zero}, + traits::{Hash as HashT, Saturating, Zero}, DispatchError, FixedPointNumber, FixedU128, }; use sp_std::{fmt::Debug, marker::PhantomData, vec, vec::Vec}; diff --git a/substrate/frame/nfts/runtime-api/Cargo.toml b/substrate/frame/nfts/runtime-api/Cargo.toml index 483c4bd32342..092edaaaa89e 100644 --- a/substrate/frame/nfts/runtime-api/Cargo.toml +++ b/substrate/frame/nfts/runtime-api/Cargo.toml @@ -14,9 +14,10 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } -pallet-nfts = { path = "..", default-features = false} -sp-api = { path = "../../../primitives/api", default-features = false} +pallet-nfts = { path = "..", default-features = false } +sp-api = { path = "../../../primitives/api", default-features = false } +sp-std = { path = "../../../primitives/std", default-features = false } [features] default = [ "std" ] -std = [ "codec/std", "pallet-nfts/std", "sp-api/std" ] +std = [ "codec/std", "pallet-nfts/std", "sp-api/std", "sp-std/std" ] diff --git a/substrate/frame/nfts/runtime-api/src/lib.rs b/substrate/frame/nfts/runtime-api/src/lib.rs index 77535c64069c..816088f1b716 100644 --- a/substrate/frame/nfts/runtime-api/src/lib.rs +++ b/substrate/frame/nfts/runtime-api/src/lib.rs @@ -20,7 +20,7 @@ #![cfg_attr(not(feature = "std"), no_std)] use codec::{Decode, Encode}; -use sp_api::vec::Vec; +use sp_std::vec::Vec; sp_api::decl_runtime_apis! { pub trait NftsApi diff --git a/substrate/frame/support/test/tests/construct_runtime.rs b/substrate/frame/support/test/tests/construct_runtime.rs index 9ad51ad530eb..b8341b25cb09 100644 --- a/substrate/frame/support/test/tests/construct_runtime.rs +++ b/substrate/frame/support/test/tests/construct_runtime.rs @@ -27,13 +27,13 @@ use frame_support::{ }; use frame_system::limits::{BlockLength, BlockWeights}; use scale_info::TypeInfo; -use sp_api::RuntimeVersion; use sp_core::{sr25519, ConstU64}; use sp_runtime::{ generic, traits::{BlakeTwo256, Verify}, DispatchError, ModuleError, }; +use sp_version::RuntimeVersion; parameter_types! { pub static IntegrityTestExec: u32 = 0; diff --git a/substrate/primitives/api/Cargo.toml b/substrate/primitives/api/Cargo.toml index c3a68af097b0..e95333c9eeb1 100644 --- a/substrate/primitives/api/Cargo.toml +++ b/substrate/primitives/api/Cargo.toml @@ -15,17 +15,17 @@ targets = ["x86_64-unknown-linux-gnu"] [dependencies] codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false } sp-api-proc-macro = { path = "proc-macro" } -sp-core = { path = "../core", default-features = false} -sp-std = { path = "../std", default-features = false} -sp-runtime = { path = "../runtime", default-features = false} -sp-externalities = { path = "../externalities", default-features = false, optional = true} -sp-version = { path = "../version", default-features = false} -sp-state-machine = { path = "../state-machine", default-features = false, optional = true} -sp-trie = { path = "../trie", default-features = false, optional = true} +sp-core = { path = "../core", default-features = false } +sp-std = { path = "../std", default-features = false } +sp-runtime = { path = "../runtime", default-features = false } +sp-externalities = { path = "../externalities", default-features = false, optional = true } +sp-version = { path = "../version", default-features = false } +sp-state-machine = { path = "../state-machine", default-features = false, optional = true } +sp-trie = { path = "../trie", default-features = false, optional = true } hash-db = { version = "0.16.0", optional = true } thiserror = { version = "1.0.48", optional = true } scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } -sp-metadata-ir = { path = "../metadata-ir", default-features = false, optional = true} +sp-metadata-ir = { path = "../metadata-ir", default-features = false, optional = true } log = { version = "0.4.17", default-features = false } [dev-dependencies] diff --git a/substrate/primitives/api/proc-macro/src/utils.rs b/substrate/primitives/api/proc-macro/src/utils.rs index e261b162b5aa..68f0a77a3995 100644 --- a/substrate/primitives/api/proc-macro/src/utils.rs +++ b/substrate/primitives/api/proc-macro/src/utils.rs @@ -28,14 +28,14 @@ use syn::{ /// Generates the access to the `sc_client` crate. pub fn generate_crate_access() -> TokenStream { match crate_name("sp-api") { - Ok(FoundCrate::Itself) => quote!(sp_api), + Ok(FoundCrate::Itself) => quote!(sp_api::__private), Ok(FoundCrate::Name(renamed_name)) => { let renamed_name = Ident::new(&renamed_name, Span::call_site()); - quote!(#renamed_name) + quote!(#renamed_name::__private) }, Err(e) => if let Ok(FoundCrate::Name(name)) = crate_name(&"frame") { - let path = format!("{}::deps::{}", name, "sp_api"); + let path = format!("{}::deps::sp_api::__private", name); let path = syn::parse_str::(&path).expect("is a valid path; qed"); quote!( #path ) } else { diff --git a/substrate/primitives/api/src/lib.rs b/substrate/primitives/api/src/lib.rs index c122a2348b8e..3b00e6b418b0 100644 --- a/substrate/primitives/api/src/lib.rs +++ b/substrate/primitives/api/src/lib.rs @@ -70,48 +70,58 @@ // Make doc tests happy extern crate self as sp_api; +/// Private exports used by the macros. +/// +/// This is seen as internal API and can change at any point. #[doc(hidden)] -pub use codec::{self, Decode, DecodeLimit, Encode}; -#[doc(hidden)] -#[cfg(feature = "std")] -pub use hash_db::Hasher; -#[doc(hidden)] -pub use scale_info; -#[doc(hidden)] -pub use sp_core::offchain; -#[doc(hidden)] -#[cfg(not(feature = "std"))] -pub use sp_core::to_substrate_wasm_fn_return_value; -#[doc(hidden)] +pub mod __private { + #[cfg(feature = "std")] + mod std_imports { + pub use hash_db::Hasher; + pub use sp_core::traits::CallContext; + pub use sp_externalities::{Extension, Extensions}; + pub use sp_runtime::StateVersion; + pub use sp_state_machine::{ + Backend as StateBackend, InMemoryBackend, OverlayedChanges, StorageProof, TrieBackend, + TrieBackendBuilder, + }; + } + #[cfg(feature = "std")] + pub use std_imports::*; + + pub use crate::*; + pub use codec::{self, Decode, DecodeLimit, Encode}; + pub use scale_info; + pub use sp_core::offchain; + #[cfg(not(feature = "std"))] + pub use sp_core::to_substrate_wasm_fn_return_value; + #[cfg(feature = "frame-metadata")] + pub use sp_metadata_ir::{self as metadata_ir, frame_metadata as metadata}; + pub use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, NumberFor}, + transaction_validity::TransactionValidity, + RuntimeString, TransactionOutcome, + }; + pub use sp_std::{mem, slice, vec}; + pub use sp_version::{create_apis_vec, ApiId, ApisVec, RuntimeVersion}; +} + #[cfg(feature = "std")] pub use sp_core::traits::CallContext; use sp_core::OpaqueMetadata; -#[doc(hidden)] #[cfg(feature = "std")] -pub use sp_externalities::{Extension, Extensions}; -#[doc(hidden)] -#[cfg(feature = "frame-metadata")] -pub use sp_metadata_ir::{self as metadata_ir, frame_metadata as metadata}; -#[doc(hidden)] +use sp_externalities::{Extension, Extensions}; +use sp_runtime::traits::Block as BlockT; #[cfg(feature = "std")] -pub use sp_runtime::StateVersion; -#[doc(hidden)] -pub use sp_runtime::{ - generic::BlockId, - traits::{Block as BlockT, Hash as HashT, HashingFor, Header as HeaderT, NumberFor}, - transaction_validity::TransactionValidity, - RuntimeString, TransactionOutcome, -}; -#[doc(hidden)] +use sp_runtime::traits::HashingFor; #[cfg(feature = "std")] -pub use sp_state_machine::{ - backend::AsTrieBackend, Backend as StateBackend, InMemoryBackend, OverlayedChanges, - StorageProof, TrieBackend, TrieBackendBuilder, -}; -#[doc(hidden)] -pub use sp_std::{mem, slice, vec}; -#[doc(hidden)] -pub use sp_version::{create_apis_vec, ApiId, ApisVec, RuntimeVersion}; +pub use sp_runtime::TransactionOutcome; +#[cfg(feature = "std")] +pub use sp_state_machine::StorageProof; +#[cfg(feature = "std")] +use sp_state_machine::{backend::AsTrieBackend, Backend as StateBackend, OverlayedChanges}; +use sp_version::RuntimeVersion; #[cfg(feature = "std")] use std::cell::RefCell; diff --git a/substrate/primitives/consensus/beefy/Cargo.toml b/substrate/primitives/consensus/beefy/Cargo.toml index 5ff0a2ebc70f..e78323c8980b 100644 --- a/substrate/primitives/consensus/beefy/Cargo.toml +++ b/substrate/primitives/consensus/beefy/Cargo.toml @@ -15,19 +15,19 @@ targets = ["x86_64-unknown-linux-gnu"] codec = { package = "parity-scale-codec", version = "3.6.1", default-features = false, features = ["derive"] } scale-info = { version = "2.10.0", default-features = false, features = ["derive"] } serde = { version = "1.0.188", default-features = false, optional = true, features = ["derive", "alloc"] } -sp-api = { path = "../../api", default-features = false} -sp-application-crypto = { path = "../../application-crypto", default-features = false} -sp-core = { path = "../../core", default-features = false} -sp-io = { path = "../../io", default-features = false} -sp-mmr-primitives = { path = "../../merkle-mountain-range", default-features = false} -sp-runtime = { path = "../../runtime", default-features = false} -sp-std = { path = "../../std", default-features = false} +sp-api = { path = "../../api", default-features = false } +sp-application-crypto = { path = "../../application-crypto", default-features = false } +sp-core = { path = "../../core", default-features = false } +sp-io = { path = "../../io", default-features = false } +sp-mmr-primitives = { path = "../../merkle-mountain-range", default-features = false } +sp-runtime = { path = "../../runtime", default-features = false } +sp-std = { path = "../../std", default-features = false } strum = { version = "0.24.1", features = ["derive"], default-features = false } lazy_static = "1.4.0" [dev-dependencies] array-bytes = "6.1" -w3f-bls = { version = "0.1.3", features = ["std"]} +w3f-bls = { version = "0.1.3", features = ["std"] } [features] default = [ "std" ] diff --git a/substrate/primitives/consensus/beefy/src/mmr.rs b/substrate/primitives/consensus/beefy/src/mmr.rs index 660506b8763f..9ac1624ca752 100644 --- a/substrate/primitives/consensus/beefy/src/mmr.rs +++ b/substrate/primitives/consensus/beefy/src/mmr.rs @@ -150,8 +150,9 @@ pub use mmr_root_provider::MmrRootProvider; mod mmr_root_provider { use super::*; use crate::{known_payloads, payload::PayloadProvider, Payload}; - use sp_api::{NumberFor, ProvideRuntimeApi}; + use sp_api::ProvideRuntimeApi; use sp_mmr_primitives::MmrApi; + use sp_runtime::traits::NumberFor; use sp_std::{marker::PhantomData, sync::Arc}; /// A [`crate::Payload`] provider where payload is Merkle Mountain Range root hash. diff --git a/substrate/utils/frame/benchmarking-cli/src/block/bench.rs b/substrate/utils/frame/benchmarking-cli/src/block/bench.rs index a028c7d438e8..ef8dc29dde84 100644 --- a/substrate/utils/frame/benchmarking-cli/src/block/bench.rs +++ b/substrate/utils/frame/benchmarking-cli/src/block/bench.rs @@ -25,9 +25,13 @@ use sc_cli::{Error, Result}; use sc_client_api::{ Backend as ClientBackend, BlockBackend, HeaderBackend, StorageProvider, UsageProvider, }; -use sp_api::{ApiExt, Core, HeaderT, ProvideRuntimeApi}; +use sp_api::{ApiExt, Core, ProvideRuntimeApi}; use sp_blockchain::Error::RuntimeApiError; -use sp_runtime::{generic::BlockId, traits::Block as BlockT, DigestItem, OpaqueExtrinsic}; +use sp_runtime::{ + generic::BlockId, + traits::{Block as BlockT, Header as HeaderT}, + DigestItem, OpaqueExtrinsic, +}; use sp_storage::StorageKey; use clap::Args; diff --git a/substrate/utils/frame/benchmarking-cli/src/storage/write.rs b/substrate/utils/frame/benchmarking-cli/src/storage/write.rs index 4def1909ead5..65941497bda4 100644 --- a/substrate/utils/frame/benchmarking-cli/src/storage/write.rs +++ b/substrate/utils/frame/benchmarking-cli/src/storage/write.rs @@ -18,10 +18,10 @@ use sc_cli::Result; use sc_client_api::{Backend as ClientBackend, StorageProvider, UsageProvider}; use sc_client_db::{DbHash, DbState, DbStateBuilder}; -use sp_api::StateBackend; use sp_blockchain::HeaderBackend; use sp_database::{ColumnId, Transaction}; use sp_runtime::traits::{Block as BlockT, HashingFor, Header as HeaderT}; +use sp_state_machine::Backend as StateBackend; use sp_trie::PrefixedMemoryDB; use log::{info, trace}; From 19c05e8d43fc4aef12668ecc9144bb9669cc7bbc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Silva?= <123550+andresilva@users.noreply.github.com> Date: Thu, 23 Nov 2023 17:00:54 +0000 Subject: [PATCH 43/48] Fix trait imports from sp-api (#2472) Broken after #2446. --- Cargo.lock | 1 + .../relay-chain-minimal-node/src/blockchain_rpc_client.rs | 3 ++- polkadot/node/subsystem-types/Cargo.toml | 1 + polkadot/node/subsystem-types/src/runtime_client.rs | 3 ++- 4 files changed, 6 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1fe9d1b81926..abd539ea799c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12860,6 +12860,7 @@ dependencies = [ "sp-authority-discovery", "sp-blockchain", "sp-consensus-babe", + "sp-runtime", "substrate-prometheus-endpoint", "thiserror", ] diff --git a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs index c40ca5c858ba..d9e4155d9c5e 100644 --- a/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs +++ b/cumulus/client/relay-chain-minimal-node/src/blockchain_rpc_client.rs @@ -28,8 +28,9 @@ use polkadot_primitives::{ }; use sc_authority_discovery::{AuthorityDiscovery, Error as AuthorityDiscoveryError}; use sc_client_api::AuxStore; -use sp_api::{ApiError, BlockT, HeaderT, NumberFor, RuntimeApiInfo}; +use sp_api::{ApiError, RuntimeApiInfo}; use sp_blockchain::Info; +use sp_runtime::traits::{Block as BlockT, Header as HeaderT, NumberFor}; #[derive(Clone)] pub struct BlockChainRpcClient { diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index 8e345cf222c6..71e5257cab98 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -19,6 +19,7 @@ sc-network = { path = "../../../substrate/client/network" } sp-api = { path = "../../../substrate/primitives/api" } sp-blockchain = { path = "../../../substrate/primitives/blockchain" } sp-consensus-babe = { path = "../../../substrate/primitives/consensus/babe" } +sp-runtime = { path = "../../../substrate/primitives/runtime" } sp-authority-discovery = { path = "../../../substrate/primitives/authority-discovery" } sc-client-api = { path = "../../../substrate/client/api" } sc-transaction-pool-api = { path = "../../../substrate/client/transaction-pool/api" } diff --git a/polkadot/node/subsystem-types/src/runtime_client.rs b/polkadot/node/subsystem-types/src/runtime_client.rs index 36e3365cf08c..21df1483b9e6 100644 --- a/polkadot/node/subsystem-types/src/runtime_client.rs +++ b/polkadot/node/subsystem-types/src/runtime_client.rs @@ -25,10 +25,11 @@ use polkadot_primitives::{ }; use sc_client_api::HeaderBackend; use sc_transaction_pool_api::OffchainTransactionPoolFactory; -use sp_api::{ApiError, ApiExt, HeaderT, NumberFor, ProvideRuntimeApi}; +use sp_api::{ApiError, ApiExt, ProvideRuntimeApi}; use sp_authority_discovery::AuthorityDiscoveryApi; use sp_blockchain::Info; use sp_consensus_babe::{BabeApi, Epoch}; +use sp_runtime::traits::{Header as HeaderT, NumberFor}; use std::{collections::BTreeMap, sync::Arc}; /// Offers header utilities. From c29b74dc36fc20a76cee4a029a70e7ad42fa0b6e Mon Sep 17 00:00:00 2001 From: Ross Bulat Date: Fri, 24 Nov 2023 00:14:59 +0700 Subject: [PATCH 44/48] Amend staking docs to account for state of controller deprecation (#2451) Amends some staking pallet docs, and deprecation comment, to adjust to the latest controller deprecation state. Note, do we need the `README.md` file, which is a duplicate of the pallet docs? Docs would be easier to maintain, and less ambiguity for devs to refer to, if we had one source of truth in the generated pallet docs. --- substrate/frame/staking/README.md | 12 +-- substrate/frame/staking/src/lib.rs | 93 ++++++++++----------- substrate/frame/staking/src/pallet/impls.rs | 2 +- 3 files changed, 50 insertions(+), 57 deletions(-) diff --git a/substrate/frame/staking/README.md b/substrate/frame/staking/README.md index 8c91cfcaa7fa..2938e2fe7706 100644 --- a/substrate/frame/staking/README.md +++ b/substrate/frame/staking/README.md @@ -24,7 +24,7 @@ be found not to be discharging its duties properly. - Nominating: The process of placing staked funds behind one or more validators in order to share in any reward, and punishment, they take. - Stash account: The account holding an owner's funds used for staking. -- Controller account: The account that controls an owner's funds for staking. +- Controller account (being deprecated): The account that controls an owner's funds for staking. - Era: A (whole) number of sessions, which is the period that the validator set (and each validator's active nominator set) is recalculated and where rewards are paid out. - Slash: The punishment of a staker by reducing its funds. @@ -45,10 +45,10 @@ The staking system in Substrate NPoS is designed to make the following possible: Almost any interaction with the Staking module requires a process of _**bonding**_ (also known as being a _staker_). To become *bonded*, a fund-holding account known as the _stash account_, which holds some or all of the funds that become -frozen in place as part of the staking process, is paired with an active **controller** account, which issues -instructions on how they shall be used. +frozen in place as part of the staking process. The controller account, which this pallet now assigns the stash account to, +issues instructions on how funds shall be used. -An account pair can become bonded using the +An account can become a bonded stash account using the [`bond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.bond) call. Stash accounts can update their associated controller back to their stash account using the @@ -231,8 +231,8 @@ following: Any funds already placed into stash can be the target of the following operations: The controller account can free a portion (or all) of the funds using the -[`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond) call. Note that the funds -are not immediately accessible. Instead, a duration denoted by +[`unbond`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.unbond) call. Note that the +funds are not immediately accessible. Instead, a duration denoted by [`BondingDuration`](https://docs.rs/pallet-staking/latest/pallet_staking/trait.Config.html#associatedtype.BondingDuration) (in number of eras) must pass until the funds can actually be removed. Once the `BondingDuration` is over, the [`withdraw_unbonded`](https://docs.rs/pallet-staking/latest/pallet_staking/enum.Call.html#variant.withdraw_unbonded) diff --git a/substrate/frame/staking/src/lib.rs b/substrate/frame/staking/src/lib.rs index 2cfee38ab4fa..de989e8943de 100644 --- a/substrate/frame/staking/src/lib.rs +++ b/substrate/frame/staking/src/lib.rs @@ -41,7 +41,7 @@ //! - Nominating: The process of placing staked funds behind one or more validators in order to //! share in any reward, and punishment, they take. //! - Stash account: The account holding an owner's funds used for staking. -//! - Controller account: The account that controls an owner's funds for staking. +//! - Controller account (being deprecated): The account that controls an owner's funds for staking. //! - Era: A (whole) number of sessions, which is the period that the validator set (and each //! validator's active nominator set) is recalculated and where rewards are paid out. //! - Slash: The punishment of a staker by reducing its funds. @@ -61,20 +61,20 @@ //! //! Almost any interaction with the Staking pallet requires a process of _**bonding**_ (also known //! as being a _staker_). To become *bonded*, a fund-holding register known as the _stash account_, -//! which holds some or all of the funds that become frozen in place as part of the staking process, -//! is paired with an active **controller** account, which issues instructions on how they shall be -//! used. +//! which holds some or all of the funds that become frozen in place as part of the staking process. +//! The controller account, which this pallet now assigns the stash account to, issues instructions +//! on how funds shall be used. //! -//! An account pair can become bonded using the [`bond`](Call::bond) call. +//! An account can become a bonded stash account using the [`bond`](Call::bond) call. //! -//! Stash accounts can update their associated controller back to the stash account using the +//! In the event stash accounts registered a unique controller account before the controller account +//! deprecation, they can update their associated controller back to the stash account using the //! [`set_controller`](Call::set_controller) call. //! //! There are three possible roles that any staked account pair can be in: `Validator`, `Nominator` -//! and `Idle` (defined in [`StakerStatus`]). There are three -//! corresponding instructions to change between roles, namely: -//! [`validate`](Call::validate), -//! [`nominate`](Call::nominate), and [`chill`](Call::chill). +//! and `Idle` (defined in [`StakerStatus`]). There are three corresponding instructions to change +//! between roles, namely: [`validate`](Call::validate), [`nominate`](Call::nominate), and +//! [`chill`](Call::chill). //! //! #### Validating //! @@ -85,14 +85,13 @@ //! _might_ get elected at the _next era_ as a validator. The result of the election is determined //! by nominators and their votes. //! -//! An account can become a validator candidate via the -//! [`validate`](Call::validate) call. +//! An account can become a validator candidate via the [`validate`](Call::validate) call. //! //! #### Nomination //! //! A **nominator** does not take any _direct_ role in maintaining the network, instead, it votes on -//! a set of validators to be elected. Once interest in nomination is stated by an account, it -//! takes effect at the next election round. The funds in the nominator's stash account indicate the +//! a set of validators to be elected. Once interest in nomination is stated by an account, it takes +//! effect at the next election round. The funds in the nominator's stash account indicate the //! _weight_ of its vote. Both the rewards and any punishment that a validator earns are shared //! between the validator and its nominators. This rule incentivizes the nominators to NOT vote for //! the misbehaving/offline validators as much as possible, simply because the nominators will also @@ -104,8 +103,8 @@ //! //! Staking is closely related to elections; actual validators are chosen from among all potential //! validators via election by the potential validators and nominators. To reduce use of the phrase -//! "potential validators and nominators", we often use the term **voters**, who are simply -//! the union of potential validators and nominators. +//! "potential validators and nominators", we often use the term **voters**, who are simply the +//! union of potential validators and nominators. //! //! #### Rewards and Slash //! @@ -117,10 +116,9 @@ //! `payout_stakers`, which pays the reward to the validator as well as its nominators. Only //! [`Config::MaxExposurePageSize`] nominator rewards can be claimed in a single call. When the //! number of nominators exceeds [`Config::MaxExposurePageSize`], then the exposed nominators are -//! stored in multiple pages, with each page containing up to -//! [`Config::MaxExposurePageSize`] nominators. To pay out all nominators, `payout_stakers` must be -//! called once for each available page. Paging exists to limit the i/o cost to mutate storage for -//! each nominator's account. +//! stored in multiple pages, with each page containing up to [`Config::MaxExposurePageSize`] +//! nominators. To pay out all nominators, `payout_stakers` must be called once for each available +//! page. Paging exists to limit the i/o cost to mutate storage for each nominator's account. //! //! Slashing can occur at any point in time, once misbehavior is reported. Once slashing is //! determined, a value is deducted from the balance of the validator and all the nominators who @@ -165,18 +163,18 @@ //! //! #[frame_support::pallet(dev_mode)] //! pub mod pallet { -//! use super::*; -//! use frame_support::pallet_prelude::*; -//! use frame_system::pallet_prelude::*; +//! use super::*; +//! use frame_support::pallet_prelude::*; +//! use frame_system::pallet_prelude::*; //! -//! #[pallet::pallet] -//! pub struct Pallet(_); +//! #[pallet::pallet] +//! pub struct Pallet(_); //! -//! #[pallet::config] -//! pub trait Config: frame_system::Config + staking::Config {} +//! #[pallet::config] +//! pub trait Config: frame_system::Config + staking::Config {} //! -//! #[pallet::call] -//! impl Pallet { +//! #[pallet::call] +//! impl Pallet { //! /// Reward a validator. //! #[pallet::weight(0)] //! pub fn reward_myself(origin: OriginFor) -> DispatchResult { @@ -193,8 +191,8 @@ //! //! ### Era payout //! -//! The era payout is computed using yearly inflation curve defined at -//! [`Config::EraPayout`] as such: +//! The era payout is computed using yearly inflation curve defined at [`Config::EraPayout`] as +//! such: //! //! ```nocompile //! staker_payout = yearly_inflation(npos_token_staked / total_tokens) * total_tokens / era_per_year @@ -204,8 +202,7 @@ //! ```nocompile //! remaining_payout = max_yearly_inflation * total_tokens / era_per_year - staker_payout //! ``` -//! The remaining reward is send to the configurable end-point -//! [`Config::RewardRemainder`]. +//! The remaining reward is send to the configurable end-point [`Config::RewardRemainder`]. //! //! ### Reward Calculation //! @@ -219,9 +216,8 @@ //! they received during the era. Points are added to a validator using //! [`reward_by_ids`](Pallet::reward_by_ids). //! -//! [`Pallet`] implements -//! [`pallet_authorship::EventHandler`] to add reward -//! points to block producer and block producer of referenced uncles. +//! [`Pallet`] implements [`pallet_authorship::EventHandler`] to add reward points to block producer +//! and block producer of referenced uncles. //! //! The validator and its nominator split their reward as following: //! @@ -232,13 +228,12 @@ //! validator, proportional to the value staked behind the validator (_i.e._ dividing the //! [`own`](Exposure::own) or [`others`](Exposure::others) by [`total`](Exposure::total) in //! [`Exposure`]). Note that payouts are made in pages with each page capped at -//! [`Config::MaxExposurePageSize`] nominators. The distribution of nominators across -//! pages may be unsorted. The total commission is paid out proportionally across pages based on the -//! total stake of the page. +//! [`Config::MaxExposurePageSize`] nominators. The distribution of nominators across pages may be +//! unsorted. The total commission is paid out proportionally across pages based on the total stake +//! of the page. //! //! All entities who receive a reward have the option to choose their reward destination through the -//! [`Payee`] storage item (see -//! [`set_payee`](Call::set_payee)), to be one of the following: +//! [`Payee`] storage item (see [`set_payee`](Call::set_payee)), to be one of the following: //! //! - Stash account, not increasing the staked value. //! - Stash account, also increasing the staked value. @@ -249,12 +244,10 @@ //! Any funds already placed into stash can be the target of the following operations: //! //! The controller account can free a portion (or all) of the funds using the -//! [`unbond`](Call::unbond) call. Note that the funds are not immediately -//! accessible. Instead, a duration denoted by -//! [`Config::BondingDuration`] (in number of eras) must -//! pass until the funds can actually be removed. Once the `BondingDuration` is over, the -//! [`withdraw_unbonded`](Call::withdraw_unbonded) call can be used to actually -//! withdraw the funds. +//! [`unbond`](Call::unbond) call. Note that the funds are not immediately accessible. Instead, a +//! duration denoted by [`Config::BondingDuration`] (in number of eras) must pass until the funds +//! can actually be removed. Once the `BondingDuration` is over, the +//! [`withdraw_unbonded`](Call::withdraw_unbonded) call can be used to actually withdraw the funds. //! //! Note that there is a limitation to the number of fund-chunks that can be scheduled to be //! unlocked in the future via [`unbond`](Call::unbond). In case this maximum @@ -274,8 +267,8 @@ //! //! ## GenesisConfig //! -//! The Staking pallet depends on the [`GenesisConfig`]. The -//! `GenesisConfig` is optional and allow to set some initial stakers. +//! The Staking pallet depends on the [`GenesisConfig`]. The `GenesisConfig` is optional and allow +//! to set some initial stakers. //! //! ## Related Modules //! @@ -405,7 +398,7 @@ pub enum RewardDestination { /// Pay into the stash account, not increasing the amount at stake. Stash, #[deprecated( - note = "`Controller` will be removed after January 2024. Use `Account(controller)` instead. This variant now behaves the same as `Stash` variant." + note = "`Controller` will be removed after January 2024. Use `Account(controller)` instead." )] Controller, /// Pay into a specified account. diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index 6ea8e4c9d3b2..e18d9378b5e9 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -359,7 +359,7 @@ impl Pallet { #[allow(deprecated)] RewardDestination::Controller => Self::bonded(stash) .map(|controller| { - defensive!("Paying out controller as reward destination which is deprecated and should be migrated"); + defensive!("Paying out controller as reward destination which is deprecated and should be migrated."); // This should never happen once payees with a `Controller` variant have been migrated. // But if it does, just pay the controller account. T::Currency::deposit_creating(&controller, amount) From a00a767604db59e4f045f8e73e90c6c8b7ff95c5 Mon Sep 17 00:00:00 2001 From: Lulu Date: Fri, 24 Nov 2023 09:07:35 +0000 Subject: [PATCH 45/48] Make publish CI consistant and bump to v0.3.0 (#2453) --- .github/workflows/check-publish.yml | 7 ++----- .github/workflows/claim-crates.yml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/check-publish.yml b/.github/workflows/check-publish.yml index c0d2b8893813..db0863888b83 100644 --- a/.github/workflows/check-publish.yml +++ b/.github/workflows/check-publish.yml @@ -10,10 +10,7 @@ on: jobs: check-publish: - strategy: - matrix: - os: ["ubuntu-latest"] - runs-on: ${{ matrix.os }} + runs-on: ubuntu-latest steps: - uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4.1.0 @@ -23,7 +20,7 @@ jobs: cache-on-failure: true - name: install parity-publish - run: cargo install parity-publish --profile dev + run: cargo install parity-publish@0.3.0 - name: parity-publish check run: parity-publish check --allow-unpublished diff --git a/.github/workflows/claim-crates.yml b/.github/workflows/claim-crates.yml index 345d24c75667..0bd5593b54f2 100644 --- a/.github/workflows/claim-crates.yml +++ b/.github/workflows/claim-crates.yml @@ -18,7 +18,7 @@ jobs: cache-on-failure: true - name: install parity-publish - run: cargo install parity-publish@0.2.0 + run: cargo install parity-publish@0.3.0 - name: parity-publish claim env: From 4163152506883c985e82d2730693910aabd65a2d Mon Sep 17 00:00:00 2001 From: Xiliang Chen Date: Fri, 24 Nov 2023 22:13:57 +1300 Subject: [PATCH 46/48] pallet-xcm: ensure xcm outcome is always complete, revert effects otherwise (#2405) On extrinsics/call, ensure local XCM execution is complete/successful. Otherwise, fail the extrinsic so that state changes don't get committed to the db. Added regression tests that fail without the fix. fixes #2237 --------- Co-authored-by: Adrian Catangiu --- Cargo.lock | 1 + .../assets/test-utils/src/test_cases.rs | 8 +- .../test-utils/src/test_cases_over_bridge.rs | 4 +- polkadot/xcm/pallet-xcm/src/lib.rs | 28 ++++-- polkadot/xcm/pallet-xcm/src/mock.rs | 7 ++ .../pallet-xcm/src/tests/assets_transfer.rs | 58 +++++++++++++ polkadot/xcm/pallet-xcm/src/tests/mod.rs | 48 ++++++++++- polkadot/xcm/src/v3/traits.rs | 4 +- .../xcm-executor/integration-tests/Cargo.toml | 1 + .../xcm-executor/integration-tests/src/lib.rs | 86 ++++++++++++++----- 10 files changed, 210 insertions(+), 35 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index abd539ea799c..ab51be3d5dbc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -21504,6 +21504,7 @@ dependencies = [ "frame-support", "frame-system", "futures", + "pallet-transaction-payment", "pallet-xcm", "parity-scale-codec", "polkadot-test-client", diff --git a/cumulus/parachains/runtimes/assets/test-utils/src/test_cases.rs b/cumulus/parachains/runtimes/assets/test-utils/src/test_cases.rs index f1cc76350a00..915d99470c36 100644 --- a/cumulus/parachains/runtimes/assets/test-utils/src/test_cases.rs +++ b/cumulus/parachains/runtimes/assets/test-utils/src/test_cases.rs @@ -152,7 +152,7 @@ pub fn teleports_for_native_asset_works< hash, RuntimeHelper::::xcm_max_weight(XcmReceivedFrom::Parent), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); // check Balances after assert_ne!(>::free_balance(&target_account), 0.into()); @@ -499,7 +499,7 @@ pub fn teleports_for_foreign_assets_works< hash, RuntimeHelper::::xcm_max_weight(XcmReceivedFrom::Sibling), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); // checks target_account after assert_eq!( @@ -1211,7 +1211,7 @@ pub fn create_and_manage_foreign_assets_for_local_consensus_parachain_assets_wor hash, RuntimeHelper::::xcm_max_weight(XcmReceivedFrom::Sibling), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); // check events let mut events = >::events() @@ -1319,7 +1319,7 @@ pub fn create_and_manage_foreign_assets_for_local_consensus_parachain_assets_wor hash, RuntimeHelper::::xcm_max_weight(XcmReceivedFrom::Sibling), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); additional_checks_after(); }) diff --git a/cumulus/parachains/runtimes/assets/test-utils/src/test_cases_over_bridge.rs b/cumulus/parachains/runtimes/assets/test-utils/src/test_cases_over_bridge.rs index 851fcd5c7d68..8007b275cb51 100644 --- a/cumulus/parachains/runtimes/assets/test-utils/src/test_cases_over_bridge.rs +++ b/cumulus/parachains/runtimes/assets/test-utils/src/test_cases_over_bridge.rs @@ -483,7 +483,7 @@ pub fn receive_reserve_asset_deposited_from_different_consensus_works< XcmReceivedFrom::Sibling, ), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); // author actual balance after (received fees from Trader for ForeignAssets) let author_received_fees = @@ -588,7 +588,7 @@ pub fn report_bridge_status_from_xcm_bridge_router_works< hash, RuntimeHelper::::xcm_max_weight(XcmReceivedFrom::Sibling), ); - assert_eq!(outcome.ensure_complete(), Ok(())); + assert_ok!(outcome.ensure_complete()); assert_eq!(is_congested, pallet_xcm_bridge_hub_router::Pallet::::bridge().is_congested); }; diff --git a/polkadot/xcm/pallet-xcm/src/lib.rs b/polkadot/xcm/pallet-xcm/src/lib.rs index 38ea7555fc32..74a24b132da7 100644 --- a/polkadot/xcm/pallet-xcm/src/lib.rs +++ b/polkadot/xcm/pallet-xcm/src/lib.rs @@ -547,7 +547,7 @@ pub mod pallet { InvalidAssetUnsupportedReserve, /// Too many assets with different reserve locations have been attempted for transfer. TooManyReserves, - /// Local XCM execution of asset transfer incomplete. + /// Local XCM execution incomplete. LocalExecutionIncomplete, } @@ -1009,8 +1009,14 @@ pub mod pallet { message: Box::RuntimeCall>>, max_weight: Weight, ) -> DispatchResultWithPostInfo { + log::trace!(target: "xcm::pallet_xcm::execute", "message {:?}, max_weight {:?}", message, max_weight); let outcome = >::execute(origin, message, max_weight)?; - Ok(Some(outcome.weight_used().saturating_add(T::WeightInfo::execute())).into()) + let weight_used = outcome.weight_used(); + outcome.ensure_complete().map_err(|error| { + log::error!(target: "xcm::pallet_xcm::execute", "XCM execution failed with error {:?}", error); + Error::::LocalExecutionIncomplete + })?; + Ok(Some(weight_used.saturating_add(T::WeightInfo::execute())).into()) } /// Extoll that a particular destination can be communicated with through a particular @@ -1495,13 +1501,25 @@ impl Pallet { let outcome = T::XcmExecutor::execute_xcm_in_credit(origin, local_xcm, hash, weight, weight); Self::deposit_event(Event::Attempted { outcome: outcome.clone() }); - if let Some(remote_xcm) = remote_xcm { - outcome.ensure_complete().map_err(|_| Error::::LocalExecutionIncomplete)?; + outcome.ensure_complete().map_err(|error| { + log::error!( + target: "xcm::pallet_xcm::build_and_execute_xcm_transfer_type", + "XCM execution failed with error {:?}", error + ); + Error::::LocalExecutionIncomplete + })?; + if let Some(remote_xcm) = remote_xcm { let (ticket, price) = validate_send::(dest, remote_xcm.clone()) .map_err(Error::::from)?; if origin != Here.into_location() { - Self::charge_fees(origin, price).map_err(|_| Error::::FeesNotMet)?; + Self::charge_fees(origin, price).map_err(|error| { + log::error!( + target: "xcm::pallet_xcm::build_and_execute_xcm_transfer_type", + "Unable to charge fee with error {:?}", error + ); + Error::::FeesNotMet + })?; } let message_id = T::XcmRouter::deliver(ticket).map_err(Error::::from)?; diff --git a/polkadot/xcm/pallet-xcm/src/mock.rs b/polkadot/xcm/pallet-xcm/src/mock.rs index e744cefb162c..0b0f795100cd 100644 --- a/polkadot/xcm/pallet-xcm/src/mock.rs +++ b/polkadot/xcm/pallet-xcm/src/mock.rs @@ -153,6 +153,7 @@ construct_runtime!( thread_local! { pub static SENT_XCM: RefCell)>> = RefCell::new(Vec::new()); + pub static FAIL_SEND_XCM: RefCell = RefCell::new(false); } pub(crate) fn sent_xcm() -> Vec<(MultiLocation, Xcm<()>)> { SENT_XCM.with(|q| (*q.borrow()).clone()) @@ -164,6 +165,9 @@ pub(crate) fn take_sent_xcm() -> Vec<(MultiLocation, Xcm<()>)> { r }) } +pub(crate) fn set_send_xcm_artificial_failure(should_fail: bool) { + FAIL_SEND_XCM.with(|q| *q.borrow_mut() = should_fail); +} /// Sender that never returns error. pub struct TestSendXcm; impl SendXcm for TestSendXcm { @@ -172,6 +176,9 @@ impl SendXcm for TestSendXcm { dest: &mut Option, msg: &mut Option>, ) -> SendResult<(MultiLocation, Xcm<()>)> { + if FAIL_SEND_XCM.with(|q| *q.borrow()) { + return Err(SendError::Transport("Intentional send failure used in tests")) + } let pair = (dest.take().unwrap(), msg.take().unwrap()); Ok((pair, MultiAssets::new())) } diff --git a/polkadot/xcm/pallet-xcm/src/tests/assets_transfer.rs b/polkadot/xcm/pallet-xcm/src/tests/assets_transfer.rs index bf39e1ca2883..d1b298765e22 100644 --- a/polkadot/xcm/pallet-xcm/src/tests/assets_transfer.rs +++ b/polkadot/xcm/pallet-xcm/src/tests/assets_transfer.rs @@ -1457,3 +1457,61 @@ fn reserve_transfer_assets_with_filtered_teleported_fee_disallowed() { ); }); } + +/// Test failure to complete execution of local XCM instructions reverts intermediate side-effects. +/// +/// Extrinsic will execute XCM to withdraw & burn reserve-based assets, then fail sending XCM to +/// reserve chain for releasing reserve assets. Assert that the previous instructions (withdraw & +/// burn) effects are reverted. +#[test] +fn intermediary_error_reverts_side_effects() { + let balances = vec![(ALICE, INITIAL_BALANCE)]; + let beneficiary: MultiLocation = + Junction::AccountId32 { network: None, id: ALICE.into() }.into(); + new_test_ext_with_balances(balances).execute_with(|| { + // create sufficient foreign asset USDC (0 total issuance) + let usdc_initial_local_amount = 142; + let (_, usdc_chain_sovereign_account, usdc_id_multilocation) = set_up_foreign_asset( + USDC_RESERVE_PARA_ID, + Some(USDC_INNER_JUNCTION), + usdc_initial_local_amount, + true, + ); + + // transfer destination is some other parachain + let dest = RelayLocation::get().pushed_with_interior(Parachain(OTHER_PARA_ID)).unwrap(); + + let assets: MultiAssets = vec![(usdc_id_multilocation, SEND_AMOUNT).into()].into(); + let fee_index = 0; + + // balances checks before + assert_eq!(Assets::balance(usdc_id_multilocation, ALICE), usdc_initial_local_amount); + assert_eq!(Balances::free_balance(ALICE), INITIAL_BALANCE); + + // introduce artificial error in sending outbound XCM + set_send_xcm_artificial_failure(true); + + // do the transfer - extrinsic should completely fail on xcm send failure + assert!(XcmPallet::limited_reserve_transfer_assets( + RuntimeOrigin::signed(ALICE), + Box::new(dest.into()), + Box::new(beneficiary.into()), + Box::new(assets.into()), + fee_index as u32, + Unlimited, + ) + .is_err()); + + // Alice no changes + assert_eq!(Assets::balance(usdc_id_multilocation, ALICE), usdc_initial_local_amount); + assert_eq!(Balances::free_balance(ALICE), INITIAL_BALANCE); + // Destination account (parachain account) no changes + assert_eq!(Balances::free_balance(usdc_chain_sovereign_account.clone()), 0); + assert_eq!(Assets::balance(usdc_id_multilocation, usdc_chain_sovereign_account), 0); + // Verify total and active issuance of USDC has not changed + assert_eq!(Assets::total_issuance(usdc_id_multilocation), usdc_initial_local_amount); + assert_eq!(Assets::active_issuance(usdc_id_multilocation), usdc_initial_local_amount); + // Verify no XCM program sent + assert_eq!(sent_xcm(), vec![]); + }); +} diff --git a/polkadot/xcm/pallet-xcm/src/tests/mod.rs b/polkadot/xcm/pallet-xcm/src/tests/mod.rs index 72814e507f2a..056c7dcc1968 100644 --- a/polkadot/xcm/pallet-xcm/src/tests/mod.rs +++ b/polkadot/xcm/pallet-xcm/src/tests/mod.rs @@ -445,7 +445,7 @@ fn trapped_assets_can_be_claimed() { assert_eq!(AssetTraps::::iter().collect::>(), vec![]); let weight = BaseXcmWeight::get() * 3; - assert_ok!(XcmPallet::execute( + assert_ok!(>::execute( RuntimeOrigin::signed(ALICE), Box::new(VersionedXcm::from(Xcm(vec![ ClaimAsset { assets: (Here, SEND_AMOUNT).into(), ticket: Here.into() }, @@ -459,6 +459,52 @@ fn trapped_assets_can_be_claimed() { }); } +/// Test failure to complete execution reverts intermediate side-effects. +/// +/// XCM program will withdraw and deposit some assets, then fail execution of a further withdraw. +/// Assert that the previous instructions effects are reverted. +#[test] +fn incomplete_execute_reverts_side_effects() { + let balances = vec![(ALICE, INITIAL_BALANCE), (BOB, INITIAL_BALANCE)]; + new_test_ext_with_balances(balances).execute_with(|| { + let weight = BaseXcmWeight::get() * 4; + let dest: MultiLocation = Junction::AccountId32 { network: None, id: BOB.into() }.into(); + assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); + let amount_to_send = INITIAL_BALANCE - ExistentialDeposit::get(); + let assets: MultiAssets = (Here, amount_to_send).into(); + let result = XcmPallet::execute( + RuntimeOrigin::signed(ALICE), + Box::new(VersionedXcm::from(Xcm(vec![ + // Withdraw + BuyExec + Deposit should work + WithdrawAsset(assets.clone()), + buy_execution(assets.inner()[0].clone()), + DepositAsset { assets: assets.clone().into(), beneficiary: dest }, + // Withdrawing once more will fail because of InsufficientBalance, and we expect to + // revert the effects of the above instructions as well + WithdrawAsset(assets), + ]))), + weight, + ); + // all effects are reverted and balances unchanged for either sender or receiver + assert_eq!(Balances::total_balance(&ALICE), INITIAL_BALANCE); + assert_eq!(Balances::total_balance(&BOB), INITIAL_BALANCE); + assert_eq!( + result, + Err(sp_runtime::DispatchErrorWithPostInfo { + post_info: frame_support::dispatch::PostDispatchInfo { + actual_weight: None, + pays_fee: frame_support::dispatch::Pays::Yes, + }, + error: sp_runtime::DispatchError::Module(sp_runtime::ModuleError { + index: 4, + error: [24, 0, 0, 0,], + message: Some("LocalExecutionIncomplete") + }) + }) + ); + }); +} + #[test] fn fake_latest_versioned_multilocation_works() { use codec::Encode; diff --git a/polkadot/xcm/src/v3/traits.rs b/polkadot/xcm/src/v3/traits.rs index 1043d17b7106..6054bf1456a5 100644 --- a/polkadot/xcm/src/v3/traits.rs +++ b/polkadot/xcm/src/v3/traits.rs @@ -275,9 +275,9 @@ pub enum Outcome { } impl Outcome { - pub fn ensure_complete(self) -> Result { + pub fn ensure_complete(self) -> result::Result { match self { - Outcome::Complete(_) => Ok(()), + Outcome::Complete(weight) => Ok(weight), Outcome::Incomplete(_, e) => Err(e), Outcome::Error(e) => Err(e), } diff --git a/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml b/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml index d869fc6f2dcd..ddb45965ee41 100644 --- a/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml +++ b/polkadot/xcm/xcm-executor/integration-tests/Cargo.toml @@ -12,6 +12,7 @@ codec = { package = "parity-scale-codec", version = "3.6.1" } frame-support = { path = "../../../../substrate/frame/support", default-features = false } frame-system = { path = "../../../../substrate/frame/system" } futures = "0.3.21" +pallet-transaction-payment = { path = "../../../../substrate/frame/transaction-payment" } pallet-xcm = { path = "../../pallet-xcm" } polkadot-test-client = { path = "../../../node/test/client" } polkadot-test-runtime = { path = "../../../runtime/test-runtime" } diff --git a/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs b/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs index d8c77f8317e1..c02cb218885f 100644 --- a/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs +++ b/polkadot/xcm/xcm-executor/integration-tests/src/lib.rs @@ -76,33 +76,59 @@ fn transact_recursion_limit_works() { sp_tracing::try_init_simple(); let mut client = TestClientBuilder::new().build(); - let mut msg = Xcm(vec![ClearOrigin]); - let max_weight = ::Weigher::weight(&mut msg).unwrap(); - let mut call = polkadot_test_runtime::RuntimeCall::Xcm(pallet_xcm::Call::execute { - message: Box::new(VersionedXcm::from(msg)), - max_weight, - }); - - for _ in 0..11 { - let mut msg = Xcm(vec![ - WithdrawAsset((Parent, 1_000).into()), - BuyExecution { fees: (Parent, 1).into(), weight_limit: Unlimited }, + let base_xcm = |call: polkadot_test_runtime::RuntimeCall| { + Xcm(vec![ + WithdrawAsset((Here, 1_000).into()), + BuyExecution { fees: (Here, 1).into(), weight_limit: Unlimited }, Transact { origin_kind: OriginKind::Native, require_weight_at_most: call.get_dispatch_info().weight, call: call.encode().into(), }, - ]); + ]) + }; + let mut call: Option = None; + // set up transacts with recursive depth of 11 + for depth in (1..12).rev() { + let mut msg; + match depth { + // this one should fail with `XcmError::ExceedsStackLimit` + 11 => { + msg = Xcm(vec![ClearOrigin]); + }, + // this one checks that the inner one (depth 11) fails as expected, + // itself should not fail => should have outcome == Complete + 10 => { + let inner_call = call.take().unwrap(); + let expected_transact_status = + sp_runtime::DispatchError::Module(sp_runtime::ModuleError { + index: 27, + error: [24, 0, 0, 0], + message: Some("LocalExecutionIncomplete"), + }) + .encode() + .into(); + msg = base_xcm(inner_call); + msg.inner_mut().push(ExpectTransactStatus(expected_transact_status)); + }, + // these are the outer 9 calls that expect `ExpectTransactStatus(Success)` + d if d >= 1 && d <= 9 => { + let inner_call = call.take().unwrap(); + msg = base_xcm(inner_call); + msg.inner_mut().push(ExpectTransactStatus(MaybeErrorCode::Success)); + }, + _ => unreachable!(), + } let max_weight = ::Weigher::weight(&mut msg).unwrap(); - call = polkadot_test_runtime::RuntimeCall::Xcm(pallet_xcm::Call::execute { - message: Box::new(VersionedXcm::from(msg)), + call = Some(polkadot_test_runtime::RuntimeCall::Xcm(pallet_xcm::Call::execute { + message: Box::new(VersionedXcm::from(msg.clone())), max_weight, - }); + })); } let mut block_builder = client.init_polkadot_block_builder(); - let execute = construct_extrinsic(&client, call, sp_keyring::Sr25519Keyring::Alice, 0); + let execute = construct_extrinsic(&client, call.unwrap(), sp_keyring::Sr25519Keyring::Alice, 0); block_builder.push_polkadot_extrinsic(execute).expect("pushes extrinsic"); @@ -113,11 +139,29 @@ fn transact_recursion_limit_works() { .expect("imports the block"); client.state_at(block_hash).expect("state should exist").inspect_state(|| { - assert!(polkadot_test_runtime::System::events().iter().any(|r| matches!( - r.event, - polkadot_test_runtime::RuntimeEvent::Xcm(pallet_xcm::Event::Attempted { - outcome: Outcome::Incomplete(_, XcmError::ExceedsStackLimit) - }), + let events = polkadot_test_runtime::System::events(); + // verify 10 pallet_xcm calls were successful + assert_eq!( + polkadot_test_runtime::System::events() + .iter() + .filter(|r| matches!( + r.event, + polkadot_test_runtime::RuntimeEvent::Xcm(pallet_xcm::Event::Attempted { + outcome: Outcome::Complete(_) + }), + )) + .count(), + 10 + ); + // verify transaction fees have been paid + assert!(events.iter().any(|r| matches!( + &r.event, + polkadot_test_runtime::RuntimeEvent::TransactionPayment( + pallet_transaction_payment::Event::TransactionFeePaid { + who: payer, + .. + } + ) if *payer == sp_keyring::Sr25519Keyring::Alice.into(), ))); }); } From f086d540ae06b891f1a03bc1281809821d38a6f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Gon=C3=A7alo=20Pestana?= Date: Fri, 24 Nov 2023 10:36:22 +0100 Subject: [PATCH 47/48] pallet-staking: Converts all math operations to safe (#2435) This PR converts unsafe math operations to safe in the staking pallet. Closes https://github.com/paritytech/polkadot-sdk/issues/2431 --------- Co-authored-by: Ankan <10196091+Ank4n@users.noreply.github.com> --- substrate/frame/staking/src/lib.rs | 11 +++++---- substrate/frame/staking/src/pallet/impls.rs | 14 ++++++----- substrate/frame/staking/src/pallet/mod.rs | 4 ++- substrate/frame/staking/src/slashing.rs | 27 ++++++++++++--------- 4 files changed, 33 insertions(+), 23 deletions(-) diff --git a/substrate/frame/staking/src/lib.rs b/substrate/frame/staking/src/lib.rs index de989e8943de..e6250bb9681a 100644 --- a/substrate/frame/staking/src/lib.rs +++ b/substrate/frame/staking/src/lib.rs @@ -530,12 +530,12 @@ impl StakingLedger { let mut unlocking_balance = BalanceOf::::zero(); while let Some(last) = self.unlocking.last_mut() { - if unlocking_balance + last.value <= value { + if unlocking_balance.defensive_saturating_add(last.value) <= value { unlocking_balance += last.value; self.active += last.value; self.unlocking.pop(); } else { - let diff = value - unlocking_balance; + let diff = value.defensive_saturating_sub(unlocking_balance); unlocking_balance += diff; self.active += diff; @@ -589,7 +589,7 @@ impl StakingLedger { // for a `slash_era = x`, any chunk that is scheduled to be unlocked at era `x + 28` // (assuming 28 is the bonding duration) onwards should be slashed. - let slashable_chunks_start = slash_era + T::BondingDuration::get(); + let slashable_chunks_start = slash_era.saturating_add(T::BondingDuration::get()); // `Some(ratio)` if this is proportional, with `ratio`, `None` otherwise. In both cases, we // slash first the active chunk, and then `slash_chunks_priority`. @@ -1185,8 +1185,9 @@ impl EraInfo { let nominator_count = exposure.others.len(); // expected page count is the number of nominators divided by the page size, rounded up. - let expected_page_count = - nominator_count.defensive_saturating_add(page_size as usize - 1) / page_size as usize; + let expected_page_count = nominator_count + .defensive_saturating_add((page_size as usize).defensive_saturating_sub(1)) + .saturating_div(page_size as usize); let (exposure_metadata, exposure_pages) = exposure.into_pages(page_size); defensive_assert!(exposure_pages.len() == expected_page_count, "unexpected page count"); diff --git a/substrate/frame/staking/src/pallet/impls.rs b/substrate/frame/staking/src/pallet/impls.rs index e18d9378b5e9..d294686751cd 100644 --- a/substrate/frame/staking/src/pallet/impls.rs +++ b/substrate/frame/staking/src/pallet/impls.rs @@ -27,8 +27,8 @@ use frame_support::{ dispatch::WithPostDispatchInfo, pallet_prelude::*, traits::{ - Currency, Defensive, EstimateNextNewSession, Get, Imbalance, Len, OnUnbalanced, TryCollect, - UnixTime, + Currency, Defensive, DefensiveSaturating, EstimateNextNewSession, Get, Imbalance, Len, + OnUnbalanced, TryCollect, UnixTime, }, weights::Weight, }; @@ -148,7 +148,7 @@ impl Pallet { // `consolidate_unlocked` strictly subtracts balance. if new_total < old_total { // Already checked that this won't overflow by entry condition. - let value = old_total - new_total; + let value = old_total.defensive_saturating_sub(new_total); Self::deposit_event(Event::::Withdrawn { stash, amount: value }); } @@ -262,7 +262,8 @@ impl Pallet { // total commission validator takes across all nominator pages let validator_total_commission_payout = validator_commission * validator_total_payout; - let validator_leftover_payout = validator_total_payout - validator_total_commission_payout; + let validator_leftover_payout = + validator_total_payout.defensive_saturating_sub(validator_total_commission_payout); // Now let's calculate how this is split to the validator. let validator_exposure_part = Perbill::from_rational(exposure.own(), exposure.total()); let validator_staking_payout = validator_exposure_part * validator_leftover_payout; @@ -475,7 +476,7 @@ impl Pallet { bonded.push((active_era, start_session)); if active_era > bonding_duration { - let first_kept = active_era - bonding_duration; + let first_kept = active_era.defensive_saturating_sub(bonding_duration); // Prune out everything that's from before the first-kept index. let n_to_prune = @@ -501,7 +502,8 @@ impl Pallet { if let Some(active_era_start) = active_era.start { let now_as_millis_u64 = T::UnixTime::now().as_millis().saturated_into::(); - let era_duration = (now_as_millis_u64 - active_era_start).saturated_into::(); + let era_duration = (now_as_millis_u64.defensive_saturating_sub(active_era_start)) + .saturated_into::(); let staked = Self::eras_total_stake(&active_era.index); let issuance = T::Currency::total_issuance(); let (validator_payout, remainder) = diff --git a/substrate/frame/staking/src/pallet/mod.rs b/substrate/frame/staking/src/pallet/mod.rs index a68e9c90da9e..1f79ef63a475 100644 --- a/substrate/frame/staking/src/pallet/mod.rs +++ b/substrate/frame/staking/src/pallet/mod.rs @@ -1068,7 +1068,9 @@ pub mod pallet { ensure!(ledger.active >= min_active_bond, Error::::InsufficientBond); // Note: in case there is no current era it is fine to bond one era more. - let era = Self::current_era().unwrap_or(0) + T::BondingDuration::get(); + let era = Self::current_era() + .unwrap_or(0) + .defensive_saturating_add(T::BondingDuration::get()); if let Some(chunk) = ledger.unlocking.last_mut().filter(|chunk| chunk.era == era) { // To keep the chunk count down, we only keep one chunk per era. Since // `unlocking` is a FiFo queue, if a chunk exists for `era` we know that it will diff --git a/substrate/frame/staking/src/slashing.rs b/substrate/frame/staking/src/slashing.rs index 0d84d503733e..709fd1441ec3 100644 --- a/substrate/frame/staking/src/slashing.rs +++ b/substrate/frame/staking/src/slashing.rs @@ -57,7 +57,7 @@ use crate::{ use codec::{Decode, Encode, MaxEncodedLen}; use frame_support::{ ensure, - traits::{Currency, Defensive, Get, Imbalance, OnUnbalanced}, + traits::{Currency, Defensive, DefensiveSaturating, Get, Imbalance, OnUnbalanced}, }; use scale_info::TypeInfo; use sp_runtime::{ @@ -85,7 +85,7 @@ pub(crate) struct SlashingSpan { impl SlashingSpan { fn contains_era(&self, era: EraIndex) -> bool { - self.start <= era && self.length.map_or(true, |l| self.start + l > era) + self.start <= era && self.length.map_or(true, |l| self.start.saturating_add(l) > era) } } @@ -123,15 +123,15 @@ impl SlashingSpans { // returns `true` if a new span was started, `false` otherwise. `false` indicates // that internal state is unchanged. pub(crate) fn end_span(&mut self, now: EraIndex) -> bool { - let next_start = now + 1; + let next_start = now.defensive_saturating_add(1); if next_start <= self.last_start { return false } - let last_length = next_start - self.last_start; + let last_length = next_start.defensive_saturating_sub(self.last_start); self.prior.insert(0, last_length); self.last_start = next_start; - self.span_index += 1; + self.span_index.defensive_saturating_accrue(1); true } @@ -141,9 +141,9 @@ impl SlashingSpans { let mut index = self.span_index; let last = SlashingSpan { index, start: last_start, length: None }; let prior = self.prior.iter().cloned().map(move |length| { - let start = last_start - length; + let start = last_start.defensive_saturating_sub(length); last_start = start; - index -= 1; + index.defensive_saturating_reduce(1); SlashingSpan { index, start, length: Some(length) } }); @@ -164,13 +164,18 @@ impl SlashingSpans { let old_idx = self .iter() .skip(1) // skip ongoing span. - .position(|span| span.length.map_or(false, |len| span.start + len <= window_start)); + .position(|span| { + span.length + .map_or(false, |len| span.start.defensive_saturating_add(len) <= window_start) + }); - let earliest_span_index = self.span_index - self.prior.len() as SpanIndex; + let earliest_span_index = + self.span_index.defensive_saturating_sub(self.prior.len() as SpanIndex); let pruned = match old_idx { Some(o) => { self.prior.truncate(o); - let new_earliest = self.span_index - self.prior.len() as SpanIndex; + let new_earliest = + self.span_index.defensive_saturating_sub(self.prior.len() as SpanIndex); Some((earliest_span_index, new_earliest)) }, None => None, @@ -500,7 +505,7 @@ impl<'a, T: 'a + Config> InspectingSpans<'a, T> { let reward = if span_record.slashed < slash { // new maximum span slash. apply the difference. - let difference = slash - span_record.slashed; + let difference = slash.defensive_saturating_sub(span_record.slashed); span_record.slashed = slash; // compute reward. From e3242d2c1e2018395c218357046cc88caaed78f3 Mon Sep 17 00:00:00 2001 From: Julian Eager Date: Fri, 24 Nov 2023 18:32:43 +0800 Subject: [PATCH 48/48] Adapt test worker to profile flag (#2450) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit closes #2194 cc @mrcnski --------- Co-authored-by: Bastian Köcher --- .../benches/host_prepare_rococo_runtime.rs | 2 +- polkadot/node/core/pvf/build.rs | 21 +++++++++++++++++++ polkadot/node/core/pvf/src/testing.rs | 11 +++++----- polkadot/node/core/pvf/tests/it/main.rs | 2 +- .../node/core/pvf/tests/it/worker_common.rs | 6 +++--- 5 files changed, 32 insertions(+), 10 deletions(-) create mode 100644 polkadot/node/core/pvf/build.rs diff --git a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs index f2551b701c2c..e625a2303b5e 100644 --- a/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs +++ b/polkadot/node/core/pvf/benches/host_prepare_rococo_runtime.rs @@ -36,7 +36,7 @@ impl TestHost { where F: FnOnce(&mut Config), { - let (prepare_worker_path, execute_worker_path) = testing::build_workers_and_get_paths(true); + let (prepare_worker_path, execute_worker_path) = testing::build_workers_and_get_paths(); let cache_dir = tempfile::tempdir().unwrap(); let mut config = Config::new( diff --git a/polkadot/node/core/pvf/build.rs b/polkadot/node/core/pvf/build.rs new file mode 100644 index 000000000000..e01cc6deecc2 --- /dev/null +++ b/polkadot/node/core/pvf/build.rs @@ -0,0 +1,21 @@ +// Copyright (C) Parity Technologies (UK) Ltd. +// This file is part of Polkadot. + +// Polkadot is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// Polkadot is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. + +// You should have received a copy of the GNU General Public License +// along with Polkadot. If not, see . + +fn main() { + if let Ok(profile) = std::env::var("PROFILE") { + println!(r#"cargo:rustc-cfg=build_type="{}""#, profile); + } +} diff --git a/polkadot/node/core/pvf/src/testing.rs b/polkadot/node/core/pvf/src/testing.rs index 400b65bfe7d8..c7c885c43421 100644 --- a/polkadot/node/core/pvf/src/testing.rs +++ b/polkadot/node/core/pvf/src/testing.rs @@ -59,21 +59,22 @@ pub fn validate_candidate( /// /// NOTE: This should only be called in dev code (tests, benchmarks) as it relies on the relative /// paths of the built workers. -pub fn build_workers_and_get_paths(is_bench: bool) -> (PathBuf, PathBuf) { +pub fn build_workers_and_get_paths() -> (PathBuf, PathBuf) { // Only needs to be called once for the current process. static WORKER_PATHS: OnceLock> = OnceLock::new(); - fn build_workers(is_bench: bool) { + fn build_workers() { let mut build_args = vec![ "build", "--package=polkadot", "--bin=polkadot-prepare-worker", "--bin=polkadot-execute-worker", ]; - if is_bench { - // Benches require --release. Regular tests are debug (no flag needed). + + if cfg!(build_type = "release") { build_args.push("--release"); } + let mut cargo = std::process::Command::new("cargo"); let cmd = cargo // wasm runtime not needed @@ -117,7 +118,7 @@ pub fn build_workers_and_get_paths(is_bench: bool) -> (PathBuf, PathBuf) { } } - build_workers(is_bench); + build_workers(); Mutex::new((prepare_worker_path, execute_worker_path)) }); diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index b53d598cd0f7..bd6b04182fd7 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -52,7 +52,7 @@ impl TestHost { where F: FnOnce(&mut Config), { - let (prepare_worker_path, execute_worker_path) = build_workers_and_get_paths(false); + let (prepare_worker_path, execute_worker_path) = build_workers_and_get_paths(); let cache_dir = tempfile::tempdir().unwrap(); let mut config = Config::new( diff --git a/polkadot/node/core/pvf/tests/it/worker_common.rs b/polkadot/node/core/pvf/tests/it/worker_common.rs index 0d33af7e096c..4b736b08ba60 100644 --- a/polkadot/node/core/pvf/tests/it/worker_common.rs +++ b/polkadot/node/core/pvf/tests/it/worker_common.rs @@ -23,7 +23,7 @@ use std::{env, time::Duration}; // Test spawning a program that immediately exits with a failure code. #[tokio::test] async fn spawn_immediate_exit() { - let (prepare_worker_path, _) = build_workers_and_get_paths(false); + let (prepare_worker_path, _) = build_workers_and_get_paths(); // There's no explicit `exit` subcommand in the worker; it will panic on an unknown // subcommand anyway @@ -41,7 +41,7 @@ async fn spawn_immediate_exit() { #[tokio::test] async fn spawn_timeout() { - let (_, execute_worker_path) = build_workers_and_get_paths(false); + let (_, execute_worker_path) = build_workers_and_get_paths(); let result = spawn_with_program_path( "integration-test", @@ -57,7 +57,7 @@ async fn spawn_timeout() { #[tokio::test] async fn should_connect() { - let (prepare_worker_path, _) = build_workers_and_get_paths(false); + let (prepare_worker_path, _) = build_workers_and_get_paths(); let _ = spawn_with_program_path( "integration-test",