Update substrate and subxt (#435)

* one test working

* fix all tests

* lint

* remove unwraps

* lint

* fix the batch sizing

* lint

* fix some pallet mocks

* clean

* fix propogation tests

* fix all tests

* fix constraints pallet

* remove commented out assert

* remove todo

Author: JesseAbram

Cargo.lock

@@ -52,7 +52,7 @@ footer="""
 """
 # postprocessors
 postprocessors=[
-  { pattern='<REPO>', replace="https://github.com/orhun/git-cliff" }, # replace repository URL
+  { pattern='<REPO>', replace="https://github.com/entropy/entropy-core" }, # replace repository URL
 ]
 
 [git]

cliff.toml

@@ -6,7 +6,7 @@ edition="2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-thiserror="1.0.40"
+thiserror="1.0.48"
 
 # parsing
 

crypto/constraints/Cargo.toml

@@ -7,7 +7,7 @@ edition="2021"
 # Common
 rand     ={ version="0.8", default-features=false }
 serde    ={ version="1.0", features=["derive"] }
-thiserror="1.0.40"
+thiserror="1.0.48"
 hex      ="0.4"
 
 # Substrate

crypto/kvdb/Cargo.toml

@@ -12,14 +12,14 @@ entropy-shared ={ path="../shared", default-features=false }
 synedrion      ={ git="ssh://[email protected]/entropyxyz/synedrion.git", tag="v0.0.9" }
 serde          ={ version="1.0", features=["derive"], default-features=false }
 serde-big-array="0.5.1"
-subxt          ={ package="subxt", git="https://github.com/paritytech/subxt.git", tag="v0.29.0", default-features=false }
+subxt          ={ package="subxt", git="https://github.com/paritytech/subxt.git", tag="v0.32.1", default-features=false }
 subxt-signer   ={ version="0.31.0", features=[] }
 tokio          ={ version="1.16", features=["sync", "rt", "macros"] }
 x25519-dalek   ="2.0.0-pre.1"
 futures        ="0.3"
 hex            ="*"
 blake2         ="0.10.4"
-thiserror      ="1.0.40"
+thiserror      ="1.0.48"
 snow           ="0.9.2"
 getrandom      ={ version="0.2", features=["js"] }
 rand_core      ={ version="0.6.4", features=["getrandom"] }

crypto/protocol/Cargo.toml

@@ -30,7 +30,7 @@ reqwest    ={ version="0.11", features=["json", "stream"] }
 axum       ={ version="0.6.18", features=["ws"] }
 axum-macros="0.3.7"
 # Substrate
-subxt             ={ package="subxt", git="https://github.com/paritytech/subxt.git", tag="v0.29.0" }
+subxt             ={ package="subxt", git="https://github.com/paritytech/subxt.git", tag="v0.32.1" }
 parity-scale-codec="3.4.0"
 sp-core           ={ version="21.0.0", default-features=false }
 

crypto/server/Cargo.toml

@@ -1,6 +1,9 @@
 #![allow(clippy::all)]
-pub use subxt::config::PolkadotConfig as EntropyConfig;
-use subxt::OnlineClient;
+pub use subxt::PolkadotConfig as EntropyConfig;
+use subxt::{
+    backend::{legacy::LegacyRpcMethods, rpc::RpcClient},
+    OnlineClient,
+};
 
 #[subxt::subxt(
     runtime_metadata_path = "entropy_metadata.scale",
@@ -21,3 +24,11 @@ pub async fn get_api(url: &str) -> Result<OnlineClient<EntropyConfig>, subxt::Er
     let api = OnlineClient::<EntropyConfig>::from_url(url).await?;
     Ok(api)
 }
+
+/// Creates a rpc instance to talk to chain
+/// Chain endpoint set on launch
+pub async fn get_rpc(url: &str) -> Result<LegacyRpcMethods<EntropyConfig>, subxt::Error> {
+    let rpc_client = RpcClient::from_url(url).await?;
+    let rpc_methods = LegacyRpcMethods::<EntropyConfig>::new(rpc_client);
+    Ok(rpc_methods)
+}

crypto/server/src/chain_api.rs

@@ -1,5 +1,8 @@
 use entropy_shared::{KeyVisibility, SIGNING_PARTY_SIZE};
-use subxt::{ext::sp_core::sr25519, tx::PairSigner, utils::AccountId32, Config, OnlineClient};
+use subxt::{
+    backend::legacy::LegacyRpcMethods, ext::sp_core::sr25519, tx::PairSigner, utils::AccountId32,
+    Config, OnlineClient,
+};
 
 use crate::{
     chain_api::{entropy, EntropyConfig},
@@ -9,16 +12,21 @@ use crate::{
 /// gets the subgroup of the working validator
 pub async fn get_subgroup(
     api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     signer: &PairSigner<EntropyConfig, sr25519::Pair>,
 ) -> Result<(Option<u8>, AccountId32), UserErr> {
     let mut subgroup: Option<u8> = None;
     let threshold_address = signer.account_id();
     let stash_address_query =
         entropy::storage().staking_extension().threshold_to_stash(threshold_address);
+    let block_hash = rpc
+        .chain_get_block_hash(None)
+        .await?
+        .ok_or_else(|| UserErr::OptionUnwrapError("Errir getting block hash"))?;
+
     let stash_address = api
         .storage()
-        .at_latest()
-        .await?
+        .at(block_hash)
         .fetch(&stash_address_query)
         .await?
         .ok_or_else(|| UserErr::SubgroupError("Stash Fetch Error"))?;
@@ -27,8 +35,7 @@ pub async fn get_subgroup(
             entropy::storage().staking_extension().signing_groups(i as u8);
         let signing_group_addresses = api
             .storage()
-            .at_latest()
-            .await?
+            .at(block_hash)
             .fetch(&signing_group_addresses_query)
             .await?
             .ok_or_else(|| UserErr::SubgroupError("Subgroup Error"))?;
@@ -43,13 +50,17 @@ pub async fn get_subgroup(
 /// Returns all the addresses of a specific subgroup
 pub async fn return_all_addresses_of_subgroup(
     api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     subgroup: u8,
 ) -> Result<Vec<AccountId32>, UserErr> {
     let subgroup_addresses_query = entropy::storage().staking_extension().signing_groups(subgroup);
+    let block_hash = rpc
+        .chain_get_block_hash(None)
+        .await?
+        .ok_or_else(|| UserErr::OptionUnwrapError("Errir getting block hash"))?;
     let subgroup_addresses = api
         .storage()
-        .at_latest()
-        .await?
+        .at(block_hash)
         .fetch(&subgroup_addresses_query)
         .await?
         .ok_or_else(|| UserErr::SubgroupError("Subgroup Error"))?;
@@ -59,14 +70,18 @@ pub async fn return_all_addresses_of_subgroup(
 /// Queries the user's program from the chain
 pub async fn get_program(
     substrate_api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     sig_req_account: &<EntropyConfig as Config>::AccountId,
 ) -> Result<Vec<u8>, UserErr> {
     let bytecode_address = entropy::storage().constraints().bytecode(sig_req_account);
+    let block_hash = rpc
+        .chain_get_block_hash(None)
+        .await?
+        .ok_or_else(|| UserErr::OptionUnwrapError("Errir getting block hash"))?;
 
     substrate_api
         .storage()
-        .at_latest()
-        .await?
+        .at(block_hash)
         .fetch(&bytecode_address)
         .await?
         .ok_or(UserErr::NoProgramDefined)
@@ -77,6 +92,7 @@ pub async fn get_program(
 #[cfg(test)]
 pub async fn make_register(
     api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     sig_req_keyring: sr25519::Pair,
     constraint_account: &AccountId32,
     key_visibility: KeyVisibility,
@@ -86,8 +102,8 @@ pub async fn make_register(
     let sig_req_account = PairSigner::<EntropyConfig, sr25519::Pair>::new(sig_req_keyring);
 
     let registering_query = entropy::storage().relayer().registering(sig_req_account.account_id());
-    let is_registering_1 =
-        api.storage().at_latest().await.unwrap().fetch(&registering_query).await.unwrap();
+    let block_hash = rpc.chain_get_block_hash(None).await.unwrap().unwrap();
+    let is_registering_1 = api.storage().at(block_hash).fetch(&registering_query).await.unwrap();
     assert!(is_registering_1.is_none());
 
     // register the user
@@ -109,21 +125,26 @@ pub async fn make_register(
         .await
         .unwrap();
 
-    let query_registering_status =
-        api.storage().at_latest().await.unwrap().fetch(&registering_query).await;
+    let block_hash_2 = rpc.chain_get_block_hash(None).await.unwrap().unwrap();
+
+    let query_registering_status = api.storage().at(block_hash_2).fetch(&registering_query).await;
     assert!(query_registering_status.unwrap().unwrap().is_registering);
 }
 
 /// Returns wether an account is registered
 pub async fn get_key_visibility(
     api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     who: &<EntropyConfig as Config>::AccountId,
 ) -> Result<KeyVisibility, UserErr> {
     let registered_info_query = entropy::storage().relayer().registered(who);
+    let block_hash = rpc
+        .chain_get_block_hash(None)
+        .await?
+        .ok_or_else(|| UserErr::OptionUnwrapError("Errir getting block hash"))?;
     let result = api
         .storage()
-        .at_latest()
-        .await?
+        .at(block_hash)
         .fetch(&registered_info_query)
         .await?
         .ok_or_else(|| UserErr::NotRegistering("Register Onchain first"))?;

crypto/server/src/helpers/substrate.rs

@@ -11,6 +11,7 @@ use rand_core::OsRng;
 use serial_test::serial;
 use sp_keyring::AccountKeyring;
 use subxt::{
+    backend::legacy::LegacyRpcMethods,
     ext::sp_core::{sr25519, Pair},
     tx::PairSigner,
     utils::{AccountId32 as SubxtAccountId32, Static},
@@ -21,7 +22,7 @@ use testing_utils::substrate_context::testing_context;
 
 use crate::{
     app,
-    chain_api::{entropy, get_api, EntropyConfig},
+    chain_api::{entropy, get_api, get_rpc, EntropyConfig},
     get_signer,
     helpers::{
         launch::{
@@ -172,15 +173,20 @@ pub async fn update_programs(
 }
 
 /// Verify that a Registering account has all confirmation, and that it is registered.
-pub async fn check_if_confirmation(api: &OnlineClient<EntropyConfig>, key: &sr25519::Pair) {
+pub async fn check_if_confirmation(
+    api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
+    key: &sr25519::Pair,
+) {
     let signer = PairSigner::<EntropyConfig, sr25519::Pair>::new(key.clone());
     let registering_query = entropy::storage().relayer().registering(signer.account_id());
     let registered_query = entropy::storage().relayer().registered(signer.account_id());
-    let is_registering = api.storage().at_latest().await.unwrap().fetch(&registering_query).await;
+    let block_hash = rpc.chain_get_block_hash(None).await.unwrap().unwrap();
+    let is_registering = api.storage().at(block_hash.clone()).fetch(&registering_query).await;
     // cleared from is_registering state
     assert!(is_registering.unwrap().is_none());
     let is_registered =
-        api.storage().at_latest().await.unwrap().fetch(&registered_query).await.unwrap();
+        api.storage().at(block_hash.clone()).fetch(&registered_query).await.unwrap();
     assert_eq!(is_registered.as_ref().unwrap().verifying_key.0.len(), 33usize);
     assert_eq!(is_registered.unwrap().key_visibility, Static(KeyVisibility::Public));
 }
@@ -192,19 +198,21 @@ async fn test_get_signing_group() {
     let cxt = testing_context().await;
     setup_client().await;
     let api = get_api(&cxt.node_proc.ws_url).await.unwrap();
+    let rpc = get_rpc(&cxt.node_proc.ws_url).await.unwrap();
+
     let p_alice = <sr25519::Pair as Pair>::from_string(DEFAULT_MNEMONIC, None).unwrap();
     let signer_alice = PairSigner::<EntropyConfig, sr25519::Pair>::new(p_alice);
-    let result_alice = get_subgroup(&api, &signer_alice).await.unwrap().0;
+    let result_alice = get_subgroup(&api, &rpc, &signer_alice).await.unwrap().0;
     assert_eq!(result_alice, Some(0));
 
     let p_bob = <sr25519::Pair as Pair>::from_string(DEFAULT_BOB_MNEMONIC, None).unwrap();
     let signer_bob = PairSigner::<EntropyConfig, sr25519::Pair>::new(p_bob);
-    let result_bob = get_subgroup(&api, &signer_bob).await.unwrap().0;
+    let result_bob = get_subgroup(&api, &rpc, &signer_bob).await.unwrap().0;
     assert_eq!(result_bob, Some(1));
 
     let p_charlie = <sr25519::Pair as Pair>::from_string("//Charlie//stash", None).unwrap();
     let signer_charlie = PairSigner::<EntropyConfig, sr25519::Pair>::new(p_charlie);
-    let result_charlie = get_subgroup(&api, &signer_charlie).await;
+    let result_charlie = get_subgroup(&api, &rpc, &signer_charlie).await;
     assert!(result_charlie.is_err());
 
     clean_tests();

crypto/server/src/helpers/tests.rs

@@ -8,6 +8,7 @@ use entropy_shared::{KeyVisibility, SETUP_TIMEOUT_SECONDS};
 use parity_scale_codec::Encode;
 use sp_core::crypto::AccountId32;
 use subxt::{
+    backend::legacy::LegacyRpcMethods,
     ext::sp_core::{sr25519, Bytes},
     tx::PairSigner,
     utils::AccountId32 as SubxtAccountId32,
@@ -97,6 +98,7 @@ pub async fn do_dkg(
 /// Send's user key share to other members of signing subgroup
 pub async fn send_key(
     api: &OnlineClient<EntropyConfig>,
+    rpc: &LegacyRpcMethods<EntropyConfig>,
     stash_address: &SubxtAccountId32,
     addresses_in_subgroup: &mut Vec<SubxtAccountId32>,
     user_registration_info: UserRegistrationInfo,
@@ -108,12 +110,16 @@ pub async fn send_key(
             .position(|address| *address == *stash_address)
             .ok_or_else(|| UserErr::OptionUnwrapError("Validator not in subgroup"))?,
     );
+    let block_hash = rpc
+        .chain_get_block_hash(None)
+        .await?
+        .ok_or_else(|| UserErr::OptionUnwrapError("Errir getting block hash"))?;
+
     for validator in addresses_in_subgroup {
         let server_info_query = entropy::storage().staking_extension().threshold_servers(validator);
         let server_info = api
             .storage()
-            .at_latest()
-            .await?
+            .at(block_hash)
             .fetch(&server_info_query)
             .await?
             .ok_or_else(|| UserErr::OptionUnwrapError("Server Info Fetch Error"))?;

crypto/server/src/helpers/user.rs

@@ -115,7 +115,7 @@ use tracing::Level;
 use validator::api::get_random_server_info;
 
 use self::{
-    chain_api::get_api,
+    chain_api::{get_api, get_rpc},
     signing_client::{api::*, ListenerState},
     user::api::*,
 };
@@ -164,11 +164,12 @@ async fn main() {
     // Below deals with syncing the kvdb
     if args.sync {
         let api = get_api(&configuration.endpoint).await.expect("Issue acquiring chain API");
+        let rpc = get_rpc(&configuration.endpoint).await.expect("Issue acquiring chain RPC");
         let mut is_syncing = true;
         let sleep_time = Duration::from_secs(20);
         // wait for chain to be fully synced before starting key swap
         while is_syncing {
-            let health = api.rpc().system_health().await.expect("Issue checking chain health");
+            let health = rpc.system_health().await.expect("Issue checking chain health");
             is_syncing = health.is_syncing;
             if is_syncing {
                 println!("chain syncing, retrying {is_syncing:?}");
@@ -177,30 +178,34 @@ async fn main() {
         }
         // TODO: find a proper batch size
         let batch_size = 10;
-        let key_amount = 10_000;
         let signer = get_signer(&kv_store).await.expect("Issue acquiring threshold signer key");
-        let has_fee_balance = check_balance_for_fees(&api, signer.account_id(), MIN_BALANCE)
+        let has_fee_balance = check_balance_for_fees(&api, &rpc, signer.account_id(), MIN_BALANCE)
             .await
             .expect("Issue checking chain for signer balance");
         if !has_fee_balance {
             panic!("threshold account needs balance: {:?}", signer.account_id());
         }
         // if not in subgroup retry until you are
-        let mut my_subgroup = get_subgroup(&api, &signer).await;
+        let mut my_subgroup = get_subgroup(&api, &rpc, &signer).await;
         while my_subgroup.is_err() {
             println!("you are not currently a validator, retrying");
             thread::sleep(sleep_time);
-            my_subgroup = Ok(get_subgroup(&api, &signer).await.expect("Failed to get subgroup."));
+            my_subgroup =
+                Ok(get_subgroup(&api, &rpc, &signer).await.expect("Failed to get subgroup."));
         }
         let (sbgrp, validator_stash) = my_subgroup.expect("Failed to get subgroup.");
-        let key_server_info =
-            get_random_server_info(&api, sbgrp.expect("failed to get subgroup"), validator_stash)
-                .await
-                .expect("Issue getting registered keys from chain.");
+        let key_server_info = get_random_server_info(
+            &api,
+            &rpc,
+            sbgrp.expect("failed to get subgroup"),
+            validator_stash,
+        )
+        .await
+        .expect("Issue getting registered keys from chain.");
         let ip_address =
             String::from_utf8(key_server_info.endpoint).expect("failed to parse IP address.");
         let recip_key = x25519_dalek::PublicKey::from(key_server_info.x25519_public_key);
-        let all_keys = get_all_keys(&api, key_amount).await.expect("failed to get all keys.");
+        let all_keys = get_all_keys(&api, &rpc).await.expect("failed to get all keys.");
         let _ = get_and_store_values(
             all_keys, &kv_store, ip_address, batch_size, args.dev, &recip_key, &signer,
         )