From 13bb49e35068279e4d15843062ae7dfc6601cc25 Mon Sep 17 00:00:00 2001 From: Zhe Wu Date: Mon, 12 Feb 2024 21:23:38 -0800 Subject: [PATCH] Server side authority overload pushback (#16096) ## Description This PR implements the server-side logic of authority overload pushback. - When txn rejected during signing, objects are still locked to prevent equivocation. - Create a new error type, ValidatorPushbackAndRetry, to indicate that the client should keep retrying the same request. ## Test Plan Unit tests added for txn selection logic Integration tests added for testing server side logic. --- If your changes are not user-facing and do not break anything, you can skip the following section. Otherwise, please briefly describe what has changed under the Release Notes section. ### Type of Change (Check all that apply) - [ ] protocol change - [ ] user-visible impact - [ ] breaking change for a client SDKs - [ ] breaking change for FNs (FN binary must upgrade) - [ ] breaking change for validators or node operators (must upgrade binaries) - [ ] breaking change for on-chain data layout - [ ] necessitate either a data wipe or data migration ### Release notes --- Cargo.lock | 2 + Cargo.toml | 1 + crates/sui-config/src/node.rs | 12 + crates/sui-core/Cargo.toml | 1 + crates/sui-core/src/authority.rs | 38 ++- crates/sui-core/src/authority_server.rs | 49 ++-- crates/sui-core/src/overload_monitor.rs | 97 ++++++++ .../src/unit_tests/execution_driver_tests.rs | 230 ++++++++++++++++++ .../src/single_node.rs | 6 +- crates/sui-types/src/error.rs | 4 + crates/workspace-hack/Cargo.toml | 4 +- 11 files changed, 423 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f145d2420c5a2..7f1f800c5722e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -12055,6 +12055,7 @@ dependencies = [ "tokio-retry", "tokio-stream", "tracing", + "twox-hash", "typed-store", "typed-store-derive", "workspace-hack", @@ -15063,6 +15064,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ "cfg-if", + "rand 0.7.3", "static_assertions", ] diff --git a/Cargo.toml b/Cargo.toml index fe94511eba1ce..2ab3799bbbfba 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -490,6 +490,7 @@ tower-http = { version = "0.3.4", features = [ ] } # tower-http = { version="0.4", features = ["trace"] } tower-layer = "0.3.2" +twox-hash = "1.6.3" tracing = "0.1.37" tracing-appender = "0.2.2" tracing-subscriber = { version = "0.3.15", default-features = false, features = [ diff --git a/crates/sui-config/src/node.rs b/crates/sui-config/src/node.rs index f20a433b378dc..39326bb4afd76 100644 --- a/crates/sui-config/src/node.rs +++ b/crates/sui-config/src/node.rs @@ -715,6 +715,16 @@ pub struct OverloadThresholdConfig { // is well under used, and will not enter load shedding mode. #[serde(default = "default_safe_transaction_ready_rate")] pub safe_transaction_ready_rate: u32, + + // When set to true, transaction signing may be rejected when the validator + // is overloaded. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub check_system_overload_at_signing: bool, + + // When set to true, transaction execution may be rejected when the validator + // is overloaded. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub check_system_overload_at_execution: bool, // TODO: Move other thresholds here as well, including `MAX_TM_QUEUE_LENGTH` // and `MAX_PER_OBJECT_QUEUE_LENGTH`. } @@ -758,6 +768,8 @@ impl Default for OverloadThresholdConfig { min_load_shedding_percentage_above_hard_limit: default_min_load_shedding_percentage_above_hard_limit(), safe_transaction_ready_rate: default_safe_transaction_ready_rate(), + check_system_overload_at_signing: false, + check_system_overload_at_execution: false, } } } diff --git a/crates/sui-core/Cargo.toml b/crates/sui-core/Cargo.toml index a4d1151efb2a6..7f362b44b3923 100644 --- a/crates/sui-core/Cargo.toml +++ b/crates/sui-core/Cargo.toml @@ -45,6 +45,7 @@ tokio = { workspace = true, features = ["full", "tracing", "test-util"] } tokio-retry.workspace = true tokio-stream.workspace = true tracing.workspace = true +twox-hash.workspace = true fastcrypto.workspace = true fastcrypto-tbls.workspace = true diff --git a/crates/sui-core/src/authority.rs b/crates/sui-core/src/authority.rs index cbc66e98500dc..12bd09db437a7 100644 --- a/crates/sui-core/src/authority.rs +++ b/crates/sui-core/src/authority.rs @@ -30,6 +30,7 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs::File; use std::io::Write; use std::path::{Path, PathBuf}; +use std::sync::atomic::Ordering; use std::time::Duration; use std::{ collections::{HashMap, HashSet}, @@ -144,7 +145,9 @@ use crate::execution_driver::execution_process; use crate::metrics::LatencyObserver; use crate::metrics::RateTracker; use crate::module_cache_metrics::ResolverMetrics; -use crate::overload_monitor::{overload_monitor, AuthorityOverloadInfo}; +use crate::overload_monitor::{ + overload_monitor, overload_monitor_accept_tx, AuthorityOverloadInfo, +}; use crate::stake_aggregator::StakeAggregator; use crate::state_accumulator::{AccumulatorStore, StateAccumulator, WrappedObject}; use crate::subscription_handler::SubscriptionHandler; @@ -931,17 +934,43 @@ impl AuthorityState { } } + pub fn check_system_overload_at_signing(&self) -> bool { + self.overload_threshold_config + .check_system_overload_at_signing + } + + pub fn check_system_overload_at_execution(&self) -> bool { + self.overload_threshold_config + .check_system_overload_at_execution + } + pub(crate) fn check_system_overload( &self, consensus_adapter: &Arc, tx_data: &SenderSignedData, + do_authority_overload_check: bool, ) -> SuiResult { + if do_authority_overload_check { + self.check_authority_overload(tx_data)?; + } self.transaction_manager .check_execution_overload(self.max_txn_age_in_queue(), tx_data)?; consensus_adapter.check_consensus_overload()?; Ok(()) } + fn check_authority_overload(&self, tx_data: &SenderSignedData) -> SuiResult { + if !self.overload_info.is_overload.load(Ordering::Relaxed) { + return Ok(()); + } + + let load_shedding_percentage = self + .overload_info + .load_shedding_percentage + .load(Ordering::Relaxed); + overload_monitor_accept_tx(load_shedding_percentage, tx_data.digest()) + } + /// Executes a transaction that's known to have correct effects. /// For such transaction, we don't have to wait for consensus to set shared object /// locks because we already know the shared object versions based on the effects. @@ -2530,8 +2559,11 @@ impl AuthorityState { rx_execution_shutdown, )); - let authority_state = Arc::downgrade(&state); - spawn_monitored_task!(overload_monitor(authority_state, overload_threshold_config)); + // Don't start the overload monitor when max_load_shedding_percentage is 0. + if overload_threshold_config.max_load_shedding_percentage > 0 { + let authority_state = Arc::downgrade(&state); + spawn_monitored_task!(overload_monitor(authority_state, overload_threshold_config)); + } // TODO: This doesn't belong to the constructor of AuthorityState. state diff --git a/crates/sui-core/src/authority_server.rs b/crates/sui-core/src/authority_server.rs index 2ed3041f02d43..3cfe734c59659 100644 --- a/crates/sui-core/src/authority_server.rs +++ b/crates/sui-core/src/authority_server.rs @@ -263,21 +263,15 @@ impl ValidatorService { pub async fn execute_certificate_for_testing( &self, cert: CertifiedTransaction, - ) -> HandleCertificateResponseV2 { - self.handle_certificate_v2(tonic::Request::new(cert)) - .await - .unwrap() - .into_inner() + ) -> Result, tonic::Status> { + self.handle_certificate_v2(tonic::Request::new(cert)).await } pub async fn handle_transaction_for_testing( &self, transaction: Transaction, - ) -> HandleTransactionResponse { - self.transaction(tonic::Request::new(transaction)) - .await - .unwrap() - .into_inner() + ) -> Result, tonic::Status> { + self.transaction(tonic::Request::new(transaction)).await } async fn handle_transaction( @@ -312,14 +306,31 @@ impl ValidatorService { .into()); } - let overload_check_res = - state.check_system_overload(&consensus_adapter, transaction.data()); + // When authority is overloaded and decide to reject this tx, we still lock the object + // and ask the client to retry in the future. This is because without locking, the + // input objects can be locked by a different tx in the future, however, the input objects + // may already be locked by this tx in other validators. This can cause non of the txes + // to have enough quorum to form a certificate, causing the objects to be locked for + // the entire epoch. By doing locking but pushback, retrying transaction will have + // higher chance to succeed. + let mut validator_pushback_error = None; + let overload_check_res = state.check_system_overload( + &consensus_adapter, + transaction.data(), + state.check_system_overload_at_signing(), + ); if let Err(error) = overload_check_res { metrics .num_rejected_tx_during_overload .with_label_values(&[error.as_ref()]) .inc(); - return Err(error.into()); + // TODO: consider change the behavior for other types of overload errors. + match error { + SuiError::ValidatorOverloadedRetryAfter { .. } => { + validator_pushback_error = Some(error) + } + _ => return Err(error.into()), + } } let _handle_tx_metrics_guard = metrics.handle_transaction_latency.start_timer(); @@ -345,6 +356,11 @@ impl ValidatorService { } })?; + if let Some(error) = validator_pushback_error { + // TODO: right now, we still sign the txn, but just don't return it. We can also skip signing + // to save more CPU. + return Err(error.into()); + } Ok(tonic::Response::new(info)) } @@ -414,8 +430,11 @@ impl ValidatorService { // 2) Verify the cert. // Check system overload - let overload_check_res = - state.check_system_overload(&consensus_adapter, certificate.data()); + let overload_check_res = state.check_system_overload( + &consensus_adapter, + certificate.data(), + state.check_system_overload_at_execution(), + ); if let Err(error) = overload_check_res { metrics .num_rejected_cert_during_overload diff --git a/crates/sui-core/src/overload_monitor.rs b/crates/sui-core/src/overload_monitor.rs index b0d0c486e564c..c96274f726f1a 100644 --- a/crates/sui-core/src/overload_monitor.rs +++ b/crates/sui-core/src/overload_monitor.rs @@ -3,12 +3,19 @@ use crate::authority::AuthorityState; use std::cmp::{max, min}; +use std::hash::Hasher; use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Weak; use std::time::Duration; +use std::time::{SystemTime, UNIX_EPOCH}; use sui_config::node::OverloadThresholdConfig; +use sui_types::digests::TransactionDigest; +use sui_types::error::SuiError; +use sui_types::error::SuiResult; +use sui_types::fp_bail; use tokio::time::sleep; use tracing::{debug, info}; +use twox_hash::XxHash64; #[derive(Default)] pub struct AuthorityOverloadInfo { @@ -204,6 +211,40 @@ fn check_overload_signals( (overload_status, load_shedding_percentage) } +// Return true if we should reject the txn with `tx_digest`. +fn should_reject_tx( + load_shedding_percentage: u32, + tx_digest: TransactionDigest, + minutes_since_epoch: u64, +) -> bool { + // TODO: we also need to add a secret salt (e.g. first consensus commit in the current epoch), + // to prevent gaming the system. + let mut hasher = XxHash64::with_seed(minutes_since_epoch); + hasher.write(tx_digest.inner()); + let value = hasher.finish(); + value % 100 < load_shedding_percentage as u64 +} + +// Checks if we can accept the transaction with `tx_digest`. +pub fn overload_monitor_accept_tx( + load_shedding_percentage: u32, + tx_digest: TransactionDigest, +) -> SuiResult { + // Using the minutes_since_epoch as the hash seed to allow rejected transaction's + // retry to have a chance to go through in the future. + let minutes_since_epoch = SystemTime::now() + .duration_since(UNIX_EPOCH) + .expect("Sui did not exist prior to 1970") + .as_secs() + / 60; + + if should_reject_tx(load_shedding_percentage, tx_digest, minutes_since_epoch) { + // TODO: complete the suggestion for client retry deadline. + fp_bail!(SuiError::ValidatorOverloadedRetryAfter { retry_after_sec: 0 }); + } + Ok(()) +} + #[cfg(test)] #[allow(clippy::disallowed_methods)] // allow unbounded_channel() since tests are simulating txn manager execution driver interaction. mod tests { @@ -215,6 +256,7 @@ mod tests { Rng, SeedableRng, }; use std::sync::Arc; + use sui_macros::sim_test; use tokio::sync::mpsc::unbounded_channel; use tokio::sync::mpsc::UnboundedReceiver; use tokio::sync::mpsc::UnboundedSender; @@ -645,4 +687,59 @@ mod tests { assert!(0.4 < dropped_ratio); assert!(dropped_ratio < 0.6); } + + // Tests that the ratio of rejected transactions created randomly matches load shedding percentage in + // the overload monitor. + #[test] + fn test_txn_rejection_rate() { + for rejection_percentage in 0..=100 { + let mut reject_count = 0; + for _ in 0..10000 { + let digest = TransactionDigest::random(); + if should_reject_tx(rejection_percentage, digest, 28455473) { + reject_count += 1; + } + } + + // Give it a 2% fluctuation. + assert!(rejection_percentage as f32 / 100.0 - 0.02 < reject_count as f32 / 10000.0); + assert!(reject_count as f32 / 10000.0 < rejection_percentage as f32 / 100.0 + 0.02); + } + } + + // Tests that rejected transaction will have a chance to be accepted in the future. + #[sim_test] + async fn test_txn_rejection_over_time() { + let start_time = Instant::now(); + let mut digest = TransactionDigest::random(); + let mut minutes_since_epoch = 28455473; + let load_shedding_percentage = 50; + + // Find a rejected transaction with 50% rejection rate. + while !should_reject_tx(load_shedding_percentage, digest, minutes_since_epoch) + && start_time.elapsed() < Duration::from_secs(30) + { + digest = TransactionDigest::random(); + } + + // It should always be rejected in the current minute. + for _ in 0..100 { + assert!(should_reject_tx( + load_shedding_percentage, + digest, + minutes_since_epoch + )); + } + + // It will be accepted in the future. + minutes_since_epoch += 1; + while should_reject_tx(load_shedding_percentage, digest, minutes_since_epoch) + && start_time.elapsed() < Duration::from_secs(30) + { + minutes_since_epoch += 1; + } + + // Make sure that the tests can finish within 30 seconds. + assert!(start_time.elapsed() < Duration::from_secs(30)); + } } diff --git a/crates/sui-core/src/unit_tests/execution_driver_tests.rs b/crates/sui-core/src/unit_tests/execution_driver_tests.rs index f5a00454ef416..435583689f9b9 100644 --- a/crates/sui-core/src/unit_tests/execution_driver_tests.rs +++ b/crates/sui-core/src/unit_tests/execution_driver_tests.rs @@ -2,18 +2,27 @@ // SPDX-License-Identifier: Apache-2.0 use crate::authority::authority_tests::{send_consensus, send_consensus_no_execution}; +use crate::authority::test_authority_builder::TestAuthorityBuilder; use crate::authority::AuthorityState; use crate::authority::EffectsNotifyRead; use crate::authority_aggregator::authority_aggregator_tests::{ create_object_move_transaction, do_cert, do_transaction, extract_cert, get_latest_ref, }; +use crate::authority_server::ValidatorService; +use crate::authority_server::ValidatorServiceMetrics; +use crate::consensus_adapter::ConnectionMonitorStatusForTests; +use crate::consensus_adapter::ConsensusAdapter; +use crate::consensus_adapter::ConsensusAdapterMetrics; +use crate::consensus_adapter::MockSubmitToConsensus; use crate::safe_client::SafeClient; use crate::test_authority_clients::LocalAuthorityClient; +use crate::test_utils::make_transfer_object_transaction; use crate::test_utils::{ init_local_authorities, init_local_authorities_with_overload_thresholds, make_transfer_object_move_transaction, }; use crate::transaction_manager::MAX_PER_OBJECT_QUEUE_LENGTH; +use sui_types::error::SuiError; use std::collections::BTreeSet; use std::sync::Arc; @@ -28,6 +37,7 @@ use sui_types::crypto::{get_key_pair, AccountKeyPair}; use sui_types::effects::{TransactionEffects, TransactionEffectsAPI}; use sui_types::error::SuiResult; use sui_types::object::{Object, Owner}; +use sui_types::transaction::CertifiedTransaction; use sui_types::transaction::{ Transaction, VerifiedCertificate, TEST_ONLY_GAS_UNIT_FOR_HEAVY_COMPUTATION_STORAGE, }; @@ -701,3 +711,223 @@ async fn test_txn_age_overload() { message ); } + +// Tests that when validator is in load shedding mode, it can pushback txn signing correctly. +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn test_authority_txn_signing_pushback() { + telemetry_subscribers::init_for_testing(); + + // Create one sender, two recipients addresses, and 2 gas objects. + let (sender, sender_key): (_, AccountKeyPair) = get_key_pair(); + let (recipient1, _): (_, AccountKeyPair) = get_key_pair(); + let (recipient2, _): (_, AccountKeyPair) = get_key_pair(); + let gas_object1 = Object::with_owner_for_testing(sender); + let gas_object2 = Object::with_owner_for_testing(sender); + + // Initialize an AuthorityState. Disable overload monitor by setting max_load_shedding_percentage to 0; + // Set check_system_overload_at_signing to true. + let overload_config = OverloadThresholdConfig { + check_system_overload_at_signing: true, + max_load_shedding_percentage: 0, + ..Default::default() + }; + let authority_state = TestAuthorityBuilder::new() + .with_overload_threshold_config(overload_config) + .build() + .await; + authority_state + .insert_genesis_objects(&[gas_object1.clone(), gas_object2.clone()]) + .await; + + // Create a validator service around the `authority_state`. + let epoch_store = authority_state.epoch_store_for_testing(); + let consensus_adapter = Arc::new(ConsensusAdapter::new( + Arc::new(MockSubmitToConsensus::new()), + authority_state.name, + Arc::new(ConnectionMonitorStatusForTests {}), + 100_000, + 100_000, + None, + None, + ConsensusAdapterMetrics::new_test(), + epoch_store.protocol_config().clone(), + )); + let validator_service = Arc::new(ValidatorService::new( + authority_state.clone(), + consensus_adapter, + Arc::new(ValidatorServiceMetrics::new_for_tests()), + )); + + // Manually make the authority into overload state and reject 100% of traffic. + authority_state.overload_info.set_overload(100); + + // First, create a transaction to tranfer `gas_object1` to `recipient1`. + let rgp = authority_state.reference_gas_price_for_testing().unwrap(); + let tx = make_transfer_object_transaction( + gas_object1.compute_object_reference(), + gas_object2.compute_object_reference(), + sender, + &sender_key, + recipient1, + rgp, + ); + + // Txn shouldn't get signed with ValidatorOverloadedRetryAfter error. + let response = validator_service + .handle_transaction_for_testing(tx.clone()) + .await; + assert!(matches!( + SuiError::from(response.err().unwrap()), + SuiError::ValidatorOverloadedRetryAfter { .. } + )); + + // Check that the input object should be locked by the above transaction. + let lock_tx = authority_state + .get_transaction_lock(&gas_object1.compute_object_reference(), &epoch_store) + .await + .unwrap() + .unwrap(); + assert_eq!(tx.digest(), lock_tx.digest()); + + // Send the same txn again. Although objects are locked, since authority is in load shedding mode, + // it should still pushback the transaction. + assert!(matches!( + validator_service + .handle_transaction_for_testing(tx.clone()) + .await + .err() + .unwrap() + .into(), + SuiError::ValidatorOverloadedRetryAfter { .. } + )); + + // Send another transaction, that send the same object to a different recipient. + // Transaction signing should failed with ObjectLockConflict error, since the object + // is already locked by the previous transaction. + let tx2 = make_transfer_object_transaction( + gas_object1.compute_object_reference(), + gas_object2.compute_object_reference(), + sender, + &sender_key, + recipient2, + rgp, + ); + assert!(matches!( + validator_service + .handle_transaction_for_testing(tx2) + .await + .err() + .unwrap() + .into(), + SuiError::ObjectLockConflict { .. } + )); + + // Clear the authority overload status. + authority_state.overload_info.clear_overload(); + + // Re-send the first transaction, now the transaction can be successfully signed. + let response = validator_service + .handle_transaction_for_testing(tx.clone()) + .await; + assert!(response.is_ok()); + assert_eq!( + &response + .unwrap() + .into_inner() + .status + .into_signed_for_testing(), + lock_tx.auth_sig() + ); +} + +// Tests that when validator is in load shedding mode, it can pushback txn execution correctly. +#[tokio::test(flavor = "current_thread", start_paused = true)] +async fn test_authority_txn_execution_pushback() { + telemetry_subscribers::init_for_testing(); + + // Create one sender, one recipient addresses, and 2 gas objects. + let (sender, sender_key): (_, AccountKeyPair) = get_key_pair(); + let (recipient, _): (_, AccountKeyPair) = get_key_pair(); + let gas_object1 = Object::with_owner_for_testing(sender); + let gas_object2 = Object::with_owner_for_testing(sender); + + // Initialize an AuthorityState. Disable overload monitor by setting max_load_shedding_percentage to 0; + // Set check_system_overload_at_execution to true. + let overload_config = OverloadThresholdConfig { + check_system_overload_at_execution: true, + max_load_shedding_percentage: 0, + ..Default::default() + }; + let authority_state = TestAuthorityBuilder::new() + .with_overload_threshold_config(overload_config) + .build() + .await; + authority_state + .insert_genesis_objects(&[gas_object1.clone(), gas_object2.clone()]) + .await; + + // Create a validator service around the `authority_state`. + let epoch_store = authority_state.epoch_store_for_testing(); + let consensus_adapter = Arc::new(ConsensusAdapter::new( + Arc::new(MockSubmitToConsensus::new()), + authority_state.name, + Arc::new(ConnectionMonitorStatusForTests {}), + 100_000, + 100_000, + None, + None, + ConsensusAdapterMetrics::new_test(), + epoch_store.protocol_config().clone(), + )); + let validator_service = Arc::new(ValidatorService::new( + authority_state.clone(), + consensus_adapter, + Arc::new(ValidatorServiceMetrics::new_for_tests()), + )); + + // Manually make the authority into overload state and reject 100% of traffic. + authority_state.overload_info.set_overload(100); + + // Create a transaction to tranfer `gas_object1` to `recipient`. + let rgp = authority_state.reference_gas_price_for_testing().unwrap(); + let tx = make_transfer_object_transaction( + gas_object1.compute_object_reference(), + gas_object2.compute_object_reference(), + sender, + &sender_key, + recipient, + rgp, + ); + + // Ask validator to sign the transaction and then create a certificate. + let response = validator_service + .handle_transaction_for_testing(tx.clone()) + .await + .unwrap() + .into_inner(); + let committee = authority_state.clone_committee_for_testing(); + let cert = CertifiedTransaction::new( + tx.into_data(), + vec![response.status.into_signed_for_testing()], + &committee, + ) + .unwrap(); + + // Ask the validator to execute the certificate, it should fail with ValidatorOverloadedRetryAfter error. + assert!(matches!( + validator_service + .execute_certificate_for_testing(cert.clone()) + .await + .err() + .unwrap() + .into(), + SuiError::ValidatorOverloadedRetryAfter { .. } + )); + + // Clear the validator overload status and retry the certificate. It should succeed. + authority_state.overload_info.clear_overload(); + assert!(validator_service + .execute_certificate_for_testing(cert) + .await + .is_ok()); +} diff --git a/crates/sui-single-node-benchmark/src/single_node.rs b/crates/sui-single-node-benchmark/src/single_node.rs index a79178d17ce7c..836a6c9ec12c7 100644 --- a/crates/sui-single-node-benchmark/src/single_node.rs +++ b/crates/sui-single-node-benchmark/src/single_node.rs @@ -157,7 +157,9 @@ impl SingleValidator { let response = self .validator_service .execute_certificate_for_testing(cert) - .await; + .await + .unwrap() + .into_inner(); response.signed_effects.into_data() } Component::TxnSigning | Component::CheckpointExecutor | Component::ExecutionOnly => { @@ -225,6 +227,8 @@ impl SingleValidator { self.validator_service .handle_transaction_for_testing(transaction) .await + .unwrap() + .into_inner() } pub(crate) async fn build_checkpoints( diff --git a/crates/sui-types/src/error.rs b/crates/sui-types/src/error.rs index 84c4b51b7662f..4440b83bbf04b 100644 --- a/crates/sui-types/src/error.rs +++ b/crates/sui-types/src/error.rs @@ -612,6 +612,9 @@ pub enum SuiError { #[error("Storage error: {0}")] Storage(String), + + #[error("Validator cannot handle the request at the moment. Please retry after at least {retry_after_sec}.")] + ValidatorOverloadedRetryAfter { retry_after_sec: u64 }, } #[repr(u64)] @@ -773,6 +776,7 @@ impl SuiError { SuiError::TooManyTransactionsPendingOnObject { .. } => true, SuiError::TooOldTransactionPendingOnObject { .. } => true, SuiError::TooManyTransactionsPendingConsensus => true, + SuiError::ValidatorOverloadedRetryAfter { .. } => true, // Non retryable error SuiError::ExecutionError(..) => false, diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.toml index 2012657bb1954..3f6ba5c8df5d8 100644 --- a/crates/workspace-hack/Cargo.toml +++ b/crates/workspace-hack/Cargo.toml @@ -792,7 +792,7 @@ try-lock = { version = "0.2", default-features = false } ttl_cache = { version = "0.5" } tui = { version = "0.17" } tungstenite = { version = "0.20", default-features = false, features = ["handshake"] } -twox-hash = { version = "1", default-features = false } +twox-hash = { version = "1" } typenum = { version = "1", default-features = false } ucd-trie = { version = "0.1", default-features = false, features = ["std"] } uint = { version = "0.9" } @@ -1752,7 +1752,7 @@ try-lock = { version = "0.2", default-features = false } ttl_cache = { version = "0.5" } tui = { version = "0.17" } tungstenite = { version = "0.20", default-features = false, features = ["handshake"] } -twox-hash = { version = "1", default-features = false } +twox-hash = { version = "1" } typenum = { version = "1", default-features = false } ucd-trie = { version = "0.1", default-features = false, features = ["std"] } uint = { version = "0.9" }