From 0589e8f8325fc47b4a2d84f1510b6e53dcc2c36e Mon Sep 17 00:00:00 2001 From: MrWad3r Date: Fri, 21 Jun 2024 12:51:40 +0200 Subject: [PATCH 01/15] feat(cli): implement method to test node storage --- cli/src/main.rs | 1 + cli/src/tools/storage_cli.rs | 274 +++++++++++++++++++++++++++++++++++ 2 files changed, 275 insertions(+) create mode 100644 cli/src/tools/storage_cli.rs diff --git a/cli/src/main.rs b/cli/src/main.rs index d18b11d1e..a1b22f303 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -9,6 +9,7 @@ mod tools { pub mod gen_dht; pub mod gen_key; pub mod gen_zerostate; + pub mod storage_cli; } mod node; diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs new file mode 100644 index 000000000..f2f255803 --- /dev/null +++ b/cli/src/tools/storage_cli.rs @@ -0,0 +1,274 @@ +use std::path::PathBuf; + +use anyhow::{Context, Result}; +use clap::{Args, Parser}; +use everscale_types::models::BlockId; +use serde::{Deserialize, Deserializer}; +use tycho_storage::{BlockConnection, KeyBlocksDirection, Storage, StorageConfig}; + +fn init_storage(path: Option<&PathBuf>) -> Result { + let default = PathBuf::from("./db"); + let config_path = path.unwrap_or(&default); + let config: StorageConfig = tycho_util::serde_helpers::load_json_from_file(config_path)?; + Storage::builder() + .with_config(config) + //.with_rpc_storage(node_config.rpc.is_some()) + .build() +} +pub struct Cmd {} + +enum SubCmd { + GetNextKeyblockIds(GetNextKeyBlockIdsCmd), + GetBlockFull(BlockCmd), + GetNextBlockFull(BlockCmd), + GetArchiveInfo(GetArchiveInfoCmd), + GetArchiveSlice(GetArchiveSliceCmd), + GetPersistentStateInfo(BlockCmd), + GetPersistentStatePart, +} +impl SubCmd { + async fn run(self) -> Result<()> { + match self { + Self::GetNextKeyblockIds(cmd) => cmd.run(), + Self::GetBlockFull(cmd) => cmd.run().await, + Self::GetNextBlockFull(cmd) => cmd.run_next().await, + Self::GetArchiveInfo(cmd) => cmd.run(), + Self::GetArchiveSlice(cmd) => cmd.run(), + Self::GetPersistentStateInfo(cmd) => cmd.get_state_info(), + Self::GetPersistentStatePart => Ok(()), + } + } +} + +#[derive(Deserialize)] +pub struct GetNextKeyBlockIdsCmd { + pub block_id: BlockId, + pub limit: usize, + pub storage_path: Option, +} + +impl GetNextKeyBlockIdsCmd { + pub fn run(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + + let block_handle_storage = storage.block_handle_storage(); + + let get_next_key_block_ids = || { + if !self.block_id.shard.is_masterchain() { + anyhow::bail!("first block id is not from masterchain"); + } + + let mut iterator = block_handle_storage + .key_blocks_iterator(KeyBlocksDirection::ForwardFrom(self.block_id.seqno)) + .take(self.limit); + + if let Some(id) = iterator.next() { + anyhow::ensure!( + id.root_hash == self.block_id.root_hash, + "first block root hash mismatch" + ); + anyhow::ensure!( + id.file_hash == self.block_id.file_hash, + "first block file hash mismatch" + ); + } + + Ok::<_, anyhow::Error>(iterator.take(self.limit).collect::>()) + }; + + match get_next_key_block_ids() { + Ok(ids) => { + if ids.len() < self.limit { + println!( + "Found {} blocks which is less then specified limit of {}", + ids.len(), + self.limit + ); + } + for i in ids.iter() { + println!("Found block {i}") + } + Ok(()) + } + Err(e) => { + println!("Operation failed: {e:?}"); + Ok(()) + } + } + } +} + +#[derive(Deserialize)] +pub struct BlockCmd { + pub block_id: BlockId, + pub storage_path: Option, +} + +impl BlockCmd { + pub async fn run(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + let block_handle_storage = storage.block_handle_storage(); + let block_storage = storage.block_storage(); + + let get_block_full = async { + let mut is_link = false; + match block_handle_storage.load_handle(&self.block_id) { + Some(handle) + if handle.meta().has_data() && handle.has_proof_or_link(&mut is_link) => + { + let block = block_storage.load_block_data_raw(&handle).await?; + let proof = block_storage.load_block_proof_raw(&handle, is_link).await?; + + println!("Found block full {}\n", &self.block_id); + println!("Block is link: {}\n", is_link); + println!("Block hex {}\n", hex::encode(&block)); + println!("Block proof {}\n", hex::encode(&proof)); + } + _ => { + println!("Found block empty {}\n", &self.block_id) + }, + }; + Ok::<(), anyhow::Error>(()) + }; + + match get_block_full.await { + Ok(()) => Ok(()), + Err(e) => { + println!("Get block full failed: {e:?}"); + Ok(()) + } + } + } + + pub async fn run_next(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + let block_handle_storage = storage.block_handle_storage(); + let block_connection_storage = storage.block_connection_storage(); + let block_storage = storage.block_storage(); + let get_next_block_full = async { + let next_block_id = match block_handle_storage.load_handle(&self.block_id) { + Some(handle) if handle.meta().has_next1() => block_connection_storage + .load_connection(&self.block_id, BlockConnection::Next1) + .context("connection not found")?, + _ => { + return Ok(()) + }, + }; + + let mut is_link = false; + match block_handle_storage.load_handle(&next_block_id) { + Some(handle) + if handle.meta().has_data() && handle.has_proof_or_link(&mut is_link) => + { + let block = block_storage.load_block_data_raw(&handle).await?; + let proof = block_storage.load_block_proof_raw(&handle, is_link).await?; + + println!("Found block full {}\n", &self.block_id); + println!("Block is link: {}\n", is_link); + println!("Block hex {}\n", hex::encode(&block)); + println!("Block proof {}\n", hex::encode(&proof)); + } + _ => { + println!("Found block empty {}\n", &self.block_id) + }, + }; + Ok::<(), anyhow::Error>(()) + }; + + match get_next_block_full.await { + Ok(_) => Ok(()), + Err(e) => { + println!("Get next block full failed: {e:?}"); + Ok(()) + } + } + } + + fn get_state_info(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + let persistent_state_storage = storage.persistent_state_storage(); + + if let Some(info) = persistent_state_storage.get_state_info(&self.block_id) { + println!("Find persistent state of size: {}", info.size); + } else { + println!("Persistent state not found"); + } + Ok(()) + } +} + +#[derive(Deserialize)] +pub struct GetArchiveInfoCmd { + pub mc_seqno: u32, + pub storage_path: Option, +} + +impl GetArchiveInfoCmd { + pub fn run(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + let node_state = storage.node_state(); + + match node_state.load_last_mc_block_id() { + Some(last_applied_mc_block) => { + if self.mc_seqno > last_applied_mc_block.seqno { + println!("Archive not found. Requested seqno is gt last applied seqno"); + return Ok(()); + } + + let block_storage = storage.block_storage(); + + match block_storage.get_archive_id(self.mc_seqno) { + Some(id) => println!("Archive found with id: {}", id), + None => println!("Archive not found"), + } + Ok(()) + } + None => { + println!("Get archive id failed: no blocks applied"); + Ok(()) + } + } + } +} + +#[derive(Deserialize)] +pub struct GetArchiveSliceCmd { + pub archive_id: u64, + pub limit: u32, + pub offset: u64, + pub storage_path: Option, +} + +impl GetArchiveSliceCmd { + pub fn run(&self) -> Result<()> { + let storage = init_storage(self.storage_path.as_ref())?; + let block_storage = storage.block_storage(); + + let get_archive_slice = || { + let Some(archive_slice) = block_storage.get_archive_slice( + self.archive_id as u32, + self.offset as usize, + self.limit as usize, + )? + else { + anyhow::bail!("Archive not found"); + }; + + Ok(archive_slice) + }; + + match get_archive_slice() { + Ok(data) => { + println!("Archive slice: {}", hex::encode(&data)); + Ok(()) + } + Err(e) => { + println!("Get archive slice failed: {e:?}"); + Ok(()) + } + } + } +} + +#[derive(Deserialize)] +pub struct GetPersistentStateInfo {} From e0c1c8061f9dbdcc178cf9f9991ae8e72b93f086 Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Mon, 24 Jun 2024 15:17:23 +0200 Subject: [PATCH 02/15] feat(storage): add archives gc --- cli/src/node/mod.rs | 6 ++- cli/src/tools/storage_cli.rs | 8 ++-- storage/src/lib.rs | 86 +++++++++++++++++++++++++++++++++++- util/src/time.rs | 20 ++++++++- 4 files changed, 112 insertions(+), 8 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index 7fad6feac..d36a5f009 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -40,7 +40,7 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{BlockMetaData, Storage}; +use tycho_storage::{start_archives_gc, BlockMetaData, Storage}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; @@ -363,6 +363,10 @@ impl Node { "initialized storage" ); + tokio::spawn(async move { + start_archives_gc(storage.clone())?; + }); + // Setup block strider let state_tracker = MinRefMcStateTracker::default(); diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index f2f255803..594afcb70 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -126,7 +126,7 @@ impl BlockCmd { } _ => { println!("Found block empty {}\n", &self.block_id) - }, + } }; Ok::<(), anyhow::Error>(()) }; @@ -150,9 +150,7 @@ impl BlockCmd { Some(handle) if handle.meta().has_next1() => block_connection_storage .load_connection(&self.block_id, BlockConnection::Next1) .context("connection not found")?, - _ => { - return Ok(()) - }, + _ => return Ok(()), }; let mut is_link = false; @@ -170,7 +168,7 @@ impl BlockCmd { } _ => { println!("Found block empty {}\n", &self.block_id) - }, + } }; Ok::<(), anyhow::Error>(()) }; diff --git a/storage/src/lib.rs b/storage/src/lib.rs index f226516db..b46382d82 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,7 +1,10 @@ +use std::ops::Add; +use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; -use std::time::Duration; +use std::time::{Duration, Instant}; use anyhow::Result; +use tokio::sync::Notify; use tycho_util::metrics::spawn_metrics_loop; use weedb::rocksdb; @@ -25,6 +28,7 @@ mod util { mod stored_value; } +use tycho_util::time::duration_between_unix_and_instant; // TODO move to weedb pub use util::owned_iterator; @@ -251,6 +255,86 @@ impl Storage { } } +pub fn start_archives_gc(storage: Storage) -> Result<()> { + let options = match &storage.inner.config.archives { + Some(options) => options, + None => return Ok(()), + }; + + struct LowerBound { + archive_id: AtomicU32, + changed: Notify, + } + + #[allow(unused_mut)] + let mut lower_bound = None::>; + + match options.gc_interval { + ArchivesGcInterval::Manual => Ok(()), + ArchivesGcInterval::PersistentStates { offset } => { + // let engine = self.clone(); + tokio::spawn(async move { + let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); + + loop { + tokio::pin!(let new_state_found = persistent_state_keeper.new_state_found();); + + let (until_id, untile_time) = match persistent_state_keeper.current() { + Some(state) => { + let untile_time = + (state.meta().gen_utime() as u64).add(offset.as_secs()); + (state.id().seqno, untile_time) + } + None => { + new_state_found.await; + continue; + } + }; + + if let Some(interval) = + duration_between_unix_and_instant(untile_time, Instant::now()) + { + tokio::select!( + _ = tokio::time::sleep(interval) => {}, + _ = &mut new_state_found => continue, + ); + } + + if let Some(lower_bound) = &lower_bound { + loop { + tokio::pin!(let lower_bound_changed = lower_bound.changed.notified();); + + let lower_bound = lower_bound.archive_id.load(Ordering::Acquire); + if until_id < lower_bound { + break; + } + + tracing::info!( + until_id, + lower_bound, + "waiting for the archives barrier" + ); + lower_bound_changed.await; + } + } + + if let Err(e) = storage + .block_storage() + .remove_outdated_archives(until_id) + .await + { + tracing::error!("failed to remove outdated archives: {e:?}"); + } + + new_state_found.await; + } + }); + + Ok(()) + } + } +} + struct Inner { root: FileDb, base_db: BaseDb, diff --git a/util/src/time.rs b/util/src/time.rs index c36fda5f0..65630e762 100644 --- a/util/src/time.rs +++ b/util/src/time.rs @@ -1,4 +1,4 @@ -use std::time::Duration; +use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; use rand::Rng; @@ -25,3 +25,21 @@ pub fn shifted_interval_immediate(period: Duration, max_shift: Duration) -> toki let shift = rand::thread_rng().gen_range(Duration::ZERO..max_shift); tokio::time::interval(period + shift) } + +pub fn duration_between_unix_and_instant(unix_time: u64, instant: Instant) -> Duration { + // Convert Unix timestamp to SystemTime + let system_time = UNIX_EPOCH + Duration::from_secs(unix_time); + + // Convert SystemTime to Instant + let instant_from_unix = Instant::now() + - SystemTime::now() + .duration_since(system_time) + .unwrap_or(Duration::ZERO); + + // Calculate the duration + if instant_from_unix <= instant { + instant - instant_from_unix + } else { + instant_from_unix - instant + } +} From 8760297b172815e5eaed00f891bf8c49656b873d Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Wed, 26 Jun 2024 18:39:46 +0200 Subject: [PATCH 03/15] feat(storage): add blocks gc (wip) --- Cargo.lock | 1 + cli/src/node/mod.rs | 15 ++-- core/src/block_strider/mod.rs | 3 +- core/src/block_strider/state_applier.rs | 5 ++ .../block_strider/subscriber/gc_subscriber.rs | 75 +++++++++++++++++++ .../subscriber/metrics_subscriber.rs | 5 ++ core/src/block_strider/subscriber/mod.rs | 39 ++++++++++ storage/Cargo.toml | 1 + storage/src/config.rs | 63 ++++++++++++++++ storage/src/lib.rs | 68 ++++++++++++++--- storage/src/store/node_state/mod.rs | 13 ++++ 11 files changed, 273 insertions(+), 15 deletions(-) create mode 100644 core/src/block_strider/subscriber/gc_subscriber.rs diff --git a/Cargo.lock b/Cargo.lock index 63abd0790..1ab3aa1d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3547,6 +3547,7 @@ dependencies = [ "parking_lot", "parking_lot_core", "quick_cache", + "rand", "rlimit", "scc", "serde", diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index d36a5f009..5bd29790b 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -10,6 +10,7 @@ use everscale_crypto::ed25519; use everscale_types::models::*; use everscale_types::prelude::*; use futures_util::future::BoxFuture; +use tracing::instrument::WithSubscriber; use tracing_subscriber::Layer; use tycho_block_util::state::{MinRefMcStateTracker, ShardStateStuff}; use tycho_collator::collator::CollatorStdImplFactory; @@ -40,7 +41,7 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{start_archives_gc, BlockMetaData, Storage}; +use tycho_storage::{start_archives_gc, BlockMetaData, Storage, prepare_blocks_gc}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; @@ -363,10 +364,6 @@ impl Node { "initialized storage" ); - tokio::spawn(async move { - start_archives_gc(storage.clone())?; - }); - // Setup block strider let state_tracker = MinRefMcStateTracker::default(); @@ -669,6 +666,8 @@ impl Node { let strider_state = PersistentBlockStriderState::new(self.zerostate.as_block_id(), self.storage.clone()); + let gc_subscriber = GcSubscriber::new(self.storage.clone()); + let block_strider = BlockStrider::builder() .with_provider(( (blockchain_block_provider, storage_block_provider), @@ -685,6 +684,12 @@ impl Node { )) .build(); + tokio::spawn(async move { + start_archives_gc(self.storage.clone())?; + }); + + prepare_blocks_gc(self.storage.clone()).await?; + // Run block strider tracing::info!("block strider started"); block_strider.run().await?; diff --git a/core/src/block_strider/mod.rs b/core/src/block_strider/mod.rs index 7afd6a688..6438ccccd 100644 --- a/core/src/block_strider/mod.rs +++ b/core/src/block_strider/mod.rs @@ -1,3 +1,4 @@ +use std::sync::atomic::Ordering; use std::sync::Arc; use anyhow::Result; @@ -24,7 +25,7 @@ pub use self::state_applier::ShardStateApplier; #[cfg(any(test, feature = "test"))] pub use self::subscriber::test::PrintSubscriber; pub use self::subscriber::{ - BlockSubscriber, BlockSubscriberContext, BlockSubscriberExt, ChainSubscriber, + BlockSubscriber, BlockSubscriberContext, BlockSubscriberExt, ChainSubscriber, GcSubscriber, MetricsSubscriber, NoopSubscriber, StateSubscriber, StateSubscriberContext, StateSubscriberExt, }; diff --git a/core/src/block_strider/state_applier.rs b/core/src/block_strider/state_applier.rs index 2b7fc67e8..c2af1f02c 100644 --- a/core/src/block_strider/state_applier.rs +++ b/core/src/block_strider/state_applier.rs @@ -245,6 +245,7 @@ where type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; + type AfterBlockHandleFut<'a> = BoxFuture<'a, Result<()>>; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { Box::pin(self.prepare_block_impl(cx)) @@ -257,6 +258,10 @@ where ) -> Self::HandleBlockFut<'a> { Box::pin(self.handle_block_impl(cx, prepared)) } + + fn after_block_handle<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + Box::pin(futures_util::future::ready(Ok(()))) + } } pub struct ShardApplierPrepared { diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs new file mode 100644 index 000000000..9a8819829 --- /dev/null +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -0,0 +1,75 @@ +use std::sync::atomic::Ordering; +use std::sync::Arc; + +use futures_util::future::BoxFuture; +use tycho_storage::Storage; + +use crate::block_strider::{BlockSubscriber, BlockSubscriberContext}; + +#[repr(transparent)] +pub struct GcSubscriber { + inner: Arc, +} + +impl GcSubscriber { + pub fn new(storage: Storage) -> Self { + Self { + inner: Arc::new(Inner { storage }), + } + } +} + +struct Inner { + storage: Storage, +} + +impl BlockSubscriber for GcSubscriber { + type Prepared = (); + type PrepareBlockFut<'a> = futures_util::future::Ready>; + type HandleBlockFut<'a> = futures_util::future::Ready>; + type AfterBlockHandleFut<'a> = BoxFuture<'a, anyhow::Result<()>>; + + fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { + futures_util::future::ready(Ok(())) + } + + fn handle_block<'a>( + &'a self, + _cx: &'a BlockSubscriberContext, + _prepared: Self::Prepared, + ) -> Self::HandleBlockFut<'a> { + futures_util::future::ready(Ok(())) + } + + fn after_block_handle<'a>( + &'a self, + cx: &'a BlockSubscriberContext, + ) -> Self::AfterBlockHandleFut<'a> { + let block_info = cx.block.block().load_info()?; + self.inner + .storage + .node_state() + .store_shards_client_mc_block_id(&cx.mc_block_id)?; + + let enabled = self + .inner + .storage + .gc_enable_for_sync() + .load(Ordering::Acquire); + + match ( + self.inner.storage.config().blocks_gc_config, + enabled, + block_info.key_block, + ) { + (Some(config), true, true) => { + Box::pin(self.inner.storage.block_storage().remove_outdated_blocks( + &cx.mc_block_id, + config.max_blocks_per_batch, + config.kind, + )) + } + _ => Box::pin(futures_util::future::ready(Ok(()))), + } + } +} diff --git a/core/src/block_strider/subscriber/metrics_subscriber.rs b/core/src/block_strider/subscriber/metrics_subscriber.rs index 4deb87ced..6088f02c5 100644 --- a/core/src/block_strider/subscriber/metrics_subscriber.rs +++ b/core/src/block_strider/subscriber/metrics_subscriber.rs @@ -15,6 +15,7 @@ impl BlockSubscriber for MetricsSubscriber { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = futures_util::future::Ready>; + type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, _: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -30,6 +31,10 @@ impl BlockSubscriber for MetricsSubscriber { } futures_util::future::ready(Ok(())) } + + fn after_block_handle<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl StateSubscriber for MetricsSubscriber { diff --git a/core/src/block_strider/subscriber/mod.rs b/core/src/block_strider/subscriber/mod.rs index ddf6828bd..c81b338e1 100644 --- a/core/src/block_strider/subscriber/mod.rs +++ b/core/src/block_strider/subscriber/mod.rs @@ -9,8 +9,10 @@ use tycho_block_util::block::BlockStuff; use tycho_block_util::state::ShardStateStuff; pub use self::metrics_subscriber::MetricsSubscriber; +pub use self::gc_subscriber::GcSubscriber; mod metrics_subscriber; +mod gc_subscriber; // === trait BlockSubscriber === @@ -25,6 +27,7 @@ pub trait BlockSubscriber: Send + Sync + 'static { type PrepareBlockFut<'a>: Future> + Send + 'a; type HandleBlockFut<'a>: Future> + Send + 'a; + type AfterBlockHandleFut<'a>: Future> + Send + 'a; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a>; @@ -33,6 +36,11 @@ pub trait BlockSubscriber: Send + Sync + 'static { cx: &'a BlockSubscriberContext, prepared: Self::Prepared, ) -> Self::HandleBlockFut<'a>; + + fn after_block_handle<'a>( + &'a self, + cx: &'a BlockSubscriberContext, + ) -> Self::AfterBlockHandleFut<'a>; } impl BlockSubscriber for Box { @@ -40,6 +48,7 @@ impl BlockSubscriber for Box { type PrepareBlockFut<'a> = T::PrepareBlockFut<'a>; type HandleBlockFut<'a> = T::HandleBlockFut<'a>; + type AfterBlockHandleFut<'a> = future::Ready>; #[inline] fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { @@ -54,6 +63,10 @@ impl BlockSubscriber for Box { ) -> Self::HandleBlockFut<'a> { ::handle_block(self, cx, prepared) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl BlockSubscriber for Arc { @@ -61,6 +74,7 @@ impl BlockSubscriber for Arc { type PrepareBlockFut<'a> = T::PrepareBlockFut<'a>; type HandleBlockFut<'a> = T::HandleBlockFut<'a>; + type AfterBlockHandleFut<'a> = future::Ready>; #[inline] fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { @@ -75,6 +89,10 @@ impl BlockSubscriber for Arc { ) -> Self::HandleBlockFut<'a> { ::handle_block(self, cx, prepared) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } pub trait BlockSubscriberExt: Sized { @@ -156,6 +174,7 @@ impl BlockSubscriber for NoopSubscriber { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = futures_util::future::Ready>; + type AfterBlockHandleFut<'a> = futures_util::future::Ready>; #[inline] fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'_> { @@ -170,6 +189,10 @@ impl BlockSubscriber for NoopSubscriber { ) -> Self::HandleBlockFut<'_> { futures_util::future::ready(Ok(())) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl StateSubscriber for NoopSubscriber { @@ -193,6 +216,8 @@ impl BlockSubscriber for ChainSubscrib type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; + type AfterBlockHandleFut<'a> = futures_util::future::Ready>; + fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { let left = self.left.prepare_block(cx); let right = self.right.prepare_block(cx); @@ -218,6 +243,10 @@ impl BlockSubscriber for ChainSubscrib right.await }) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl StateSubscriber for ChainSubscriber { @@ -241,6 +270,7 @@ impl BlockSubscriber for (T1, T2) { type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; + type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { let left = self.0.prepare_block(cx); @@ -267,6 +297,10 @@ impl BlockSubscriber for (T1, T2) { l.and(r) }) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl StateSubscriber for (T1, T2) { @@ -295,6 +329,7 @@ pub mod test { type PrepareBlockFut<'a> = future::Ready>; type HandleBlockFut<'a> = future::Ready>; + type AfterBlockHandleFut<'a> = future::Ready>; fn prepare_block<'a>( &'a self, @@ -320,6 +355,10 @@ pub mod test { ); future::ready(Ok(())) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } impl StateSubscriber for PrintSubscriber { diff --git a/storage/Cargo.toml b/storage/Cargo.toml index 49276acf2..4f5c59716 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -27,6 +27,7 @@ num-traits = { workspace = true } parking_lot = { workspace = true } parking_lot_core = { workspace = true } quick_cache = { workspace = true } +rand = { workspace = true } rlimit = { workspace = true } scc = { workspace = true } serde = { workspace = true } diff --git a/storage/src/config.rs b/storage/src/config.rs index 27ae954a8..e296b1e6b 100644 --- a/storage/src/config.rs +++ b/storage/src/config.rs @@ -2,9 +2,12 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use bytesize::ByteSize; +use rand::Rng; use serde::{Deserialize, Serialize}; use tycho_util::serde_helpers; +use crate::BlocksGcKind; + #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct StorageConfig { @@ -32,6 +35,18 @@ pub struct StorageConfig { /// /// Archives are disabled if this field is `None`. pub archives: Option, + + /// States GC config. + /// + /// States GC is disabled if this field is `None`. + pub states_gc_options: Option, + + /// Blocks GC config. + /// + /// Blocks GC is disabled if this field is `None`. + pub blocks_gc_config: Option, + + } impl StorageConfig { @@ -43,6 +58,8 @@ impl StorageConfig { cells_cache_size: ByteSize::kb(1024), rocksdb_enable_metrics: false, archives: Some(ArchivesConfig::default()), + states_gc_options: Some(StateGcOptions::default()), + blocks_gc_config: Some(BlocksGcOptions::default()) } } } @@ -92,6 +109,8 @@ impl Default for StorageConfig { rocksdb_lru_capacity, rocksdb_enable_metrics: true, archives: Some(ArchivesConfig::default()), + states_gc_options: Some(StateGcOptions::default()), + blocks_gc_config: Some(BlocksGcOptions::default()) } } } @@ -122,3 +141,47 @@ impl Default for ArchivesGcInterval { } } } + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct StateGcOptions { + /// Default: rand[0,900) + pub offset_sec: u64, + /// Default: 900 + pub interval_sec: u64, +} + +impl Default for StateGcOptions { + fn default() -> Self { + Self { + offset_sec: rand::thread_rng().gen_range(0..60), + interval_sec: 60, + } + } +} + +#[derive(Debug, Copy, Clone, Serialize, Deserialize)] +#[serde(default, deny_unknown_fields)] +pub struct BlocksGcOptions { + /// Blocks GC type + /// - `before_previous_key_block` - on each new key block delete all blocks before the previous one + /// - `before_previous_persistent_state` - on each new key block delete all blocks before the + /// previous key block with persistent state + pub kind: BlocksGcKind, + + /// Whether to enable blocks GC during sync. Default: true + pub enable_for_sync: bool, + + /// Max `WriteBatch` entries before apply + pub max_blocks_per_batch: Option, +} + +impl Default for BlocksGcOptions { + fn default() -> Self { + Self { + kind: BlocksGcKind::BeforePreviousPersistentState, + enable_for_sync: true, + max_blocks_per_batch: Some(100_000), + } + } +} diff --git a/storage/src/lib.rs b/storage/src/lib.rs index b46382d82..20e438953 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,5 +1,5 @@ use std::ops::Add; -use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; use std::time::{Duration, Instant}; @@ -143,6 +143,13 @@ impl StorageBuilder { // TODO: preload archive ids + let gc_enabled_for_sync = AtomicBool::new( + self.config + .blocks_gc_config + .map(|x| x.enable_for_sync) + .unwrap_or(false), + ); + let inner = Arc::new(Inner { root, base_db, @@ -156,6 +163,9 @@ impl StorageBuilder { runtime_storage, rpc_state, internal_queue_storage, + blocks_gc_state: BlockGcState { + enabled_for_sync: gc_enabled_for_sync, + }, }); spawn_metrics_loop(&inner, Duration::from_secs(5), |this| async move { @@ -253,6 +263,44 @@ impl Storage { pub fn internal_queue_storage(&self) -> &InternalQueueStorage { &self.inner.internal_queue_storage } + + pub fn gc_enable_for_sync(&self) -> &AtomicBool { + &self.inner.blocks_gc_state.enabled_for_sync + } +} + +pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { + let blocks_gc_config = match &storage.inner.config.blocks_gc_config { + Some(state) => state, + None => return Ok(()), + }; + + storage + .inner + .blocks_gc_state + .enabled_for_sync + .store(true, Ordering::Release); + + let Some(key_block) = storage.block_handle_storage().find_last_key_block() else { + return Ok(()) + }; + + let Some(block) = storage.node_state().load_shards_client_mc_block_id() else { + return Ok(()) + }; + // Blocks GC will be called later when the shards client will reach the key block + if block.seqno < key_block.id().seqno { + return Ok(()); + } + + storage + .block_storage() + .remove_outdated_blocks( + key_block.id(), + blocks_gc_config.max_blocks_per_batch, + blocks_gc_config.kind, + ) + .await } pub fn start_archives_gc(storage: Storage) -> Result<()> { @@ -291,14 +339,10 @@ pub fn start_archives_gc(storage: Storage) -> Result<()> { } }; - if let Some(interval) = - duration_between_unix_and_instant(untile_time, Instant::now()) - { - tokio::select!( - _ = tokio::time::sleep(interval) => {}, - _ = &mut new_state_found => continue, - ); - } + tokio::select!( + _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, + _ = &mut new_state_found => continue, + ); if let Some(lower_bound) = &lower_bound { loop { @@ -340,6 +384,8 @@ struct Inner { base_db: BaseDb, config: StorageConfig, + blocks_gc_state: BlockGcState, + runtime_storage: Arc, block_handle_storage: Arc, block_connection_storage: Arc, @@ -350,3 +396,7 @@ struct Inner { rpc_state: Option, internal_queue_storage: InternalQueueStorage, } + +struct BlockGcState { + enabled_for_sync: AtomicBool, +} diff --git a/storage/src/store/node_state/mod.rs b/storage/src/store/node_state/mod.rs index a067d0f65..8eaa4b65e 100644 --- a/storage/src/store/node_state/mod.rs +++ b/storage/src/store/node_state/mod.rs @@ -8,6 +8,7 @@ pub struct NodeStateStorage { db: BaseDb, last_mc_block_id: BlockIdCache, init_mc_block_id: BlockIdCache, + shards_client_mc_block_id: BlockIdCache, } impl NodeStateStorage { @@ -16,6 +17,7 @@ impl NodeStateStorage { db, last_mc_block_id: (Default::default(), LAST_MC_BLOCK_ID), init_mc_block_id: (Default::default(), INIT_MC_BLOCK_ID), + shards_client_mc_block_id: (Default::default(), SHARDS_CLIENT_MC_BLOCK_ID) } } @@ -35,6 +37,15 @@ impl NodeStateStorage { self.load_block_id(&self.init_mc_block_id) } + pub fn store_shards_client_mc_block_id(&self, id: &BlockId) { + self.store_block_id(&self.shards_client_mc_block_id, id) + } + + pub fn load_shards_client_mc_block_id(&self) -> Option { + self.load_block_id(&self.shards_client_mc_block_id) + } + + #[inline(always)] fn store_block_id(&self, (cache, key): &BlockIdCache, block_id: &BlockId) { let node_states = &self.db.state; @@ -60,3 +71,5 @@ type BlockIdCache = (Mutex>, &'static [u8]); const LAST_MC_BLOCK_ID: &[u8] = b"last_mc_block"; const INIT_MC_BLOCK_ID: &[u8] = b"init_mc_block"; +const SHARDS_CLIENT_MC_BLOCK_ID: &[u8] = b"shards_client_mc_block"; + From 0e019d8749c99b2000f91693ec96a3a1704991f9 Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Thu, 27 Jun 2024 13:53:09 +0200 Subject: [PATCH 04/15] fix(cli): fix build (wip) --- cli/src/node/mod.rs | 5 +- cli/src/tools/storage_cli.rs | 20 ++++- .../block_strider/subscriber/gc_subscriber.rs | 7 +- rpc/src/state/mod.rs | 5 ++ storage/src/lib.rs | 89 +++++++++++++++++++ 5 files changed, 121 insertions(+), 5 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index 5bd29790b..bd150901f 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -684,8 +684,11 @@ impl Node { )) .build(); + let st = self.storage.clone(); tokio::spawn(async move { - start_archives_gc(self.storage.clone())?; + if let Err(e) = start_archives_gc(st) { + tracing::error!("Failed to execute archives gc. {e:?}"); + }; }); prepare_blocks_gc(self.storage.clone()).await?; diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 594afcb70..405a674dd 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -1,7 +1,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; -use clap::{Args, Parser}; +use clap::{Args, Parser, Subcommand}; use everscale_types::models::BlockId; use serde::{Deserialize, Deserializer}; use tycho_storage::{BlockConnection, KeyBlocksDirection, Storage, StorageConfig}; @@ -15,8 +15,20 @@ fn init_storage(path: Option<&PathBuf>) -> Result { //.with_rpc_storage(node_config.rpc.is_some()) .build() } -pub struct Cmd {} +#[derive(clap::Parser)] +pub struct Cmd { + #[clap(subcommand)] + cmd: SubCmd, +} + +impl Cmd { + pub async fn run(self) -> Result<()> { + self.cmd.run().await + } +} + +#[derive(Subcommand)] enum SubCmd { GetNextKeyblockIds(GetNextKeyBlockIdsCmd), GetBlockFull(BlockCmd), @@ -41,6 +53,7 @@ impl SubCmd { } #[derive(Deserialize)] +#[derive(Parser)] pub struct GetNextKeyBlockIdsCmd { pub block_id: BlockId, pub limit: usize, @@ -99,6 +112,7 @@ impl GetNextKeyBlockIdsCmd { } #[derive(Deserialize)] +#[derive(Parser)] pub struct BlockCmd { pub block_id: BlockId, pub storage_path: Option, @@ -196,6 +210,7 @@ impl BlockCmd { } #[derive(Deserialize)] +#[derive(Parser)] pub struct GetArchiveInfoCmd { pub mc_seqno: u32, pub storage_path: Option, @@ -230,6 +245,7 @@ impl GetArchiveInfoCmd { } #[derive(Deserialize)] +#[derive(Parser)] pub struct GetArchiveSliceCmd { pub archive_id: u64, pub limit: u32, diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 9a8819829..e720f8826 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -45,11 +45,14 @@ impl BlockSubscriber for GcSubscriber { &'a self, cx: &'a BlockSubscriberContext, ) -> Self::AfterBlockHandleFut<'a> { - let block_info = cx.block.block().load_info()?; + let Ok(block_info) = cx.block.block().load_info() else { + return Box::pin(futures_util::future::ready(Ok(()))) + }; + self.inner .storage .node_state() - .store_shards_client_mc_block_id(&cx.mc_block_id)?; + .store_shards_client_mc_block_id(&cx.mc_block_id); let enabled = self .inner diff --git a/rpc/src/state/mod.rs b/rpc/src/state/mod.rs index 2b2e0fc83..adcb64ac3 100644 --- a/rpc/src/state/mod.rs +++ b/rpc/src/state/mod.rs @@ -216,6 +216,7 @@ impl BlockSubscriber for RpcState { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; + type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, _: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -228,6 +229,10 @@ impl BlockSubscriber for RpcState { ) -> Self::HandleBlockFut<'a> { Box::pin(self.inner.update(&cx.block, None)) } + + fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { + futures_util::future::ready(Ok(())) + } } struct Inner { diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 20e438953..fd4854ad1 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -303,6 +303,95 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { .await } +// fn start_states_gc(storage: Storage) { +// let options = match storage.inner.config.states_gc_options { +// Some(options) => options, +// None => return, +// }; +// +// //let engine = Arc::downgrade(self); +// +// // if let Some(ttl) = self.shard_states_cache.ttl() { +// // let engine = engine.clone(); +// // tokio::spawn(async move { +// // loop { +// // tokio::time::sleep(ttl).await; +// // match engine.upgrade() { +// // Some(engine) => engine.shard_states_cache.clear(), +// // None => break, +// // } +// // } +// // }); +// // } +// +// // Compute gc timestamp aligned to `interval_sec` with an offset `offset_sec` +// let mut gc_at = Instant::now(); +// gc_at = (gc_at - gc_at % options.interval_sec) + options.offset_sec; +// +// tokio::spawn(async move { +// 'gc: loop { +// // Shift gc timestamp one iteration further +// gc_at += options.interval_sec; +// // Check if there is some time left before the GC +// if let Some(interval) = gc_at.checked_sub(broxus_util::now_sec_u64()) { +// tokio::time::sleep(Duration::from_secs(interval)).await; +// } +// +// // let engine = match engine.upgrade() { +// // Some(engine) => engine, +// // None => return, +// // }; +// +// let subscriber = engine.subscriber.as_ref(); +// +// let start = Instant::now(); +// let mut shards_gc_lock = storage +// .runtime_storage() +// .persistent_state_keeper() +// .shards_gc_lock() +// .subscribe(); +// metrics::histogram!("shard_states_gc_lock_time").record(start.elapsed()); +// +// let block_id = loop { +// // Load the latest block id +// let block_id = match engine.load_shards_client_mc_block_id() { +// Ok(block_id) => block_id, +// Err(e) => { +// tracing::error!("failed to load last shards client block: {e:?}"); +// continue 'gc; +// } +// }; +// +// if *shards_gc_lock.borrow_and_update() { +// if shards_gc_lock.changed().await.is_err() { +// tracing::warn!("stopping shard states GC"); +// return; +// } +// continue; +// } +// +// break block_id; +// }; +// +// subscriber.on_before_states_gc(&block_id).await; +// +// let shard_state_storage = storage.shard_state_storage(); +// let top_blocks = match shard_state_storage +// .remove_outdated_states(block_id.seq_no) +// .await +// { +// Ok(top_blocks) => Some(top_blocks), +// Err(e) => { +// tracing::error!("Failed to GC state: {e:?}"); +// None +// } +// }; +// +// subscriber.on_after_states_gc(&block_id, &top_blocks).await; +// } +// }); +// } + pub fn start_archives_gc(storage: Storage) -> Result<()> { let options = match &storage.inner.config.archives { Some(options) => options, From ec86fce91eaabb7e8b199ec1f393aeb206d6049a Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Thu, 27 Jun 2024 14:22:02 +0200 Subject: [PATCH 05/15] fix(cli): fix cli commands --- cli/src/main.rs | 16 +++++++++++----- cli/src/tools/storage_cli.rs | 17 +++-------------- 2 files changed, 14 insertions(+), 19 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index a1b22f303..fd9461cb2 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,6 +3,7 @@ use std::sync::OnceLock; use anyhow::Result; use clap::{Parser, Subcommand}; +use crate::tools::storage_cli::StorageCmd; mod tools { pub mod gen_account; @@ -19,13 +20,14 @@ mod util; #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -fn main() -> ExitCode { +#[tokio::main] +async fn main() -> ExitCode { if std::env::var("RUST_BACKTRACE").is_err() { // Enable backtraces on panics by default. std::env::set_var("RUST_BACKTRACE", "1"); } - match App::parse().run() { + match App::parse().run().await { Ok(()) => ExitCode::SUCCESS, Err(err) => { eprintln!("Error: {err}"); @@ -45,8 +47,8 @@ struct App { } impl App { - fn run(self) -> Result<()> { - self.cmd.run() + async fn run(self) -> Result<()> { + self.cmd.run().await } } @@ -57,13 +59,17 @@ enum Cmd { #[clap(subcommand)] Tool(ToolCmd), + #[clap(subcommand)] + Storage(StorageCmd), + } impl Cmd { - fn run(self) -> Result<()> { + async fn run(self) -> Result<()> { match self { Cmd::Node(cmd) => cmd.run(), Cmd::Tool(cmd) => cmd.run(), + Cmd::Storage(cmd) => cmd.run().await } } } diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 405a674dd..0a0d480ae 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -15,21 +15,10 @@ fn init_storage(path: Option<&PathBuf>) -> Result { //.with_rpc_storage(node_config.rpc.is_some()) .build() } -#[derive(clap::Parser)] -pub struct Cmd { - #[clap(subcommand)] - cmd: SubCmd, -} - -impl Cmd { - pub async fn run(self) -> Result<()> { - self.cmd.run().await - } -} #[derive(Subcommand)] -enum SubCmd { +pub enum StorageCmd { GetNextKeyblockIds(GetNextKeyBlockIdsCmd), GetBlockFull(BlockCmd), GetNextBlockFull(BlockCmd), @@ -38,8 +27,8 @@ enum SubCmd { GetPersistentStateInfo(BlockCmd), GetPersistentStatePart, } -impl SubCmd { - async fn run(self) -> Result<()> { +impl StorageCmd { + pub(crate) async fn run(self) -> Result<()> { match self { Self::GetNextKeyblockIds(cmd) => cmd.run(), Self::GetBlockFull(cmd) => cmd.run().await, From 14299d74d2c2e7ba2de8a6252b82d1a16144d52f Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Fri, 28 Jun 2024 15:42:51 +0200 Subject: [PATCH 06/15] feat(storage): add persistent state gc --- cli/src/node/mod.rs | 12 +- storage/src/lib.rs | 163 ++++++++---------- .../store/runtime/persistent_state_keeper.rs | 27 ++- 3 files changed, 103 insertions(+), 99 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index bd150901f..baa783320 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -41,7 +41,7 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{start_archives_gc, BlockMetaData, Storage, prepare_blocks_gc}; +use tycho_storage::{start_archives_gc, BlockMetaData, Storage, prepare_blocks_gc, start_states_gc}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; @@ -684,13 +684,11 @@ impl Node { )) .build(); - let st = self.storage.clone(); - tokio::spawn(async move { - if let Err(e) = start_archives_gc(st) { - tracing::error!("Failed to execute archives gc. {e:?}"); - }; - }); + if let Err(e) = start_archives_gc(self.storage.clone()) { + tracing::error!("Failed to execute archives gc. {e:?}"); + }; + start_states_gc(self.storage.clone()); prepare_blocks_gc(self.storage.clone()).await?; // Run block strider diff --git a/storage/src/lib.rs b/storage/src/lib.rs index fd4854ad1..e37f6b2b3 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -5,7 +5,8 @@ use std::time::{Duration, Instant}; use anyhow::Result; use tokio::sync::Notify; -use tycho_util::metrics::spawn_metrics_loop; +use tycho_util::metrics::{spawn_metrics_loop, HistogramGuard}; +use tycho_util::time::now_sec; use weedb::rocksdb; pub use self::config::*; @@ -282,11 +283,11 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { .store(true, Ordering::Release); let Some(key_block) = storage.block_handle_storage().find_last_key_block() else { - return Ok(()) + return Ok(()); }; let Some(block) = storage.node_state().load_shards_client_mc_block_id() else { - return Ok(()) + return Ok(()); }; // Blocks GC will be called later when the shards client will reach the key block if block.seqno < key_block.id().seqno { @@ -303,94 +304,74 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { .await } -// fn start_states_gc(storage: Storage) { -// let options = match storage.inner.config.states_gc_options { -// Some(options) => options, -// None => return, -// }; -// -// //let engine = Arc::downgrade(self); -// -// // if let Some(ttl) = self.shard_states_cache.ttl() { -// // let engine = engine.clone(); -// // tokio::spawn(async move { -// // loop { -// // tokio::time::sleep(ttl).await; -// // match engine.upgrade() { -// // Some(engine) => engine.shard_states_cache.clear(), -// // None => break, -// // } -// // } -// // }); -// // } -// -// // Compute gc timestamp aligned to `interval_sec` with an offset `offset_sec` -// let mut gc_at = Instant::now(); -// gc_at = (gc_at - gc_at % options.interval_sec) + options.offset_sec; -// -// tokio::spawn(async move { -// 'gc: loop { -// // Shift gc timestamp one iteration further -// gc_at += options.interval_sec; -// // Check if there is some time left before the GC -// if let Some(interval) = gc_at.checked_sub(broxus_util::now_sec_u64()) { -// tokio::time::sleep(Duration::from_secs(interval)).await; -// } -// -// // let engine = match engine.upgrade() { -// // Some(engine) => engine, -// // None => return, -// // }; -// -// let subscriber = engine.subscriber.as_ref(); -// -// let start = Instant::now(); -// let mut shards_gc_lock = storage -// .runtime_storage() -// .persistent_state_keeper() -// .shards_gc_lock() -// .subscribe(); -// metrics::histogram!("shard_states_gc_lock_time").record(start.elapsed()); -// -// let block_id = loop { -// // Load the latest block id -// let block_id = match engine.load_shards_client_mc_block_id() { -// Ok(block_id) => block_id, -// Err(e) => { -// tracing::error!("failed to load last shards client block: {e:?}"); -// continue 'gc; -// } -// }; -// -// if *shards_gc_lock.borrow_and_update() { -// if shards_gc_lock.changed().await.is_err() { -// tracing::warn!("stopping shard states GC"); -// return; -// } -// continue; -// } -// -// break block_id; -// }; -// -// subscriber.on_before_states_gc(&block_id).await; -// -// let shard_state_storage = storage.shard_state_storage(); -// let top_blocks = match shard_state_storage -// .remove_outdated_states(block_id.seq_no) -// .await -// { -// Ok(top_blocks) => Some(top_blocks), -// Err(e) => { -// tracing::error!("Failed to GC state: {e:?}"); -// None -// } -// }; -// -// subscriber.on_after_states_gc(&block_id, &top_blocks).await; -// } -// }); -// } +pub fn start_states_gc(storage: Storage) { + let options = match storage.inner.config.states_gc_options { + Some(options) => options, + None => return, + }; + + // Compute gc timestamp aligned to `interval_sec` with an offset `offset_sec` + let mut gc_at = now_sec() as u64; + gc_at = (gc_at - gc_at % options.interval_sec) + options.offset_sec; + + tokio::spawn(async move { + 'gc: loop { + // Shift gc timestamp one iteration further + gc_at += options.interval_sec; + + // Check if there is some time left before the GC + if let Some(interval) = gc_at.checked_sub(now_sec() as u64) { + tokio::time::sleep(Duration::from_secs(interval)).await; + } + + let start = Instant::now(); + let mut shards_gc_lock = storage + .runtime_storage() + .persistent_state_keeper() + .shards_gc_lock() + .subscribe(); + metrics::histogram!("tycho_storage_shard_states_gc_lock_time").record(start.elapsed()); + + let block_id = loop { + // Load the latest block id + let block_id = match storage.node_state().load_shards_client_mc_block_id() { + Some(block_id) => block_id, + None => { + tracing::error!(target: "storage", "Failed to load last shards client block. Block not found"); + continue 'gc; + } + }; + + if *shards_gc_lock.borrow_and_update() { + if shards_gc_lock.changed().await.is_err() { + tracing::warn!(target: "storage", "Stopping shard states GC"); + return; + } + continue; + } + + break block_id; + }; + + // subscriber.on_before_states_gc(&block_id).await; + + let _histogram = HistogramGuard::begin("tycho_storage_remove_outdated_states_time"); + let shard_state_storage = storage.shard_state_storage(); + let _ = match shard_state_storage + .remove_outdated_states(block_id.seqno) + .await + { + Ok(top_blocks) => Some(top_blocks), + Err(e) => { + tracing::error!(target: "storage", "Failed to GC state: {e:?}"); + None + } + }; + + // subscriber.on_after_states_gc(&block_id, &top_blocks).await; + } + }); +} pub fn start_archives_gc(storage: Storage) -> Result<()> { let options = match &storage.inner.config.archives { diff --git a/storage/src/store/runtime/persistent_state_keeper.rs b/storage/src/store/runtime/persistent_state_keeper.rs index 98968bb6d..2dee0b4e8 100644 --- a/storage/src/store/runtime/persistent_state_keeper.rs +++ b/storage/src/store/runtime/persistent_state_keeper.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Result; use arc_swap::ArcSwapAny; -use tokio::sync::Notify; +use tokio::sync::{Notify, watch}; use tycho_block_util::state::*; use crate::models::{BlockHandle, BriefBlockMeta}; @@ -15,16 +15,22 @@ pub struct PersistentStateKeeper { persistent_state_changed: Notify, current_persistent_state: ArcSwapAny>, last_utime: AtomicU32, + lock_gc_on_states: AtomicBool, + shards_gc_lock: watch::Sender, } impl PersistentStateKeeper { pub fn new(block_handle_storage: Arc) -> Self { + let (shards_gc_lock, _) = watch::channel(true); + Self { block_handle_storage, initialized: Default::default(), persistent_state_changed: Default::default(), current_persistent_state: Default::default(), last_utime: Default::default(), + lock_gc_on_states: AtomicBool::new(true), + shards_gc_lock, } } @@ -58,6 +64,10 @@ impl PersistentStateKeeper { } if is_persistent_state(block_utime, prev_utime) { + if self.lock_gc_on_states.load(Ordering::Acquire) { + self.shards_gc_lock.send_replace(true); + } + self.last_utime.store(block_utime, Ordering::Release); self.current_persistent_state .store(Some(block_handle.clone())); @@ -85,4 +95,19 @@ impl PersistentStateKeeper { pub fn new_state_found(&self) -> tokio::sync::futures::Notified<'_> { self.persistent_state_changed.notified() } + + pub fn set_shards_gc_lock_enabled(&self, enabled: bool) { + self.lock_gc_on_states.store(enabled, Ordering::Release); + if !enabled { + self.unlock_states_gc(); + } + } + + pub fn unlock_states_gc(&self) { + self.shards_gc_lock.send_replace(false); + } + + pub fn shards_gc_lock(&self) -> &watch::Sender { + &self.shards_gc_lock + } } From a6b6323d43e8b9aa278a03192e1d90d667c70102 Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Fri, 28 Jun 2024 17:23:07 +0200 Subject: [PATCH 07/15] fix(storage): remove AfterBlockHandleFut. Move GcSubscribers to state subscriber from block subscriber --- cli/src/node/mod.rs | 8 +-- cli/src/tools/storage_cli.rs | 4 +- core/src/block_strider/state_applier.rs | 5 -- .../block_strider/subscriber/gc_subscriber.rs | 62 +++++++++++-------- .../subscriber/metrics_subscriber.rs | 5 -- core/src/block_strider/subscriber/mod.rs | 36 ----------- rpc/src/state/mod.rs | 5 -- storage/src/lib.rs | 4 +- storage/src/store/node_state/mod.rs | 11 ---- 9 files changed, 43 insertions(+), 97 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index baa783320..c77c56d3e 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -10,7 +10,6 @@ use everscale_crypto::ed25519; use everscale_types::models::*; use everscale_types::prelude::*; use futures_util::future::BoxFuture; -use tracing::instrument::WithSubscriber; use tracing_subscriber::Layer; use tycho_block_util::state::{MinRefMcStateTracker, ShardStateStuff}; use tycho_collator::collator::CollatorStdImplFactory; @@ -41,7 +40,9 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{start_archives_gc, BlockMetaData, Storage, prepare_blocks_gc, start_states_gc}; +use tycho_storage::{ + prepare_blocks_gc, start_archives_gc, start_states_gc, BlockMetaData, Storage, +}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; @@ -680,11 +681,10 @@ impl Node { self.storage.clone(), (collator_state_subscriber, rpc_state), ), - MetricsSubscriber, + (MetricsSubscriber, gc_subscriber), )) .build(); - if let Err(e) = start_archives_gc(self.storage.clone()) { tracing::error!("Failed to execute archives gc. {e:?}"); }; diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 0a0d480ae..77225da90 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -1,9 +1,9 @@ use std::path::PathBuf; use anyhow::{Context, Result}; -use clap::{Args, Parser, Subcommand}; +use clap::{Parser, Subcommand}; use everscale_types::models::BlockId; -use serde::{Deserialize, Deserializer}; +use serde::{Deserialize}; use tycho_storage::{BlockConnection, KeyBlocksDirection, Storage, StorageConfig}; fn init_storage(path: Option<&PathBuf>) -> Result { diff --git a/core/src/block_strider/state_applier.rs b/core/src/block_strider/state_applier.rs index c2af1f02c..2b7fc67e8 100644 --- a/core/src/block_strider/state_applier.rs +++ b/core/src/block_strider/state_applier.rs @@ -245,7 +245,6 @@ where type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; - type AfterBlockHandleFut<'a> = BoxFuture<'a, Result<()>>; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { Box::pin(self.prepare_block_impl(cx)) @@ -258,10 +257,6 @@ where ) -> Self::HandleBlockFut<'a> { Box::pin(self.handle_block_impl(cx, prepared)) } - - fn after_block_handle<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - Box::pin(futures_util::future::ready(Ok(()))) - } } pub struct ShardApplierPrepared { diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index e720f8826..f7e8182a8 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -4,7 +4,9 @@ use std::sync::Arc; use futures_util::future::BoxFuture; use tycho_storage::Storage; -use crate::block_strider::{BlockSubscriber, BlockSubscriberContext}; +use crate::block_strider::{ + BlockSubscriber, BlockSubscriberContext, StateSubscriber, StateSubscriberContext, +}; #[repr(transparent)] pub struct GcSubscriber { @@ -23,36 +25,24 @@ struct Inner { storage: Storage, } -impl BlockSubscriber for GcSubscriber { - type Prepared = (); - type PrepareBlockFut<'a> = futures_util::future::Ready>; - type HandleBlockFut<'a> = futures_util::future::Ready>; - type AfterBlockHandleFut<'a> = BoxFuture<'a, anyhow::Result<()>>; - - fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { - futures_util::future::ready(Ok(())) - } - - fn handle_block<'a>( - &'a self, - _cx: &'a BlockSubscriberContext, - _prepared: Self::Prepared, - ) -> Self::HandleBlockFut<'a> { - futures_util::future::ready(Ok(())) - } +impl StateSubscriber for GcSubscriber { + type HandleStateFut<'a> = BoxFuture<'a, anyhow::Result<()>>; - fn after_block_handle<'a>( - &'a self, - cx: &'a BlockSubscriberContext, - ) -> Self::AfterBlockHandleFut<'a> { + fn handle_state<'a>(&'a self, cx: &'a StateSubscriberContext) -> Self::HandleStateFut<'a> { let Ok(block_info) = cx.block.block().load_info() else { - return Box::pin(futures_util::future::ready(Ok(()))) + return Box::pin(futures_util::future::ready(Ok(()))); }; - self.inner - .storage - .node_state() - .store_shards_client_mc_block_id(&cx.mc_block_id); + if !block_info.shard.is_masterchain() { + return Box::pin(futures_util::future::ready(Ok(()))); + } + + // TODO: STORE here mc block? + + // self.inner + // .storage + // .node_state() + // .store_last_mc_block_id(&cx.mc_block_id); let enabled = self .inner @@ -76,3 +66,21 @@ impl BlockSubscriber for GcSubscriber { } } } + +impl BlockSubscriber for GcSubscriber { + type Prepared = (); + type PrepareBlockFut<'a> = futures_util::future::Ready>; + type HandleBlockFut<'a> = futures_util::future::Ready>; + + fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { + futures_util::future::ready(Ok(())) + } + + fn handle_block<'a>( + &'a self, + _cx: &'a BlockSubscriberContext, + _prepared: Self::Prepared, + ) -> Self::HandleBlockFut<'a> { + futures_util::future::ready(Ok(())) + } +} diff --git a/core/src/block_strider/subscriber/metrics_subscriber.rs b/core/src/block_strider/subscriber/metrics_subscriber.rs index 6088f02c5..4deb87ced 100644 --- a/core/src/block_strider/subscriber/metrics_subscriber.rs +++ b/core/src/block_strider/subscriber/metrics_subscriber.rs @@ -15,7 +15,6 @@ impl BlockSubscriber for MetricsSubscriber { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = futures_util::future::Ready>; - type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, _: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -31,10 +30,6 @@ impl BlockSubscriber for MetricsSubscriber { } futures_util::future::ready(Ok(())) } - - fn after_block_handle<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl StateSubscriber for MetricsSubscriber { diff --git a/core/src/block_strider/subscriber/mod.rs b/core/src/block_strider/subscriber/mod.rs index c81b338e1..b8ef96c68 100644 --- a/core/src/block_strider/subscriber/mod.rs +++ b/core/src/block_strider/subscriber/mod.rs @@ -27,7 +27,6 @@ pub trait BlockSubscriber: Send + Sync + 'static { type PrepareBlockFut<'a>: Future> + Send + 'a; type HandleBlockFut<'a>: Future> + Send + 'a; - type AfterBlockHandleFut<'a>: Future> + Send + 'a; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a>; @@ -36,11 +35,6 @@ pub trait BlockSubscriber: Send + Sync + 'static { cx: &'a BlockSubscriberContext, prepared: Self::Prepared, ) -> Self::HandleBlockFut<'a>; - - fn after_block_handle<'a>( - &'a self, - cx: &'a BlockSubscriberContext, - ) -> Self::AfterBlockHandleFut<'a>; } impl BlockSubscriber for Box { @@ -48,7 +42,6 @@ impl BlockSubscriber for Box { type PrepareBlockFut<'a> = T::PrepareBlockFut<'a>; type HandleBlockFut<'a> = T::HandleBlockFut<'a>; - type AfterBlockHandleFut<'a> = future::Ready>; #[inline] fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { @@ -63,10 +56,6 @@ impl BlockSubscriber for Box { ) -> Self::HandleBlockFut<'a> { ::handle_block(self, cx, prepared) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl BlockSubscriber for Arc { @@ -74,7 +63,6 @@ impl BlockSubscriber for Arc { type PrepareBlockFut<'a> = T::PrepareBlockFut<'a>; type HandleBlockFut<'a> = T::HandleBlockFut<'a>; - type AfterBlockHandleFut<'a> = future::Ready>; #[inline] fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { @@ -89,10 +77,6 @@ impl BlockSubscriber for Arc { ) -> Self::HandleBlockFut<'a> { ::handle_block(self, cx, prepared) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } pub trait BlockSubscriberExt: Sized { @@ -174,7 +158,6 @@ impl BlockSubscriber for NoopSubscriber { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = futures_util::future::Ready>; - type AfterBlockHandleFut<'a> = futures_util::future::Ready>; #[inline] fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'_> { @@ -189,10 +172,6 @@ impl BlockSubscriber for NoopSubscriber { ) -> Self::HandleBlockFut<'_> { futures_util::future::ready(Ok(())) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl StateSubscriber for NoopSubscriber { @@ -216,8 +195,6 @@ impl BlockSubscriber for ChainSubscrib type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; - type AfterBlockHandleFut<'a> = futures_util::future::Ready>; - fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { let left = self.left.prepare_block(cx); let right = self.right.prepare_block(cx); @@ -243,10 +220,6 @@ impl BlockSubscriber for ChainSubscrib right.await }) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl StateSubscriber for ChainSubscriber { @@ -270,7 +243,6 @@ impl BlockSubscriber for (T1, T2) { type PrepareBlockFut<'a> = BoxFuture<'a, Result>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; - type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { let left = self.0.prepare_block(cx); @@ -297,10 +269,6 @@ impl BlockSubscriber for (T1, T2) { l.and(r) }) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl StateSubscriber for (T1, T2) { @@ -329,7 +297,6 @@ pub mod test { type PrepareBlockFut<'a> = future::Ready>; type HandleBlockFut<'a> = future::Ready>; - type AfterBlockHandleFut<'a> = future::Ready>; fn prepare_block<'a>( &'a self, @@ -356,9 +323,6 @@ pub mod test { future::ready(Ok(())) } - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } impl StateSubscriber for PrintSubscriber { diff --git a/rpc/src/state/mod.rs b/rpc/src/state/mod.rs index adcb64ac3..2b2e0fc83 100644 --- a/rpc/src/state/mod.rs +++ b/rpc/src/state/mod.rs @@ -216,7 +216,6 @@ impl BlockSubscriber for RpcState { type PrepareBlockFut<'a> = futures_util::future::Ready>; type HandleBlockFut<'a> = BoxFuture<'a, Result<()>>; - type AfterBlockHandleFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, _: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -229,10 +228,6 @@ impl BlockSubscriber for RpcState { ) -> Self::HandleBlockFut<'a> { Box::pin(self.inner.update(&cx.block, None)) } - - fn after_block_handle<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::AfterBlockHandleFut<'a> { - futures_util::future::ready(Ok(())) - } } struct Inner { diff --git a/storage/src/lib.rs b/storage/src/lib.rs index e37f6b2b3..f6c5bcc7f 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -286,7 +286,7 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { return Ok(()); }; - let Some(block) = storage.node_state().load_shards_client_mc_block_id() else { + let Some(block) = storage.node_state().load_init_mc_block_id() else { return Ok(()); }; // Blocks GC will be called later when the shards client will reach the key block @@ -334,7 +334,7 @@ pub fn start_states_gc(storage: Storage) { let block_id = loop { // Load the latest block id - let block_id = match storage.node_state().load_shards_client_mc_block_id() { + let block_id = match storage.node_state().load_last_mc_block_id() { Some(block_id) => block_id, None => { tracing::error!(target: "storage", "Failed to load last shards client block. Block not found"); diff --git a/storage/src/store/node_state/mod.rs b/storage/src/store/node_state/mod.rs index 8eaa4b65e..0fb482149 100644 --- a/storage/src/store/node_state/mod.rs +++ b/storage/src/store/node_state/mod.rs @@ -8,7 +8,6 @@ pub struct NodeStateStorage { db: BaseDb, last_mc_block_id: BlockIdCache, init_mc_block_id: BlockIdCache, - shards_client_mc_block_id: BlockIdCache, } impl NodeStateStorage { @@ -17,7 +16,6 @@ impl NodeStateStorage { db, last_mc_block_id: (Default::default(), LAST_MC_BLOCK_ID), init_mc_block_id: (Default::default(), INIT_MC_BLOCK_ID), - shards_client_mc_block_id: (Default::default(), SHARDS_CLIENT_MC_BLOCK_ID) } } @@ -37,14 +35,6 @@ impl NodeStateStorage { self.load_block_id(&self.init_mc_block_id) } - pub fn store_shards_client_mc_block_id(&self, id: &BlockId) { - self.store_block_id(&self.shards_client_mc_block_id, id) - } - - pub fn load_shards_client_mc_block_id(&self) -> Option { - self.load_block_id(&self.shards_client_mc_block_id) - } - #[inline(always)] fn store_block_id(&self, (cache, key): &BlockIdCache, block_id: &BlockId) { @@ -71,5 +61,4 @@ type BlockIdCache = (Mutex>, &'static [u8]); const LAST_MC_BLOCK_ID: &[u8] = b"last_mc_block"; const INIT_MC_BLOCK_ID: &[u8] = b"init_mc_block"; -const SHARDS_CLIENT_MC_BLOCK_ID: &[u8] = b"shards_client_mc_block"; From 85f822894f3ee7230cefc25e94a2a4c85099adef Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Fri, 28 Jun 2024 17:57:26 +0200 Subject: [PATCH 08/15] fix(storage): refactoring --- core/src/block_strider/mod.rs | 2 - .../block_strider/subscriber/gc_subscriber.rs | 58 +++++++++---------- storage/src/lib.rs | 30 +++------- 3 files changed, 35 insertions(+), 55 deletions(-) diff --git a/core/src/block_strider/mod.rs b/core/src/block_strider/mod.rs index 6438ccccd..8ecd8cb2d 100644 --- a/core/src/block_strider/mod.rs +++ b/core/src/block_strider/mod.rs @@ -1,6 +1,4 @@ -use std::sync::atomic::Ordering; use std::sync::Arc; - use anyhow::Result; use everscale_types::models::{BlockId, PrevBlockRef}; use futures_util::stream::{FuturesUnordered, StreamExt}; diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index f7e8182a8..14a0adc74 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,7 +1,8 @@ -use std::sync::atomic::Ordering; use std::sync::Arc; +use everscale_types::models::BlockId; use futures_util::future::BoxFuture; +use tycho_block_util::block::BlockStuff; use tycho_storage::Storage; use crate::block_strider::{ @@ -19,36 +20,21 @@ impl GcSubscriber { inner: Arc::new(Inner { storage }), } } -} - -struct Inner { - storage: Storage, -} - -impl StateSubscriber for GcSubscriber { - type HandleStateFut<'a> = BoxFuture<'a, anyhow::Result<()>>; - fn handle_state<'a>(&'a self, cx: &'a StateSubscriberContext) -> Self::HandleStateFut<'a> { - let Ok(block_info) = cx.block.block().load_info() else { - return Box::pin(futures_util::future::ready(Ok(()))); - }; - - if !block_info.shard.is_masterchain() { + fn handle<'a>( + &'a self, + block: &'a BlockStuff, + mc_block_id: &'a BlockId + ) -> BoxFuture<'a, anyhow::Result<()>> { + if !block.id().is_masterchain() { return Box::pin(futures_util::future::ready(Ok(()))); } - // TODO: STORE here mc block? - - // self.inner - // .storage - // .node_state() - // .store_last_mc_block_id(&cx.mc_block_id); + let Ok(block_info) = block.block().load_info() else { + return Box::pin(futures_util::future::ready(Ok(()))); + }; - let enabled = self - .inner - .storage - .gc_enable_for_sync() - .load(Ordering::Acquire); + let enabled = self.inner.storage.gc_enable_for_sync(); match ( self.inner.storage.config().blocks_gc_config, @@ -57,7 +43,7 @@ impl StateSubscriber for GcSubscriber { ) { (Some(config), true, true) => { Box::pin(self.inner.storage.block_storage().remove_outdated_blocks( - &cx.mc_block_id, + &mc_block_id, config.max_blocks_per_batch, config.kind, )) @@ -67,10 +53,22 @@ impl StateSubscriber for GcSubscriber { } } +struct Inner { + storage: Storage, +} + +impl StateSubscriber for GcSubscriber { + type HandleStateFut<'a> = BoxFuture<'a, anyhow::Result<()>>; + + fn handle_state<'a>(&'a self, cx: &'a StateSubscriberContext) -> Self::HandleStateFut<'a> { + self.handle(&cx.block, &cx.mc_block_id) + } +} + impl BlockSubscriber for GcSubscriber { type Prepared = (); type PrepareBlockFut<'a> = futures_util::future::Ready>; - type HandleBlockFut<'a> = futures_util::future::Ready>; + type HandleBlockFut<'a> = BoxFuture<'a, anyhow::Result<()>>; fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -78,9 +76,9 @@ impl BlockSubscriber for GcSubscriber { fn handle_block<'a>( &'a self, - _cx: &'a BlockSubscriberContext, + cx: &'a BlockSubscriberContext, _prepared: Self::Prepared, ) -> Self::HandleBlockFut<'a> { - futures_util::future::ready(Ok(())) + self.handle(&cx.block, &cx.mc_block_id) } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index f6c5bcc7f..46a2920ea 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -144,12 +144,6 @@ impl StorageBuilder { // TODO: preload archive ids - let gc_enabled_for_sync = AtomicBool::new( - self.config - .blocks_gc_config - .map(|x| x.enable_for_sync) - .unwrap_or(false), - ); let inner = Arc::new(Inner { root, @@ -164,9 +158,6 @@ impl StorageBuilder { runtime_storage, rpc_state, internal_queue_storage, - blocks_gc_state: BlockGcState { - enabled_for_sync: gc_enabled_for_sync, - }, }); spawn_metrics_loop(&inner, Duration::from_secs(5), |this| async move { @@ -265,8 +256,13 @@ impl Storage { &self.inner.internal_queue_storage } - pub fn gc_enable_for_sync(&self) -> &AtomicBool { - &self.inner.blocks_gc_state.enabled_for_sync + pub fn gc_enable_for_sync(&self) -> bool { + self + .inner + .config + .blocks_gc_config + .map(|x| x.enable_for_sync) + .unwrap_or(false) } } @@ -276,12 +272,6 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { None => return Ok(()), }; - storage - .inner - .blocks_gc_state - .enabled_for_sync - .store(true, Ordering::Release); - let Some(key_block) = storage.block_handle_storage().find_last_key_block() else { return Ok(()); }; @@ -454,8 +444,6 @@ struct Inner { base_db: BaseDb, config: StorageConfig, - blocks_gc_state: BlockGcState, - runtime_storage: Arc, block_handle_storage: Arc, block_connection_storage: Arc, @@ -466,7 +454,3 @@ struct Inner { rpc_state: Option, internal_queue_storage: InternalQueueStorage, } - -struct BlockGcState { - enabled_for_sync: AtomicBool, -} From 84a702cb9a53ad6c59cbdaaae5a7718457d90ef9 Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Fri, 28 Jun 2024 17:57:42 +0200 Subject: [PATCH 09/15] fix(storage): fmt --- cli/src/main.rs | 4 ++-- cli/src/tools/storage_cli.rs | 15 +++++---------- core/src/block_strider/mod.rs | 1 + .../src/block_strider/subscriber/gc_subscriber.rs | 2 +- core/src/block_strider/subscriber/mod.rs | 5 ++--- storage/src/config.rs | 6 ++---- storage/src/lib.rs | 4 +--- storage/src/store/node_state/mod.rs | 2 -- .../src/store/runtime/persistent_state_keeper.rs | 2 +- 9 files changed, 15 insertions(+), 26 deletions(-) diff --git a/cli/src/main.rs b/cli/src/main.rs index fd9461cb2..24211ab29 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -3,6 +3,7 @@ use std::sync::OnceLock; use anyhow::Result; use clap::{Parser, Subcommand}; + use crate::tools::storage_cli::StorageCmd; mod tools { @@ -61,7 +62,6 @@ enum Cmd { Tool(ToolCmd), #[clap(subcommand)] Storage(StorageCmd), - } impl Cmd { @@ -69,7 +69,7 @@ impl Cmd { match self { Cmd::Node(cmd) => cmd.run(), Cmd::Tool(cmd) => cmd.run(), - Cmd::Storage(cmd) => cmd.run().await + Cmd::Storage(cmd) => cmd.run().await, } } } diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 77225da90..27ad4a378 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -3,7 +3,7 @@ use std::path::PathBuf; use anyhow::{Context, Result}; use clap::{Parser, Subcommand}; use everscale_types::models::BlockId; -use serde::{Deserialize}; +use serde::Deserialize; use tycho_storage::{BlockConnection, KeyBlocksDirection, Storage, StorageConfig}; fn init_storage(path: Option<&PathBuf>) -> Result { @@ -16,7 +16,6 @@ fn init_storage(path: Option<&PathBuf>) -> Result { .build() } - #[derive(Subcommand)] pub enum StorageCmd { GetNextKeyblockIds(GetNextKeyBlockIdsCmd), @@ -41,8 +40,7 @@ impl StorageCmd { } } -#[derive(Deserialize)] -#[derive(Parser)] +#[derive(Deserialize, Parser)] pub struct GetNextKeyBlockIdsCmd { pub block_id: BlockId, pub limit: usize, @@ -100,8 +98,7 @@ impl GetNextKeyBlockIdsCmd { } } -#[derive(Deserialize)] -#[derive(Parser)] +#[derive(Deserialize, Parser)] pub struct BlockCmd { pub block_id: BlockId, pub storage_path: Option, @@ -198,8 +195,7 @@ impl BlockCmd { } } -#[derive(Deserialize)] -#[derive(Parser)] +#[derive(Deserialize, Parser)] pub struct GetArchiveInfoCmd { pub mc_seqno: u32, pub storage_path: Option, @@ -233,8 +229,7 @@ impl GetArchiveInfoCmd { } } -#[derive(Deserialize)] -#[derive(Parser)] +#[derive(Deserialize, Parser)] pub struct GetArchiveSliceCmd { pub archive_id: u64, pub limit: u32, diff --git a/core/src/block_strider/mod.rs b/core/src/block_strider/mod.rs index 8ecd8cb2d..a32f72b3c 100644 --- a/core/src/block_strider/mod.rs +++ b/core/src/block_strider/mod.rs @@ -1,4 +1,5 @@ use std::sync::Arc; + use anyhow::Result; use everscale_types::models::{BlockId, PrevBlockRef}; use futures_util::stream::{FuturesUnordered, StreamExt}; diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 14a0adc74..7551e03a9 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -24,7 +24,7 @@ impl GcSubscriber { fn handle<'a>( &'a self, block: &'a BlockStuff, - mc_block_id: &'a BlockId + mc_block_id: &'a BlockId, ) -> BoxFuture<'a, anyhow::Result<()>> { if !block.id().is_masterchain() { return Box::pin(futures_util::future::ready(Ok(()))); diff --git a/core/src/block_strider/subscriber/mod.rs b/core/src/block_strider/subscriber/mod.rs index b8ef96c68..eb2b18edd 100644 --- a/core/src/block_strider/subscriber/mod.rs +++ b/core/src/block_strider/subscriber/mod.rs @@ -8,11 +8,11 @@ use tycho_block_util::archive::ArchiveData; use tycho_block_util::block::BlockStuff; use tycho_block_util::state::ShardStateStuff; -pub use self::metrics_subscriber::MetricsSubscriber; pub use self::gc_subscriber::GcSubscriber; +pub use self::metrics_subscriber::MetricsSubscriber; -mod metrics_subscriber; mod gc_subscriber; +mod metrics_subscriber; // === trait BlockSubscriber === @@ -322,7 +322,6 @@ pub mod test { ); future::ready(Ok(())) } - } impl StateSubscriber for PrintSubscriber { diff --git a/storage/src/config.rs b/storage/src/config.rs index e296b1e6b..5e4d5cb52 100644 --- a/storage/src/config.rs +++ b/storage/src/config.rs @@ -45,8 +45,6 @@ pub struct StorageConfig { /// /// Blocks GC is disabled if this field is `None`. pub blocks_gc_config: Option, - - } impl StorageConfig { @@ -59,7 +57,7 @@ impl StorageConfig { rocksdb_enable_metrics: false, archives: Some(ArchivesConfig::default()), states_gc_options: Some(StateGcOptions::default()), - blocks_gc_config: Some(BlocksGcOptions::default()) + blocks_gc_config: Some(BlocksGcOptions::default()), } } } @@ -110,7 +108,7 @@ impl Default for StorageConfig { rocksdb_enable_metrics: true, archives: Some(ArchivesConfig::default()), states_gc_options: Some(StateGcOptions::default()), - blocks_gc_config: Some(BlocksGcOptions::default()) + blocks_gc_config: Some(BlocksGcOptions::default()), } } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 46a2920ea..1d54a061b 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -144,7 +144,6 @@ impl StorageBuilder { // TODO: preload archive ids - let inner = Arc::new(Inner { root, base_db, @@ -257,8 +256,7 @@ impl Storage { } pub fn gc_enable_for_sync(&self) -> bool { - self - .inner + self.inner .config .blocks_gc_config .map(|x| x.enable_for_sync) diff --git a/storage/src/store/node_state/mod.rs b/storage/src/store/node_state/mod.rs index 0fb482149..a067d0f65 100644 --- a/storage/src/store/node_state/mod.rs +++ b/storage/src/store/node_state/mod.rs @@ -35,7 +35,6 @@ impl NodeStateStorage { self.load_block_id(&self.init_mc_block_id) } - #[inline(always)] fn store_block_id(&self, (cache, key): &BlockIdCache, block_id: &BlockId) { let node_states = &self.db.state; @@ -61,4 +60,3 @@ type BlockIdCache = (Mutex>, &'static [u8]); const LAST_MC_BLOCK_ID: &[u8] = b"last_mc_block"; const INIT_MC_BLOCK_ID: &[u8] = b"init_mc_block"; - diff --git a/storage/src/store/runtime/persistent_state_keeper.rs b/storage/src/store/runtime/persistent_state_keeper.rs index 2dee0b4e8..f9cf8f635 100644 --- a/storage/src/store/runtime/persistent_state_keeper.rs +++ b/storage/src/store/runtime/persistent_state_keeper.rs @@ -3,7 +3,7 @@ use std::sync::Arc; use anyhow::Result; use arc_swap::ArcSwapAny; -use tokio::sync::{Notify, watch}; +use tokio::sync::{watch, Notify}; use tycho_block_util::state::*; use crate::models::{BlockHandle, BriefBlockMeta}; From 8793d64a192a518f473fa790ab8d66466ca756fb Mon Sep 17 00:00:00 2001 From: MrWad3r Date: Mon, 1 Jul 2024 18:03:33 +0200 Subject: [PATCH 10/15] feat(storage): use watch instead of direct gc call in subscription --- cli/src/node/mod.rs | 6 +- .../block_strider/subscriber/gc_subscriber.rs | 215 +++++++++++++++--- storage/src/lib.rs | 154 +------------ 3 files changed, 189 insertions(+), 186 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index c77c56d3e..2fe4007aa 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -41,7 +41,7 @@ use tycho_network::{ }; use tycho_rpc::{RpcConfig, RpcState}; use tycho_storage::{ - prepare_blocks_gc, start_archives_gc, start_states_gc, BlockMetaData, Storage, + prepare_blocks_gc, BlockMetaData, Storage, }; use tycho_util::FastHashMap; @@ -685,10 +685,6 @@ impl Node { )) .build(); - if let Err(e) = start_archives_gc(self.storage.clone()) { - tracing::error!("Failed to execute archives gc. {e:?}"); - }; - start_states_gc(self.storage.clone()); prepare_blocks_gc(self.storage.clone()).await?; // Run block strider diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 7551e03a9..2ef275d40 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,9 +1,16 @@ +use std::ops::Add; use std::sync::Arc; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::time::Instant; use everscale_types::models::BlockId; use futures_util::future::BoxFuture; +use tokio::sync::Notify; use tycho_block_util::block::BlockStuff; -use tycho_storage::Storage; +use tycho_storage::{ArchivesGcInterval, Storage}; +use tycho_util::futures::JoinTask; +use tycho_util::metrics::HistogramGuard; +use tycho_util::time::duration_between_unix_and_instant; use crate::block_strider::{ BlockSubscriber, BlockSubscriberContext, StateSubscriber, StateSubscriberContext, @@ -16,59 +23,208 @@ pub struct GcSubscriber { impl GcSubscriber { pub fn new(storage: Storage) -> Self { + let (block_sender, mut block_receiver) = + tokio::sync::watch::channel::>(None); + let (state_sender, mut state_receiver) = + tokio::sync::watch::channel::>(None); + + + tokio::spawn(Self::handle_block_gc(block_receiver, storage.clone())); + tokio::spawn(Self::handle_state_gc(state_receiver, storage.clone())); + tokio::spawn(Self::handle_archives_gc(storage.clone())); + Self { - inner: Arc::new(Inner { storage }), + inner: Arc::new(Inner { + storage, + block_sender, + state_sender, + }), } } - fn handle<'a>( - &'a self, - block: &'a BlockStuff, - mc_block_id: &'a BlockId, - ) -> BoxFuture<'a, anyhow::Result<()>> { - if !block.id().is_masterchain() { - return Box::pin(futures_util::future::ready(Ok(()))); + pub fn handle(&self, block_stuff: BlockStuff) { + if block_stuff.id().is_masterchain() { + return (); } + let block = match block_stuff.load_info() { + Ok(block) => block, + Err(e) => { + tracing::error!("Failed to load block info: {:?} {e:?}", block_stuff.id()); + return (); + } + }; - let Ok(block_info) = block.block().load_info() else { - return Box::pin(futures_util::future::ready(Ok(()))); + if block.key_block { + if let Err(e) = self.inner.block_sender.send(Some(block_stuff.clone())) { + tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); + } + if let Err(e) = self.inner.state_sender.send(Some(block_stuff.clone())) { + tracing::error!("Failed to execute handle_state for state_sender. {e:?} "); + } + } + } + + async fn handle_archives_gc( + storage: Storage, + ) { + let options = match &storage.config().archives { + Some(options) => options, + None => return, }; - let enabled = self.inner.storage.gc_enable_for_sync(); - - match ( - self.inner.storage.config().blocks_gc_config, - enabled, - block_info.key_block, - ) { - (Some(config), true, true) => { - Box::pin(self.inner.storage.block_storage().remove_outdated_blocks( - &mc_block_id, - config.max_blocks_per_batch, - config.kind, - )) + struct LowerBound { + archive_id: AtomicU32, + changed: Notify, + } + + #[allow(unused_mut)] + let mut lower_bound = None::>; + + match options.gc_interval { + ArchivesGcInterval::Manual => return, + ArchivesGcInterval::PersistentStates { offset } => { + + tokio::spawn(async move { + let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); + + loop { + tokio::pin!(let new_state_found = persistent_state_keeper.new_state_found();); + + let (until_id, untile_time) = match persistent_state_keeper.current() { + Some(state) => { + let untile_time = + (state.meta().gen_utime() as u64).add(offset.as_secs()); + (state.id().seqno, untile_time) + } + None => { + new_state_found.await; + continue; + } + }; + + tokio::select!( + _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, + _ = &mut new_state_found => continue, + ); + + if let Some(lower_bound) = &lower_bound { + loop { + tokio::pin!(let lower_bound_changed = lower_bound.changed.notified();); + + let lower_bound = lower_bound.archive_id.load(Ordering::Acquire); + if until_id < lower_bound { + break; + } + + tracing::info!( + until_id, + lower_bound, + "waiting for the archives barrier" + ); + lower_bound_changed.await; + } + } + + if let Err(e) = storage + .block_storage() + .remove_outdated_archives(until_id) + .await + { + tracing::error!("failed to remove outdated archives: {e:?}"); + } + + new_state_found.await; + } + }); } - _ => Box::pin(futures_util::future::ready(Ok(()))), + } + } + + async fn handle_block_gc( + mut block_receiver: tokio::sync::watch::Receiver>, + storage: Storage, + ) { + let _ = JoinTask::new(async move { + loop { + if let Err(e) = block_receiver.changed().await { + tracing::error!("Failed to receive block from block_receiver. {e:?}"); + continue; + } + + let block = block_receiver.borrow_and_update().clone(); + + match ( + block, + storage.config().blocks_gc_config, + storage.gc_enable_for_sync(), + ) { + (Some(block_stuff), Some(config), true) => { + if let Err(e) = storage + .block_storage() + .remove_outdated_blocks( + &block_stuff.id(), + config.max_blocks_per_batch, + config.kind, + ) + .await { + tracing::error!("Failed to remove_outdated_blocks. {e:?}") + } + } + _ => continue, + } + } + }); + } + + async fn handle_state_gc( + mut state_receiver: tokio::sync::watch::Receiver>, + storage: Storage, + ) { + + loop { + if let Err(e) = state_receiver.changed().await { + tracing::error!("Failed to receive block from block_receiver. {e:?}"); + continue; + } + + let Some(block) = state_receiver.borrow_and_update().clone() else { + continue; + }; + + let shard_state_storage = storage.shard_state_storage(); + + match shard_state_storage + .remove_outdated_states(block.id().seqno) + .await + { + Ok(_) => (), + Err(e) => { + tracing::error!(target: "storage", "Failed to GC state: {e:?}"); + } + }; } } } struct Inner { storage: Storage, + block_sender: tokio::sync::watch::Sender>, + state_sender: tokio::sync::watch::Sender>, } impl StateSubscriber for GcSubscriber { - type HandleStateFut<'a> = BoxFuture<'a, anyhow::Result<()>>; + type HandleStateFut<'a> = futures_util::future::Ready>; fn handle_state<'a>(&'a self, cx: &'a StateSubscriberContext) -> Self::HandleStateFut<'a> { - self.handle(&cx.block, &cx.mc_block_id) + self.handle(cx.block.clone()); + futures_util::future::ready(Ok(())) } } impl BlockSubscriber for GcSubscriber { type Prepared = (); type PrepareBlockFut<'a> = futures_util::future::Ready>; - type HandleBlockFut<'a> = BoxFuture<'a, anyhow::Result<()>>; + type HandleBlockFut<'a> = futures_util::future::Ready>; fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) @@ -79,6 +235,7 @@ impl BlockSubscriber for GcSubscriber { cx: &'a BlockSubscriberContext, _prepared: Self::Prepared, ) -> Self::HandleBlockFut<'a> { - self.handle(&cx.block, &cx.mc_block_id) + self.handle(cx.block.clone()); + futures_util::future::ready(Ok(())) } } diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 1d54a061b..ff103789e 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,12 +1,8 @@ -use std::ops::Add; -use std::sync::atomic::{AtomicBool, AtomicU32, Ordering}; use std::sync::Arc; -use std::time::{Duration, Instant}; +use std::time::{Duration}; use anyhow::Result; -use tokio::sync::Notify; -use tycho_util::metrics::{spawn_metrics_loop, HistogramGuard}; -use tycho_util::time::now_sec; +use tycho_util::metrics::{spawn_metrics_loop}; use weedb::rocksdb; pub use self::config::*; @@ -29,7 +25,6 @@ mod util { mod stored_value; } -use tycho_util::time::duration_between_unix_and_instant; // TODO move to weedb pub use util::owned_iterator; @@ -292,151 +287,6 @@ pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { .await } -pub fn start_states_gc(storage: Storage) { - let options = match storage.inner.config.states_gc_options { - Some(options) => options, - None => return, - }; - - // Compute gc timestamp aligned to `interval_sec` with an offset `offset_sec` - let mut gc_at = now_sec() as u64; - gc_at = (gc_at - gc_at % options.interval_sec) + options.offset_sec; - - tokio::spawn(async move { - 'gc: loop { - // Shift gc timestamp one iteration further - gc_at += options.interval_sec; - - // Check if there is some time left before the GC - if let Some(interval) = gc_at.checked_sub(now_sec() as u64) { - tokio::time::sleep(Duration::from_secs(interval)).await; - } - - let start = Instant::now(); - let mut shards_gc_lock = storage - .runtime_storage() - .persistent_state_keeper() - .shards_gc_lock() - .subscribe(); - metrics::histogram!("tycho_storage_shard_states_gc_lock_time").record(start.elapsed()); - - let block_id = loop { - // Load the latest block id - let block_id = match storage.node_state().load_last_mc_block_id() { - Some(block_id) => block_id, - None => { - tracing::error!(target: "storage", "Failed to load last shards client block. Block not found"); - continue 'gc; - } - }; - - if *shards_gc_lock.borrow_and_update() { - if shards_gc_lock.changed().await.is_err() { - tracing::warn!(target: "storage", "Stopping shard states GC"); - return; - } - continue; - } - - break block_id; - }; - - // subscriber.on_before_states_gc(&block_id).await; - - let _histogram = HistogramGuard::begin("tycho_storage_remove_outdated_states_time"); - let shard_state_storage = storage.shard_state_storage(); - let _ = match shard_state_storage - .remove_outdated_states(block_id.seqno) - .await - { - Ok(top_blocks) => Some(top_blocks), - Err(e) => { - tracing::error!(target: "storage", "Failed to GC state: {e:?}"); - None - } - }; - - // subscriber.on_after_states_gc(&block_id, &top_blocks).await; - } - }); -} - -pub fn start_archives_gc(storage: Storage) -> Result<()> { - let options = match &storage.inner.config.archives { - Some(options) => options, - None => return Ok(()), - }; - - struct LowerBound { - archive_id: AtomicU32, - changed: Notify, - } - - #[allow(unused_mut)] - let mut lower_bound = None::>; - - match options.gc_interval { - ArchivesGcInterval::Manual => Ok(()), - ArchivesGcInterval::PersistentStates { offset } => { - // let engine = self.clone(); - tokio::spawn(async move { - let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); - - loop { - tokio::pin!(let new_state_found = persistent_state_keeper.new_state_found();); - - let (until_id, untile_time) = match persistent_state_keeper.current() { - Some(state) => { - let untile_time = - (state.meta().gen_utime() as u64).add(offset.as_secs()); - (state.id().seqno, untile_time) - } - None => { - new_state_found.await; - continue; - } - }; - - tokio::select!( - _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, - _ = &mut new_state_found => continue, - ); - - if let Some(lower_bound) = &lower_bound { - loop { - tokio::pin!(let lower_bound_changed = lower_bound.changed.notified();); - - let lower_bound = lower_bound.archive_id.load(Ordering::Acquire); - if until_id < lower_bound { - break; - } - - tracing::info!( - until_id, - lower_bound, - "waiting for the archives barrier" - ); - lower_bound_changed.await; - } - } - - if let Err(e) = storage - .block_storage() - .remove_outdated_archives(until_id) - .await - { - tracing::error!("failed to remove outdated archives: {e:?}"); - } - - new_state_found.await; - } - }); - - Ok(()) - } - } -} - struct Inner { root: FileDb, base_db: BaseDb, From 825fee910174ac34b0423993e184ce275cfe5a7e Mon Sep 17 00:00:00 2001 From: MrWad3r Date: Mon, 1 Jul 2024 18:03:56 +0200 Subject: [PATCH 11/15] fix(storage): fmt --- cli/src/node/mod.rs | 4 +-- .../block_strider/subscriber/gc_subscriber.rs | 29 +++++++++---------- storage/src/lib.rs | 4 +-- 3 files changed, 16 insertions(+), 21 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index 2fe4007aa..763c429c0 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -40,9 +40,7 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{ - prepare_blocks_gc, BlockMetaData, Storage, -}; +use tycho_storage::{prepare_blocks_gc, BlockMetaData, Storage}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 2ef275d40..b00e01a20 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,6 +1,6 @@ use std::ops::Add; -use std::sync::Arc; use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; use std::time::Instant; use everscale_types::models::BlockId; @@ -28,7 +28,6 @@ impl GcSubscriber { let (state_sender, mut state_receiver) = tokio::sync::watch::channel::>(None); - tokio::spawn(Self::handle_block_gc(block_receiver, storage.clone())); tokio::spawn(Self::handle_state_gc(state_receiver, storage.clone())); tokio::spawn(Self::handle_archives_gc(storage.clone())); @@ -64,9 +63,7 @@ impl GcSubscriber { } } - async fn handle_archives_gc( - storage: Storage, - ) { + async fn handle_archives_gc(storage: Storage) { let options = match &storage.config().archives { Some(options) => options, None => return, @@ -83,9 +80,9 @@ impl GcSubscriber { match options.gc_interval { ArchivesGcInterval::Manual => return, ArchivesGcInterval::PersistentStates { offset } => { - tokio::spawn(async move { - let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); + let persistent_state_keeper = + storage.runtime_storage().persistent_state_keeper(); loop { tokio::pin!(let new_state_found = persistent_state_keeper.new_state_found();); @@ -103,9 +100,9 @@ impl GcSubscriber { }; tokio::select!( - _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, - _ = &mut new_state_found => continue, - ); + _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, + _ = &mut new_state_found => continue, + ); if let Some(lower_bound) = &lower_bound { loop { @@ -117,10 +114,10 @@ impl GcSubscriber { } tracing::info!( - until_id, - lower_bound, - "waiting for the archives barrier" - ); + until_id, + lower_bound, + "waiting for the archives barrier" + ); lower_bound_changed.await; } } @@ -166,7 +163,8 @@ impl GcSubscriber { config.max_blocks_per_batch, config.kind, ) - .await { + .await + { tracing::error!("Failed to remove_outdated_blocks. {e:?}") } } @@ -180,7 +178,6 @@ impl GcSubscriber { mut state_receiver: tokio::sync::watch::Receiver>, storage: Storage, ) { - loop { if let Err(e) = state_receiver.changed().await { tracing::error!("Failed to receive block from block_receiver. {e:?}"); diff --git a/storage/src/lib.rs b/storage/src/lib.rs index ff103789e..22a117c91 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -1,8 +1,8 @@ use std::sync::Arc; -use std::time::{Duration}; +use std::time::Duration; use anyhow::Result; -use tycho_util::metrics::{spawn_metrics_loop}; +use tycho_util::metrics::spawn_metrics_loop; use weedb::rocksdb; pub use self::config::*; From c3dd45259d9ba1b16d7ca627f63d9cd6e0c112f6 Mon Sep 17 00:00:00 2001 From: MrWad3r Date: Mon, 1 Jul 2024 18:51:18 +0200 Subject: [PATCH 12/15] feat(storage): implement Drop for inner and gc tasks --- cli/src/tools/storage_cli.rs | 4 +- .../block_strider/subscriber/gc_subscriber.rs | 133 +++++++++++------- storage/src/lib.rs | 3 +- 3 files changed, 84 insertions(+), 56 deletions(-) diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 27ad4a378..4c260430b 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -86,7 +86,7 @@ impl GetNextKeyBlockIdsCmd { ); } for i in ids.iter() { - println!("Found block {i}") + println!("Found block {i}"); } Ok(()) } @@ -125,7 +125,7 @@ impl BlockCmd { println!("Block proof {}\n", hex::encode(&proof)); } _ => { - println!("Found block empty {}\n", &self.block_id) + println!("Found block empty {}\n", &self.block_id); } }; Ok::<(), anyhow::Error>(()) diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index b00e01a20..94369ad78 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,3 +1,4 @@ +use std::arch::x86_64::_mm256_insert_epi16; use std::ops::Add; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; @@ -6,6 +7,7 @@ use std::time::Instant; use everscale_types::models::BlockId; use futures_util::future::BoxFuture; use tokio::sync::Notify; +use tokio::task::{AbortHandle, JoinHandle}; use tycho_block_util::block::BlockStuff; use tycho_storage::{ArchivesGcInterval, Storage}; use tycho_util::futures::JoinTask; @@ -28,16 +30,26 @@ impl GcSubscriber { let (state_sender, mut state_receiver) = tokio::sync::watch::channel::>(None); - tokio::spawn(Self::handle_block_gc(block_receiver, storage.clone())); - tokio::spawn(Self::handle_state_gc(state_receiver, storage.clone())); - tokio::spawn(Self::handle_archives_gc(storage.clone())); + let mut inner = Inner { + storage: storage.clone(), + block_sender, + state_sender, + handle_block_task: None, + handle_state_task: None, + archives_handle: None, + }; + + let hb_handle = Self::handle_block_gc(block_receiver, storage.clone()).abort_handle(); + let hs_handle = Self::handle_state_gc(state_receiver, storage.clone()).abort_handle(); + let archives_handle = + tokio::spawn(Self::handle_archives_gc(storage.clone())).abort_handle(); + + inner.handle_block_task = Some(hb_handle); + inner.handle_state_task = Some(hs_handle); + inner.archives_handle = Some(archives_handle); Self { - inner: Arc::new(Inner { - storage, - block_sender, - state_sender, - }), + inner: Arc::new(inner), } } @@ -137,11 +149,11 @@ impl GcSubscriber { } } - async fn handle_block_gc( + fn handle_block_gc( mut block_receiver: tokio::sync::watch::Receiver>, storage: Storage, - ) { - let _ = JoinTask::new(async move { + ) -> JoinHandle<()> { + tokio::spawn(async move { loop { if let Err(e) = block_receiver.changed().await { tracing::error!("Failed to receive block from block_receiver. {e:?}"); @@ -150,56 +162,53 @@ impl GcSubscriber { let block = block_receiver.borrow_and_update().clone(); - match ( - block, - storage.config().blocks_gc_config, - storage.gc_enable_for_sync(), - ) { - (Some(block_stuff), Some(config), true) => { - if let Err(e) = storage - .block_storage() - .remove_outdated_blocks( - &block_stuff.id(), - config.max_blocks_per_batch, - config.kind, - ) - .await - { - tracing::error!("Failed to remove_outdated_blocks. {e:?}") - } - } - _ => continue, + if !storage.gc_enable_for_sync() { + continue; + } + + let (Some(bs), Some(config)) = (block, storage.config().blocks_gc_config) else { + continue; + }; + + if let Err(e) = storage + .block_storage() + .remove_outdated_blocks(&bs.id(), config.max_blocks_per_batch, config.kind) + .await + { + tracing::error!("Failed to remove_outdated_blocks. {e:?}") } } - }); + }) } - async fn handle_state_gc( + fn handle_state_gc( mut state_receiver: tokio::sync::watch::Receiver>, storage: Storage, - ) { - loop { - if let Err(e) = state_receiver.changed().await { - tracing::error!("Failed to receive block from block_receiver. {e:?}"); - continue; - } + ) -> JoinHandle<()> { + tokio::spawn(async move { + loop { + if let Err(e) = state_receiver.changed().await { + tracing::error!("Failed to receive block from block_receiver. {e:?}"); + continue; + } - let Some(block) = state_receiver.borrow_and_update().clone() else { - continue; - }; + let Some(block) = state_receiver.borrow_and_update().clone() else { + continue; + }; - let shard_state_storage = storage.shard_state_storage(); + let shard_state_storage = storage.shard_state_storage(); - match shard_state_storage - .remove_outdated_states(block.id().seqno) - .await - { - Ok(_) => (), - Err(e) => { - tracing::error!(target: "storage", "Failed to GC state: {e:?}"); - } - }; - } + match shard_state_storage + .remove_outdated_states(block.id().seqno) + .await + { + Ok(_) => (), + Err(e) => { + tracing::error!(target: "storage", "Failed to GC state: {e:?}"); + } + }; + } + }) } } @@ -207,6 +216,26 @@ struct Inner { storage: Storage, block_sender: tokio::sync::watch::Sender>, state_sender: tokio::sync::watch::Sender>, + + handle_block_task: Option, + handle_state_task: Option, + archives_handle: Option, +} + +impl Drop for Inner { + fn drop(&mut self) { + if let Some(handle) = self.handle_block_task.take() { + handle.abort(); + } + + if let Some(handle) = self.handle_state_task.take() { + handle.abort(); + } + + if let Some(handle) = self.archives_handle.take() { + handle.abort(); + } + } } impl StateSubscriber for GcSubscriber { diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 22a117c91..3f45629f9 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -254,8 +254,7 @@ impl Storage { self.inner .config .blocks_gc_config - .map(|x| x.enable_for_sync) - .unwrap_or(false) + .is_some_and(|x| x.enable_for_sync) } } From 6f1e9d5c0e21ad1d7b7367465df5a62f24a6f992 Mon Sep 17 00:00:00 2001 From: MrWad3r Date: Wed, 3 Jul 2024 09:31:05 +0200 Subject: [PATCH 13/15] feat(storage): gc improvements wip --- .../block_strider/subscriber/gc_subscriber.rs | 24 +++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 94369ad78..6b89223e6 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -54,7 +54,7 @@ impl GcSubscriber { } pub fn handle(&self, block_stuff: BlockStuff) { - if block_stuff.id().is_masterchain() { + if !block_stuff.id().is_masterchain() { return (); } let block = match block_stuff.load_info() { @@ -65,13 +65,14 @@ impl GcSubscriber { } }; + if let Err(e) = self.inner.state_sender.send(Some(block_stuff.clone())) { + tracing::error!("Failed to execute handle_state for state_sender. {e:?} "); + } + if block.key_block { if let Err(e) = self.inner.block_sender.send(Some(block_stuff.clone())) { tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); } - if let Err(e) = self.inner.state_sender.send(Some(block_stuff.clone())) { - tracing::error!("Failed to execute handle_state for state_sender. {e:?} "); - } } } @@ -160,16 +161,27 @@ impl GcSubscriber { continue; } + tracing::info!("Block GC executed..."); + let block = block_receiver.borrow_and_update().clone(); if !storage.gc_enable_for_sync() { + tracing::debug!("Block GC is not enabled for sync."); continue; } let (Some(bs), Some(config)) = (block, storage.config().blocks_gc_config) else { + tracing::debug!("Block GC is disabled by config or boundary block not found"); continue; }; + tracing::debug!( + "Removing outdated blocks with boundary block {}, batch of {:?} and kind {:?}", + bs.id(), + config.max_blocks_per_batch, + config.kind + ); + if let Err(e) = storage .block_storage() .remove_outdated_blocks(&bs.id(), config.max_blocks_per_batch, config.kind) @@ -192,9 +204,13 @@ impl GcSubscriber { continue; } + tracing::info!("State GC executed..."); + let Some(block) = state_receiver.borrow_and_update().clone() else { + tracing::info!("Boundary GC block is not found"); continue; }; + tracing::info!("GC state is executing for block: {:?}", block.id()); let shard_state_storage = storage.shard_state_storage(); From 5710d4901c5c4146eb11414ade753411a23a72ae Mon Sep 17 00:00:00 2001 From: Stanislav Eliseev Date: Wed, 3 Jul 2024 16:49:29 +0200 Subject: [PATCH 14/15] feat(storage): implement gc for case when there is no key blocks --- cli/src/node/mod.rs | 4 +- collator/src/state_node.rs | 10 +- .../block_strider/subscriber/gc_subscriber.rs | 118 ++++++++++++++---- storage/src/lib.rs | 28 ----- storage/src/store/block/mod.rs | 41 +++++- storage/src/store/block_handle/mod.rs | 12 ++ storage/src/store/runtime/mod.rs | 16 +++ 7 files changed, 165 insertions(+), 64 deletions(-) diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index 763c429c0..7407c3ece 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -40,7 +40,7 @@ use tycho_network::{ PublicOverlay, Router, }; use tycho_rpc::{RpcConfig, RpcState}; -use tycho_storage::{prepare_blocks_gc, BlockMetaData, Storage}; +use tycho_storage::{BlockMetaData, Storage}; use tycho_util::FastHashMap; use self::config::{MetricsConfig, NodeConfig, NodeKeys}; @@ -683,8 +683,6 @@ impl Node { )) .build(); - prepare_blocks_gc(self.storage.clone()).await?; - // Run block strider tracing::info!("block strider started"); block_strider.run().await?; diff --git a/collator/src/state_node.rs b/collator/src/state_node.rs index 7ff55667c..9392c9ff1 100644 --- a/collator/src/state_node.rs +++ b/collator/src/state_node.rs @@ -255,12 +255,10 @@ impl StateNodeAdapterStdImpl { loop { if let Some(shard_blocks) = self.blocks.get(&block_id.shard()) { if let Some(block) = shard_blocks.get(&block_id.seqno()) { - match self.save_block_proof(block).await { - Ok(_) => return Some(Ok(block.block_stuff_aug.clone())), - Err(e) => { - return Some(Err(anyhow!("failed to save block proof: {e:?}"))); - } - } + return match self.save_block_proof(block).await { + Ok(_) => Some(Ok(block.block_stuff_aug.clone())), + Err(e) => Some(Err(anyhow!("failed to save block proof: {e:?}"))), + }; } } diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index 6b89223e6..fbeae51d5 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,32 +1,43 @@ -use std::arch::x86_64::_mm256_insert_epi16; use std::ops::Add; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; use std::time::Instant; -use everscale_types::models::BlockId; -use futures_util::future::BoxFuture; use tokio::sync::Notify; use tokio::task::{AbortHandle, JoinHandle}; use tycho_block_util::block::BlockStuff; use tycho_storage::{ArchivesGcInterval, Storage}; -use tycho_util::futures::JoinTask; -use tycho_util::metrics::HistogramGuard; use tycho_util::time::duration_between_unix_and_instant; use crate::block_strider::{ BlockSubscriber, BlockSubscriberContext, StateSubscriber, StateSubscriberContext, }; +const GC_MC_BLOCK_INTERVAL: u32 = 10000; + #[repr(transparent)] pub struct GcSubscriber { inner: Arc, } +#[derive(Clone)] +enum BlockGcTrigger { + Common(BlockStuff), + Interval(BlockStuff), +} + +impl BlockGcTrigger { + fn get_block_seqno(&self) -> u32 { + match self { + BlockGcTrigger::Common(stuff) | BlockGcTrigger::Interval(stuff) => stuff.id().seqno, + } + } +} + impl GcSubscriber { pub fn new(storage: Storage) -> Self { let (block_sender, mut block_receiver) = - tokio::sync::watch::channel::>(None); + tokio::sync::watch::channel::>(None); let (state_sender, mut state_receiver) = tokio::sync::watch::channel::>(None); @@ -55,13 +66,13 @@ impl GcSubscriber { pub fn handle(&self, block_stuff: BlockStuff) { if !block_stuff.id().is_masterchain() { - return (); + return; } let block = match block_stuff.load_info() { Ok(block) => block, Err(e) => { tracing::error!("Failed to load block info: {:?} {e:?}", block_stuff.id()); - return (); + return; } }; @@ -70,9 +81,43 @@ impl GcSubscriber { } if block.key_block { - if let Err(e) = self.inner.block_sender.send(Some(block_stuff.clone())) { + if let Err(e) = self + .inner + .block_sender + .send(Some(BlockGcTrigger::Common(block_stuff.clone()))) + { tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); } + return; + } + + let last = self + .inner + .storage + .runtime_storage() + .get_last_block_gc_collection(); + match last { + Some(last_mc_seqno) if last_mc_seqno < block.seqno - GC_MC_BLOCK_INTERVAL => { + if let Err(e) = self + .inner + .block_sender + .send(Some(BlockGcTrigger::Interval(block_stuff.clone()))) + { + tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); + } + } + None => { + if let Err(e) = self + .inner + .block_sender + .send(Some(BlockGcTrigger::Interval(block_stuff.clone()))) + { + tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); + } + } + _ => { + tracing::debug!("Skipping boundary GC. Block are too fresh"); + } } } @@ -151,7 +196,7 @@ impl GcSubscriber { } fn handle_block_gc( - mut block_receiver: tokio::sync::watch::Receiver>, + mut block_receiver: tokio::sync::watch::Receiver>, storage: Storage, ) -> JoinHandle<()> { tokio::spawn(async move { @@ -175,20 +220,47 @@ impl GcSubscriber { continue; }; - tracing::debug!( - "Removing outdated blocks with boundary block {}, batch of {:?} and kind {:?}", - bs.id(), - config.max_blocks_per_batch, - config.kind - ); + let cleaning_seqno = bs.get_block_seqno(); - if let Err(e) = storage - .block_storage() - .remove_outdated_blocks(&bs.id(), config.max_blocks_per_batch, config.kind) - .await - { - tracing::error!("Failed to remove_outdated_blocks. {e:?}") + match bs { + BlockGcTrigger::Common(bs) => { + tracing::debug!( + "Removing outdated blocks for block {}, batch of {:?} and kind {:?}", + bs.id(), + config.max_blocks_per_batch, + config.kind + ); + if let Err(e) = storage + .block_storage() + .remove_outdated_blocks( + bs.id(), + config.max_blocks_per_batch, + config.kind, + ) + .await + { + tracing::error!("Failed to remove_outdated_blocks. {e:?}"); + } + } + BlockGcTrigger::Interval(bs) => { + let Some(handle) = storage.block_handle_storage().load_handle(bs.id()) + else { + return; + }; + + if let Err(e) = storage + .block_storage() + .remove_boundary_blocks(handle, config.max_blocks_per_batch) + .await + { + tracing::error!("Failed to remove_boundary_blocks. {e:?}"); + } + } } + + storage + .runtime_storage() + .set_last_block_gc_collection(cleaning_seqno); } }) } @@ -230,7 +302,7 @@ impl GcSubscriber { struct Inner { storage: Storage, - block_sender: tokio::sync::watch::Sender>, + block_sender: tokio::sync::watch::Sender>, state_sender: tokio::sync::watch::Sender>, handle_block_task: Option, diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 3f45629f9..463c0bbcd 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -258,34 +258,6 @@ impl Storage { } } -pub async fn prepare_blocks_gc(storage: Storage) -> Result<()> { - let blocks_gc_config = match &storage.inner.config.blocks_gc_config { - Some(state) => state, - None => return Ok(()), - }; - - let Some(key_block) = storage.block_handle_storage().find_last_key_block() else { - return Ok(()); - }; - - let Some(block) = storage.node_state().load_init_mc_block_id() else { - return Ok(()); - }; - // Blocks GC will be called later when the shards client will reach the key block - if block.seqno < key_block.id().seqno { - return Ok(()); - } - - storage - .block_storage() - .remove_outdated_blocks( - key_block.id(), - blocks_gc_config.max_blocks_per_batch, - blocks_gc_config.kind, - ) - .await -} - struct Inner { root: FileDb, base_db: BaseDb, diff --git a/storage/src/store/block/mod.rs b/storage/src/store/block/mod.rs index 6ddbcb62e..ee323ae9a 100644 --- a/storage/src/store/block/mod.rs +++ b/storage/src/store/block/mod.rs @@ -531,6 +531,28 @@ impl BlockStorage { } } + pub async fn remove_boundary_blocks( + &self, + mc_block_handle: BlockHandle, + max_blocks_per_batch: Option, + ) -> Result<()> { + tracing::info!( + target_block_id = %mc_block_handle.id(), + "starting blocks GC", + ); + + let top_blocks = self + .load_block_data(&mc_block_handle) + .await + .context("Failed to load target key block data") + .and_then(|block_data| TopBlocks::from_mc_block(&block_data)) + .context("Failed to compute top blocks for target block")?; + + self.remove_outdated_block_internal(mc_block_handle.id(), top_blocks, max_blocks_per_batch) + .await?; + Ok(()) + } + pub async fn remove_outdated_blocks( &self, key_block_id: &BlockId, @@ -562,11 +584,24 @@ impl BlockStorage { .context("Failed to compute top blocks for target block")? } _ => { - tracing::info!(%key_block_id, "blocks GC skipped"); + tracing::info!(%key_block_id, "Blocks GC skipped"); return Ok(()); } }; + self.remove_outdated_block_internal(key_block_id, top_blocks, max_blocks_per_batch) + .await?; + + // Done + Ok(()) + } + + async fn remove_outdated_block_internal( + &self, + target_block_id: &BlockId, + top_blocks: TopBlocks, + max_blocks_per_batch: Option, + ) -> Result<()> { // Remove all expired entries let total_cached_handles_removed = self.block_handle_storage.gc_handles_cache(&top_blocks); @@ -578,15 +613,13 @@ impl BlockStorage { } = rayon_run(move || remove_blocks(db, max_blocks_per_batch, &top_blocks)).await?; tracing::info!( - %key_block_id, + %target_block_id, total_cached_handles_removed, mc_package_entries_removed, total_package_entries_removed, total_handles_removed, "finished blocks GC" ); - - // Done Ok(()) } diff --git a/storage/src/store/block_handle/mod.rs b/storage/src/store/block_handle/mod.rs index d71dd8efd..82fd07a43 100644 --- a/storage/src/store/block_handle/mod.rs +++ b/storage/src/store/block_handle/mod.rs @@ -165,6 +165,18 @@ impl BlockHandleStorage { self.load_handle(&key_block_id) } + pub fn find_master_block(&self, seqno: u32) -> Option { + if seqno == 0 { + return None; + } + + let mut iter = self.db.key_blocks.raw_iterator(); + iter.seek(seqno.to_be_bytes()); + + let mc_block = BlockId::from_slice(iter.value()?); + self.load_handle(&mc_block) + } + pub fn find_prev_persistent_key_block(&self, seqno: u32) -> Option { if seqno == 0 { return None; diff --git a/storage/src/store/runtime/mod.rs b/storage/src/store/runtime/mod.rs index 6cf671e3c..c4f033f86 100644 --- a/storage/src/store/runtime/mod.rs +++ b/storage/src/store/runtime/mod.rs @@ -1,5 +1,7 @@ use std::sync::Arc; +use parking_lot::lock_api::RwLock; + pub use self::persistent_state_keeper::PersistentStateKeeper; use super::BlockHandleStorage; @@ -7,12 +9,14 @@ mod persistent_state_keeper; pub struct RuntimeStorage { persistent_state_keeper: PersistentStateKeeper, + last_gc_block_collection: parking_lot::RwLock>, } impl RuntimeStorage { pub fn new(block_handle_storage: Arc) -> Self { Self { persistent_state_keeper: PersistentStateKeeper::new(block_handle_storage), + last_gc_block_collection: RwLock::new(None), } } @@ -20,4 +24,16 @@ impl RuntimeStorage { pub fn persistent_state_keeper(&self) -> &PersistentStateKeeper { &self.persistent_state_keeper } + + #[inline(always)] + pub fn set_last_block_gc_collection(&self, mc_seqno: u32) { + let mut guard = self.last_gc_block_collection.write(); + *guard = Some(mc_seqno); + } + + #[inline(always)] + pub fn get_last_block_gc_collection(&self) -> Option { + let mut guard = self.last_gc_block_collection.read(); + *guard + } } From 9aade0beb1d980521b3bf130ba02ab9958dfef06 Mon Sep 17 00:00:00 2001 From: Ivan Kalinin Date: Thu, 4 Jul 2024 19:21:56 +0200 Subject: [PATCH 15/15] feat(core): unify gc subscriber --- Cargo.lock | 1 + Cargo.toml | 1 + block-util/src/archive/entry_id.rs | 46 +- block-util/src/archive/mod.rs | 2 +- cli/src/main.rs | 13 +- cli/src/node/mod.rs | 28 +- cli/src/tools/storage_cli.rs | 25 +- core/Cargo.toml | 1 + core/src/block_strider/mod.rs | 9 +- core/src/block_strider/state_applier.rs | 3 +- .../block_strider/subscriber/gc_subscriber.rs | 492 ++++++++---------- core/src/block_strider/subscriber/mod.rs | 2 + storage/src/config.rs | 98 ++-- storage/src/lib.rs | 7 - storage/src/store/block/mod.rs | 192 ++++--- storage/src/store/block_handle/mod.rs | 23 +- storage/src/store/internal_queue/mod.rs | 4 +- storage/src/store/persistent_state/mod.rs | 2 + storage/src/store/runtime/mod.rs | 18 +- storage/src/store/shard_state/mod.rs | 108 ++-- util/src/time.rs | 20 +- 21 files changed, 519 insertions(+), 576 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1ab3aa1d2..66403854d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3425,6 +3425,7 @@ dependencies = [ "metrics", "parking_lot", "rand", + "scopeguard", "serde", "sha2", "tempfile", diff --git a/Cargo.toml b/Cargo.toml index 749848653..fee1c7e81 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -72,6 +72,7 @@ rustc_version = "0.4" rustls = "0.23" rustls-webpki = "0.102" scc = "2.1" +scopeguard = "1.2" serde = "1.0" serde_json = "1.0.114" serde_path_to_error = "0.1" diff --git a/block-util/src/archive/entry_id.rs b/block-util/src/archive/entry_id.rs index a3f934aff..057fd294c 100644 --- a/block-util/src/archive/entry_id.rs +++ b/block-util/src/archive/entry_id.rs @@ -9,7 +9,7 @@ use smallvec::SmallVec; /// Package entry id. #[derive(Debug, Hash, Eq, PartialEq)] -pub enum ArchiveEntryId { +pub enum ArchiveEntryId { /// Block data entry. Block(I), /// Block proof entry. @@ -37,6 +37,37 @@ impl ArchiveEntryId { } } +impl ArchiveEntryId { + pub fn extract_kind(data: &[u8]) -> Option { + if data.len() < SERIALIZED_LEN { + return None; + } + + Some(match data[SERIALIZED_LEN - 1] { + 0 => ArchiveEntryIdKind::Block, + 1 => ArchiveEntryIdKind::Proof, + 2 => ArchiveEntryIdKind::ProofLink, + _ => return None, + }) + } + + pub fn kind(&self) -> ArchiveEntryIdKind { + match self { + Self::Block(_) => ArchiveEntryIdKind::Block, + Self::Proof(_) => ArchiveEntryIdKind::Proof, + Self::ProofLink(_) => ArchiveEntryIdKind::ProofLink, + } + } +} + +#[derive(Debug, Clone, Copy, Hash, Eq, PartialEq)] +#[repr(u8)] +pub enum ArchiveEntryIdKind { + Block = 0, + Proof = 1, + ProofLink = 2, +} + impl ArchiveEntryId where I: Borrow, @@ -51,13 +82,12 @@ where } /// Constructs on-stack buffer with the serialized object - pub fn to_vec(&self) -> SmallVec<[u8; 4 + 8 + 4 + 32 + 1]> { - // TODO: - let mut result = SmallVec::with_capacity(4 + 8 + 4 + 32 + 1); // TODO: + pub fn to_vec(&self) -> SmallVec<[u8; SERIALIZED_LEN]> { + let mut result = SmallVec::with_capacity(SERIALIZED_LEN); let (block_id, ty) = match self { - Self::Block(id) => (id, 0), - Self::Proof(id) => (id, 1), - Self::ProofLink(id) => (id, 2), + Self::Block(id) => (id, ArchiveEntryIdKind::Block as u8), + Self::Proof(id) => (id, ArchiveEntryIdKind::Proof as u8), + Self::ProofLink(id) => (id, ArchiveEntryIdKind::ProofLink as u8), }; let block_id = block_id.borrow(); @@ -71,6 +101,8 @@ where } } +const SERIALIZED_LEN: usize = 4 + 8 + 4 + 32 + 1; + pub trait GetFileName { fn filename(&self) -> String; } diff --git a/block-util/src/archive/mod.rs b/block-util/src/archive/mod.rs index c4a739389..35de0b3c0 100644 --- a/block-util/src/archive/mod.rs +++ b/block-util/src/archive/mod.rs @@ -1,6 +1,6 @@ use bytes::Bytes; -pub use self::entry_id::{ArchiveEntryId, GetFileName}; +pub use self::entry_id::{ArchiveEntryId, ArchiveEntryIdKind, GetFileName}; pub use self::reader::{ArchiveEntry, ArchiveReader, ArchiveReaderError, ArchiveVerifier}; mod entry_id; diff --git a/cli/src/main.rs b/cli/src/main.rs index 24211ab29..faf010cb7 100644 --- a/cli/src/main.rs +++ b/cli/src/main.rs @@ -21,14 +21,13 @@ mod util; #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[tokio::main] -async fn main() -> ExitCode { +fn main() -> ExitCode { if std::env::var("RUST_BACKTRACE").is_err() { // Enable backtraces on panics by default. std::env::set_var("RUST_BACKTRACE", "1"); } - match App::parse().run().await { + match App::parse().run() { Ok(()) => ExitCode::SUCCESS, Err(err) => { eprintln!("Error: {err}"); @@ -48,8 +47,8 @@ struct App { } impl App { - async fn run(self) -> Result<()> { - self.cmd.run().await + fn run(self) -> Result<()> { + self.cmd.run() } } @@ -65,11 +64,11 @@ enum Cmd { } impl Cmd { - async fn run(self) -> Result<()> { + fn run(self) -> Result<()> { match self { Cmd::Node(cmd) => cmd.run(), Cmd::Tool(cmd) => cmd.run(), - Cmd::Storage(cmd) => cmd.run().await, + Cmd::Storage(cmd) => cmd.run(), } } } diff --git a/cli/src/node/mod.rs b/cli/src/node/mod.rs index 7407c3ece..2ee689bff 100644 --- a/cli/src/node/mod.rs +++ b/cli/src/node/mod.rs @@ -25,9 +25,10 @@ use tycho_collator::validator::client::retry::BackoffConfig; use tycho_collator::validator::config::ValidatorConfig; use tycho_collator::validator::validator::ValidatorStdImplFactory; use tycho_core::block_strider::{ - BlockProvider, BlockStrider, BlockchainBlockProvider, BlockchainBlockProviderConfig, - MetricsSubscriber, OptionalBlockStuff, PersistentBlockStriderState, ShardStateApplier, - StateSubscriber, StateSubscriberContext, StorageBlockProvider, + BlockProvider, BlockStrider, BlockSubscriberExt, BlockchainBlockProvider, + BlockchainBlockProviderConfig, GcSubscriber, MetricsSubscriber, OptionalBlockStuff, + PersistentBlockStriderState, ShardStateApplier, StateSubscriber, StateSubscriberContext, + StorageBlockProvider, }; use tycho_core::blockchain_rpc::{ BlockchainRpcClient, BlockchainRpcService, BlockchainRpcServiceConfig, BroadcastListener, @@ -665,22 +666,23 @@ impl Node { let strider_state = PersistentBlockStriderState::new(self.zerostate.as_block_id(), self.storage.clone()); - let gc_subscriber = GcSubscriber::new(self.storage.clone()); - let block_strider = BlockStrider::builder() .with_provider(( (blockchain_block_provider, storage_block_provider), collator_block_provider, )) .with_state(strider_state) - .with_block_subscriber(( - ShardStateApplier::new( - self.state_tracker.clone(), - self.storage.clone(), - (collator_state_subscriber, rpc_state), - ), - (MetricsSubscriber, gc_subscriber), - )) + .with_block_subscriber( + ( + ShardStateApplier::new( + self.state_tracker.clone(), + self.storage.clone(), + (collator_state_subscriber, rpc_state), + ), + MetricsSubscriber, + ) + .chain(GcSubscriber::new(self.storage.clone())), + ) .build(); // Run block strider diff --git a/cli/src/tools/storage_cli.rs b/cli/src/tools/storage_cli.rs index 4c260430b..16b76820c 100644 --- a/cli/src/tools/storage_cli.rs +++ b/cli/src/tools/storage_cli.rs @@ -27,16 +27,21 @@ pub enum StorageCmd { GetPersistentStatePart, } impl StorageCmd { - pub(crate) async fn run(self) -> Result<()> { - match self { - Self::GetNextKeyblockIds(cmd) => cmd.run(), - Self::GetBlockFull(cmd) => cmd.run().await, - Self::GetNextBlockFull(cmd) => cmd.run_next().await, - Self::GetArchiveInfo(cmd) => cmd.run(), - Self::GetArchiveSlice(cmd) => cmd.run(), - Self::GetPersistentStateInfo(cmd) => cmd.get_state_info(), - Self::GetPersistentStatePart => Ok(()), - } + pub(crate) fn run(self) -> Result<()> { + let rt = tokio::runtime::Builder::new_multi_thread() + .enable_all() + .build()?; + rt.block_on(async move { + match self { + Self::GetNextKeyblockIds(cmd) => cmd.run(), + Self::GetBlockFull(cmd) => cmd.run().await, + Self::GetNextBlockFull(cmd) => cmd.run_next().await, + Self::GetArchiveInfo(cmd) => cmd.run(), + Self::GetArchiveSlice(cmd) => cmd.run(), + Self::GetPersistentStateInfo(cmd) => cmd.get_state_info(), + Self::GetPersistentStatePart => Ok(()), + } + }) } } diff --git a/core/Cargo.toml b/core/Cargo.toml index 095a4194e..1073e8d38 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -21,6 +21,7 @@ itertools = { workspace = true } metrics = { workspace = true } parking_lot = { workspace = true } rand = { workspace = true } +scopeguard = { workspace = true } serde = { workspace = true } sha2 = { workspace = true } thiserror = { workspace = true } diff --git a/core/src/block_strider/mod.rs b/core/src/block_strider/mod.rs index a32f72b3c..ce8f6f285 100644 --- a/core/src/block_strider/mod.rs +++ b/core/src/block_strider/mod.rs @@ -156,10 +156,16 @@ where let started_at = Instant::now(); + let (is_key_block, shards) = { + let custom = block.load_custom()?; + (custom.config.is_some(), custom.shards) + }; + // Begin preparing master block in the background let prepared_master = { let cx = Box::new(BlockSubscriberContext { mc_block_id, + is_key_block, block: block.clone(), archive_data, }); @@ -174,7 +180,7 @@ where // Start downloading shard blocks let mut shard_heights = FastHashMap::default(); let mut download_futures = FuturesUnordered::new(); - for entry in block.load_custom()?.shards.latest_blocks() { + for entry in shards.latest_blocks() { let top_block_id = entry?; shard_heights.insert(top_block_id.shard, top_block_id.seqno); download_futures.push(Box::pin(self.download_shard_blocks(top_block_id))); @@ -258,6 +264,7 @@ where let start_preparing_block = |block: BlockStuffAug| { let cx = Box::new(BlockSubscriberContext { mc_block_id: *mc_block_id, + is_key_block: false, block: block.data, archive_data: block.archive_data, }); diff --git a/core/src/block_strider/state_applier.rs b/core/src/block_strider/state_applier.rs index 2b7fc67e8..dfb65b493 100644 --- a/core/src/block_strider/state_applier.rs +++ b/core/src/block_strider/state_applier.rs @@ -146,6 +146,7 @@ where let started_at = std::time::Instant::now(); let cx = StateSubscriberContext { mc_block_id: cx.mc_block_id, + is_key_block: cx.is_key_block, block: cx.block.clone(), // TODO: rewrite without clone archive_data: cx.archive_data.clone(), // TODO: rewrite without clone state: prepared.state, @@ -160,7 +161,7 @@ where .block_handle_storage() .store_block_applied(&prepared.handle); - if applied && self.inner.storage.config().archives.is_some() { + if applied && self.inner.storage.config().archives_gc.is_some() { tracing::trace!(block_id = %prepared.handle.id(), "saving block into archive"); self.inner .storage diff --git a/core/src/block_strider/subscriber/gc_subscriber.rs b/core/src/block_strider/subscriber/gc_subscriber.rs index fbeae51d5..9774890e9 100644 --- a/core/src/block_strider/subscriber/gc_subscriber.rs +++ b/core/src/block_strider/subscriber/gc_subscriber.rs @@ -1,355 +1,309 @@ -use std::ops::Add; +use std::pin::pin; use std::sync::atomic::{AtomicU32, Ordering}; use std::sync::Arc; -use std::time::Instant; - -use tokio::sync::Notify; -use tokio::task::{AbortHandle, JoinHandle}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use everscale_types::models::BlockId; +use rand::Rng; +use scopeguard::defer; +use tokio::sync::watch; +use tokio::task::AbortHandle; use tycho_block_util::block::BlockStuff; -use tycho_storage::{ArchivesGcInterval, Storage}; -use tycho_util::time::duration_between_unix_and_instant; +use tycho_storage::{BlocksGcType, Storage}; use crate::block_strider::{ BlockSubscriber, BlockSubscriberContext, StateSubscriber, StateSubscriberContext, }; -const GC_MC_BLOCK_INTERVAL: u32 = 10000; - +#[derive(Clone)] #[repr(transparent)] pub struct GcSubscriber { inner: Arc, } -#[derive(Clone)] -enum BlockGcTrigger { - Common(BlockStuff), - Interval(BlockStuff), -} - -impl BlockGcTrigger { - fn get_block_seqno(&self) -> u32 { - match self { - BlockGcTrigger::Common(stuff) | BlockGcTrigger::Interval(stuff) => stuff.id().seqno, - } - } -} - impl GcSubscriber { pub fn new(storage: Storage) -> Self { - let (block_sender, mut block_receiver) = - tokio::sync::watch::channel::>(None); - let (state_sender, mut state_receiver) = - tokio::sync::watch::channel::>(None); - - let mut inner = Inner { - storage: storage.clone(), - block_sender, - state_sender, - handle_block_task: None, - handle_state_task: None, - archives_handle: None, - }; - - let hb_handle = Self::handle_block_gc(block_receiver, storage.clone()).abort_handle(); - let hs_handle = Self::handle_state_gc(state_receiver, storage.clone()).abort_handle(); - let archives_handle = - tokio::spawn(Self::handle_archives_gc(storage.clone())).abort_handle(); + let last_key_block_seqno = storage + .block_handle_storage() + .find_last_key_block() + .map_or(0, |handle| handle.id().seqno); - inner.handle_block_task = Some(hb_handle); - inner.handle_state_task = Some(hs_handle); - inner.archives_handle = Some(archives_handle); + let (trigger_tx, trigger_rx) = watch::channel(None::); + let blocks_gc = tokio::spawn(Self::blocks_gc(trigger_rx.clone(), storage.clone())); + let states_gc = tokio::spawn(Self::states_gc(trigger_rx, storage.clone())); + let archives_gc = tokio::spawn(Self::archives_gc(storage)); Self { - inner: Arc::new(inner), + inner: Arc::new(Inner { + trigger_tx, + last_key_block_seqno: AtomicU32::new(last_key_block_seqno), + handle_block_task: blocks_gc.abort_handle(), + handle_state_task: states_gc.abort_handle(), + archives_handle: archives_gc.abort_handle(), + }), } } - pub fn handle(&self, block_stuff: BlockStuff) { - if !block_stuff.id().is_masterchain() { + pub fn handle(&self, is_key_block: bool, block: &BlockStuff) { + if !block.id().is_masterchain() { return; } - let block = match block_stuff.load_info() { - Ok(block) => block, - Err(e) => { - tracing::error!("Failed to load block info: {:?} {e:?}", block_stuff.id()); - return; - } - }; - if let Err(e) = self.inner.state_sender.send(Some(block_stuff.clone())) { - tracing::error!("Failed to execute handle_state for state_sender. {e:?} "); + if is_key_block { + self.inner + .last_key_block_seqno + .store(block.id().seqno, Ordering::Relaxed); } - if block.key_block { - if let Err(e) = self - .inner - .block_sender - .send(Some(BlockGcTrigger::Common(block_stuff.clone()))) - { - tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); - } + self.inner.trigger_tx.send_replace(Some(Trigger { + last_key_block_seqno: self.inner.last_key_block_seqno.load(Ordering::Relaxed), + mc_block_id: *block.id(), + })); + } + + #[tracing::instrument(skip_all)] + async fn archives_gc(storage: Storage) { + let Some(config) = storage.config().archives_gc else { + tracing::warn!("manager disabled"); return; + }; + tracing::info!("manager started"); + defer! { + tracing::info!("manager stopped"); } - let last = self - .inner - .storage - .runtime_storage() - .get_last_block_gc_collection(); - match last { - Some(last_mc_seqno) if last_mc_seqno < block.seqno - GC_MC_BLOCK_INTERVAL => { - if let Err(e) = self - .inner - .block_sender - .send(Some(BlockGcTrigger::Interval(block_stuff.clone()))) - { - tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); - } + let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); + + loop { + let mut new_state_found = pin!(persistent_state_keeper.new_state_found()); + let Some(pss_handle) = persistent_state_keeper.current() else { + new_state_found.await; + continue; + }; + + // Compute the wait duration until the safe time point + let time_to_wait = { + let gen_utime = pss_handle.meta().gen_utime(); + let created_at = std::time::UNIX_EPOCH + Duration::from_secs(gen_utime as _); + (created_at + config.persistent_state_offset) + .duration_since(std::time::SystemTime::now()) + .unwrap_or_default() + }; + + tokio::select! { + _ = tokio::time::sleep(time_to_wait) => {}, + _ = &mut new_state_found => continue, } - None => { - if let Err(e) = self - .inner - .block_sender - .send(Some(BlockGcTrigger::Interval(block_stuff.clone()))) - { - tracing::error!("Failed to execute handle_state for block_sender. {e:?} "); - } - } - _ => { - tracing::debug!("Skipping boundary GC. Block are too fresh"); + + if let Err(e) = storage + .block_storage() + .remove_outdated_archives(pss_handle.id().seqno) + .await + { + tracing::error!("failed to remove outdated archives: {e:?}"); } + + new_state_found.await; } } - async fn handle_archives_gc(storage: Storage) { - let options = match &storage.config().archives { - Some(options) => options, - None => return, + #[tracing::instrument(skip_all)] + async fn blocks_gc(mut trigger_rx: TriggerRx, storage: Storage) { + let Some(config) = storage.config().blocks_gc else { + tracing::warn!("manager disabled"); + return; }; - - struct LowerBound { - archive_id: AtomicU32, - changed: Notify, + tracing::info!("manager started"); + defer! { + tracing::info!("manager stopped"); } - #[allow(unused_mut)] - let mut lower_bound = None::>; - - match options.gc_interval { - ArchivesGcInterval::Manual => return, - ArchivesGcInterval::PersistentStates { offset } => { - tokio::spawn(async move { - let persistent_state_keeper = - storage.runtime_storage().persistent_state_keeper(); - - loop { - tokio::pin!(let new_state_found = persistent_state_keeper.new_state_found();); - - let (until_id, untile_time) = match persistent_state_keeper.current() { - Some(state) => { - let untile_time = - (state.meta().gen_utime() as u64).add(offset.as_secs()); - (state.id().seqno, untile_time) - } - None => { - new_state_found.await; - continue; - } - }; - - tokio::select!( - _ = tokio::time::sleep(duration_between_unix_and_instant(untile_time, Instant::now())) => {}, - _ = &mut new_state_found => continue, - ); - - if let Some(lower_bound) = &lower_bound { - loop { - tokio::pin!(let lower_bound_changed = lower_bound.changed.notified();); - - let lower_bound = lower_bound.archive_id.load(Ordering::Acquire); - if until_id < lower_bound { - break; - } - - tracing::info!( - until_id, - lower_bound, - "waiting for the archives barrier" - ); - lower_bound_changed.await; - } - } - - if let Err(e) = storage - .block_storage() - .remove_outdated_archives(until_id) - .await - { - tracing::error!("failed to remove outdated archives: {e:?}"); + let block_handles = storage.block_handle_storage(); + + let mut last_tiggered_at = None::; + let mut known_key_block_seqno = 0; + + while trigger_rx.changed().await.is_ok() { + let Some(trigger) = trigger_rx.borrow_and_update().clone() else { + continue; + }; + tracing::debug!(?trigger); + + // NOTE: Track the last mc block seqno since we cannot rely on the broadcasted block. + // It may be updated faster than the iteration of the GC manager. + let has_new_key_block = trigger.last_key_block_seqno > known_key_block_seqno; + known_key_block_seqno = trigger.last_key_block_seqno; + + let target_seqno = match config.ty { + BlocksGcType::BeforeSafeDistance { + safe_distance, + min_interval, + } => { + // Compute the target masterchain block seqno + let target_seqno = match trigger.mc_block_id.seqno.checked_sub(safe_distance) { + // Skip GC for the first N blocks + None | Some(0) => continue, + Some(seqno) => seqno, + }; + + // Debounce GC + if let Some(last) = last_tiggered_at { + if last.elapsed() < min_interval { + // Sleep until the desired interval + // AND continue to wait for the next trigger + tokio::time::sleep_until((last + min_interval).into()).await; + continue; } - - new_state_found.await; } - }); - } - } - } - - fn handle_block_gc( - mut block_receiver: tokio::sync::watch::Receiver>, - storage: Storage, - ) -> JoinHandle<()> { - tokio::spawn(async move { - loop { - if let Err(e) = block_receiver.changed().await { - tracing::error!("Failed to receive block from block_receiver. {e:?}"); - continue; - } - - tracing::info!("Block GC executed..."); - let block = block_receiver.borrow_and_update().clone(); - - if !storage.gc_enable_for_sync() { - tracing::debug!("Block GC is not enabled for sync."); - continue; + // NOTE: You should update this in other branches as well, + // if we want to debounce other types of GC. + last_tiggered_at = Some(Instant::now()); + target_seqno } + BlocksGcType::BeforePreviousKeyBlock => { + if !has_new_key_block { + continue; + } - let (Some(bs), Some(config)) = (block, storage.config().blocks_gc_config) else { - tracing::debug!("Block GC is disabled by config or boundary block not found"); - continue; - }; - - let cleaning_seqno = bs.get_block_seqno(); - - match bs { - BlockGcTrigger::Common(bs) => { - tracing::debug!( - "Removing outdated blocks for block {}, batch of {:?} and kind {:?}", - bs.id(), - config.max_blocks_per_batch, - config.kind - ); - if let Err(e) = storage - .block_storage() - .remove_outdated_blocks( - bs.id(), - config.max_blocks_per_batch, - config.kind, - ) - .await - { - tracing::error!("Failed to remove_outdated_blocks. {e:?}"); + // Find a key block before the last key block from the trigger + let target_seqno = trigger.last_key_block_seqno; + match block_handles.find_prev_key_block(target_seqno) { + Some(handle) => handle.id().seqno, + None => { + tracing::warn!(target_seqno, "previous key block not found"); + continue; } } - BlockGcTrigger::Interval(bs) => { - let Some(handle) = storage.block_handle_storage().load_handle(bs.id()) - else { - return; - }; - - if let Err(e) = storage - .block_storage() - .remove_boundary_blocks(handle, config.max_blocks_per_batch) - .await - { - tracing::error!("Failed to remove_boundary_blocks. {e:?}"); + } + BlocksGcType::BeforePreviousPersistentState => { + if !has_new_key_block { + continue; + } + + // Find a persistent block before the last key block from the trigger + let target_seqno = trigger.last_key_block_seqno; + match block_handles.find_prev_persistent_key_block(target_seqno) { + Some(handle) => handle.id().seqno, + None => { + tracing::warn!(target_seqno, "previous persistent block not found"); + continue; } } } + }; - storage - .runtime_storage() - .set_last_block_gc_collection(cleaning_seqno); + if let Err(e) = storage + .block_storage() + .remove_outdated_blocks(target_seqno, config.max_blocks_per_batch) + .await + { + tracing::error!("failed to remove outdated blocks: {e:?}"); } - }) + } } - fn handle_state_gc( - mut state_receiver: tokio::sync::watch::Receiver>, - storage: Storage, - ) -> JoinHandle<()> { - tokio::spawn(async move { - loop { - if let Err(e) = state_receiver.changed().await { - tracing::error!("Failed to receive block from block_receiver. {e:?}"); - continue; - } - - tracing::info!("State GC executed..."); - - let Some(block) = state_receiver.borrow_and_update().clone() else { - tracing::info!("Boundary GC block is not found"); - continue; - }; - tracing::info!("GC state is executing for block: {:?}", block.id()); - - let shard_state_storage = storage.shard_state_storage(); + #[tracing::instrument(skip_all)] + async fn states_gc(mut trigger_rx: TriggerRx, storage: Storage) { + let Some(config) = storage.config().states_gc else { + tracing::warn!("manager disabled"); + return; + }; + tracing::info!("manager started"); + defer! { + tracing::info!("manager stopped"); + } - match shard_state_storage - .remove_outdated_states(block.id().seqno) - .await - { - Ok(_) => (), - Err(e) => { - tracing::error!(target: "storage", "Failed to GC state: {e:?}"); + let mut last_tiggered_at = None::; + + while trigger_rx.changed().await.is_ok() { + match last_tiggered_at { + // Wait for an offset before the first GC but after the first trigger + None => { + let offset = if config.random_offset { + rand::thread_rng().gen_range(Duration::ZERO..config.interval) + } else { + config.interval + }; + tokio::time::sleep(offset).await + } + // Wait to maintaint the interval between GCs + Some(last) => { + if last.elapsed() < config.interval { + tokio::time::sleep_until((last + config.interval).into()).await; } - }; + } } - }) + last_tiggered_at = Some(Instant::now()); + + // Get the most recent trigger + let Some(trigger) = trigger_rx.borrow_and_update().clone() else { + continue; + }; + tracing::debug!(?trigger); + + if let Err(e) = storage + .shard_state_storage() + .remove_outdated_states(trigger.mc_block_id.seqno) + .await + { + tracing::error!("failed to remove outdated states: {e:?}"); + } + } } } struct Inner { - storage: Storage, - block_sender: tokio::sync::watch::Sender>, - state_sender: tokio::sync::watch::Sender>, + trigger_tx: TriggerTx, + last_key_block_seqno: AtomicU32, - handle_block_task: Option, - handle_state_task: Option, - archives_handle: Option, + handle_block_task: AbortHandle, + handle_state_task: AbortHandle, + archives_handle: AbortHandle, } impl Drop for Inner { fn drop(&mut self) { - if let Some(handle) = self.handle_block_task.take() { - handle.abort(); - } - - if let Some(handle) = self.handle_state_task.take() { - handle.abort(); - } - - if let Some(handle) = self.archives_handle.take() { - handle.abort(); - } + self.handle_block_task.abort(); + self.handle_state_task.abort(); + self.archives_handle.abort(); } } +#[derive(Debug, Clone)] +struct Trigger { + mc_block_id: BlockId, + last_key_block_seqno: u32, +} + +type TriggerTx = watch::Sender>; +type TriggerRx = watch::Receiver>; + impl StateSubscriber for GcSubscriber { - type HandleStateFut<'a> = futures_util::future::Ready>; + type HandleStateFut<'a> = futures_util::future::Ready>; fn handle_state<'a>(&'a self, cx: &'a StateSubscriberContext) -> Self::HandleStateFut<'a> { - self.handle(cx.block.clone()); + self.handle(cx.is_key_block, &cx.block); futures_util::future::ready(Ok(())) } } impl BlockSubscriber for GcSubscriber { type Prepared = (); - type PrepareBlockFut<'a> = futures_util::future::Ready>; - type HandleBlockFut<'a> = futures_util::future::Ready>; + type PrepareBlockFut<'a> = futures_util::future::Ready>; + type HandleBlockFut<'a> = futures_util::future::Ready>; - fn prepare_block<'a>(&'a self, _cx: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { + fn prepare_block<'a>(&'a self, _: &'a BlockSubscriberContext) -> Self::PrepareBlockFut<'a> { futures_util::future::ready(Ok(())) } fn handle_block<'a>( &'a self, cx: &'a BlockSubscriberContext, - _prepared: Self::Prepared, + _: Self::Prepared, ) -> Self::HandleBlockFut<'a> { - self.handle(cx.block.clone()); + self.handle(cx.is_key_block, &cx.block); futures_util::future::ready(Ok(())) } } diff --git a/core/src/block_strider/subscriber/mod.rs b/core/src/block_strider/subscriber/mod.rs index eb2b18edd..54a8fca65 100644 --- a/core/src/block_strider/subscriber/mod.rs +++ b/core/src/block_strider/subscriber/mod.rs @@ -18,6 +18,7 @@ mod metrics_subscriber; pub struct BlockSubscriberContext { pub mc_block_id: BlockId, + pub is_key_block: bool, pub block: BlockStuff, pub archive_data: ArchiveData, } @@ -96,6 +97,7 @@ impl BlockSubscriberExt for B { pub struct StateSubscriberContext { pub mc_block_id: BlockId, + pub is_key_block: bool, pub block: BlockStuff, pub archive_data: ArchiveData, pub state: ShardStateStuff, diff --git a/storage/src/config.rs b/storage/src/config.rs index 5e4d5cb52..df395bb75 100644 --- a/storage/src/config.rs +++ b/storage/src/config.rs @@ -2,12 +2,9 @@ use std::path::{Path, PathBuf}; use std::time::Duration; use bytesize::ByteSize; -use rand::Rng; use serde::{Deserialize, Serialize}; use tycho_util::serde_helpers; -use crate::BlocksGcKind; - #[derive(Debug, Clone, Serialize, Deserialize)] #[serde(deny_unknown_fields, default)] pub struct StorageConfig { @@ -34,17 +31,17 @@ pub struct StorageConfig { /// Archives storage config. /// /// Archives are disabled if this field is `None`. - pub archives: Option, + pub archives_gc: Option, /// States GC config. /// /// States GC is disabled if this field is `None`. - pub states_gc_options: Option, + pub states_gc: Option, /// Blocks GC config. /// /// Blocks GC is disabled if this field is `None`. - pub blocks_gc_config: Option, + pub blocks_gc: Option, } impl StorageConfig { @@ -55,9 +52,9 @@ impl StorageConfig { rocksdb_lru_capacity: ByteSize::kb(1024), cells_cache_size: ByteSize::kb(1024), rocksdb_enable_metrics: false, - archives: Some(ArchivesConfig::default()), - states_gc_options: Some(StateGcOptions::default()), - blocks_gc_config: Some(BlocksGcOptions::default()), + archives_gc: None, + states_gc: None, + blocks_gc: None, } } } @@ -106,66 +103,59 @@ impl Default for StorageConfig { cells_cache_size, rocksdb_lru_capacity, rocksdb_enable_metrics: true, - archives: Some(ArchivesConfig::default()), - states_gc_options: Some(StateGcOptions::default()), - blocks_gc_config: Some(BlocksGcOptions::default()), + archives_gc: Some(ArchivesGcConfig::default()), + states_gc: Some(StatesGcConfig::default()), + blocks_gc: Some(BlocksGcConfig::default()), } } } -#[derive(Default, Debug, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields)] -pub struct ArchivesConfig { - pub gc_interval: ArchivesGcInterval, -} - #[derive(Debug, Clone, Copy, Serialize, Deserialize)] -#[serde(deny_unknown_fields, tag = "type", rename_all = "snake_case")] -pub enum ArchivesGcInterval { - /// Do not perform archives GC - Manual, - /// Archives GC triggers on each persistent state - PersistentStates { - /// Remove archives after this interval after the new persistent state - #[serde(with = "serde_helpers::humantime")] - offset: Duration, - }, +#[serde(default, deny_unknown_fields)] +pub struct ArchivesGcConfig { + /// Remove archives after this interval after the new persistent state + #[serde(with = "serde_helpers::humantime")] + pub persistent_state_offset: Duration, } -impl Default for ArchivesGcInterval { +impl Default for ArchivesGcConfig { fn default() -> Self { - Self::PersistentStates { - offset: Duration::from_secs(300), + Self { + persistent_state_offset: Duration::from_secs(300), } } } #[derive(Debug, Copy, Clone, Serialize, Deserialize)] #[serde(default, deny_unknown_fields)] -pub struct StateGcOptions { - /// Default: rand[0,900) - pub offset_sec: u64, +pub struct StatesGcConfig { + /// Wether to add random offset to the first interval. + /// + /// Default: true. + pub random_offset: bool, /// Default: 900 - pub interval_sec: u64, + #[serde(with = "serde_helpers::humantime")] + pub interval: Duration, } -impl Default for StateGcOptions { +impl Default for StatesGcConfig { fn default() -> Self { Self { - offset_sec: rand::thread_rng().gen_range(0..60), - interval_sec: 60, + random_offset: true, + interval: Duration::from_secs(60), } } } #[derive(Debug, Copy, Clone, Serialize, Deserialize)] -#[serde(default, deny_unknown_fields)] -pub struct BlocksGcOptions { +#[serde(default)] +pub struct BlocksGcConfig { /// Blocks GC type /// - `before_previous_key_block` - on each new key block delete all blocks before the previous one /// - `before_previous_persistent_state` - on each new key block delete all blocks before the /// previous key block with persistent state - pub kind: BlocksGcKind, + #[serde(flatten)] + pub ty: BlocksGcType, /// Whether to enable blocks GC during sync. Default: true pub enable_for_sync: bool, @@ -174,12 +164,34 @@ pub struct BlocksGcOptions { pub max_blocks_per_batch: Option, } -impl Default for BlocksGcOptions { +impl Default for BlocksGcConfig { fn default() -> Self { Self { - kind: BlocksGcKind::BeforePreviousPersistentState, + ty: BlocksGcType::BeforeSafeDistance { + safe_distance: 1000, + min_interval: Duration::from_secs(60), + }, enable_for_sync: true, max_blocks_per_batch: Some(100_000), } } } + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] +#[serde(tag = "type")] +pub enum BlocksGcType { + /// Remove all blocks before the specified safe distance (of mc blocks). + BeforeSafeDistance { + /// Number of masterchain blocks to keep. + safe_distance: u32, + /// Minimum interval between GC runs. + /// + /// Should be about 1 minute. + #[serde(with = "serde_helpers::humantime")] + min_interval: Duration, + }, + /// Remove all blocks before the previous key block. + BeforePreviousKeyBlock, + /// Remove all blocks before the previous persistent state. + BeforePreviousPersistentState, +} diff --git a/storage/src/lib.rs b/storage/src/lib.rs index 463c0bbcd..f226516db 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -249,13 +249,6 @@ impl Storage { pub fn internal_queue_storage(&self) -> &InternalQueueStorage { &self.inner.internal_queue_storage } - - pub fn gc_enable_for_sync(&self) -> bool { - self.inner - .config - .blocks_gc_config - .is_some_and(|x| x.enable_for_sync) - } } struct Inner { diff --git a/storage/src/store/block/mod.rs b/storage/src/store/block/mod.rs index ee323ae9a..6e1795289 100644 --- a/storage/src/store/block/mod.rs +++ b/storage/src/store/block/mod.rs @@ -6,15 +6,16 @@ use std::sync::Arc; use std::time::Instant; use anyhow::{Context, Result}; +use everscale_types::boc::BocRepr; +use everscale_types::cell::HashBytes; use everscale_types::models::*; use parking_lot::RwLock; -use serde::{Deserialize, Serialize}; use tycho_block_util::archive::{ - make_archive_entry, ArchiveData, ArchiveEntryId, ArchiveReaderError, ArchiveVerifier, - GetFileName, + make_archive_entry, ArchiveData, ArchiveEntryId, ArchiveEntryIdKind, ArchiveReaderError, + ArchiveVerifier, GetFileName, }; use tycho_block_util::block::{ - BlockProofStuff, BlockProofStuffAug, BlockStuff, BlockStuffAug, TopBlocks, + BlockProofStuff, BlockProofStuffAug, BlockStuff, BlockStuffAug, ShardHeights, }; use tycho_util::metrics::HistogramGuard; use tycho_util::sync::rayon_run; @@ -197,6 +198,41 @@ impl BlockStorage { .await } + pub fn find_mc_block_data(&self, mc_seqno: u32) -> Result> { + let package_entries = &self.db.package_entries; + + let mut bound = BlockId { + shard: ShardIdent::MASTERCHAIN, + seqno: mc_seqno, + root_hash: HashBytes::ZERO, + file_hash: HashBytes::ZERO, + }; + + let mut readopts = package_entries.new_read_config(); + readopts.set_iterate_lower_bound(ArchiveEntryId::Block(bound).to_vec().as_slice()); + bound.seqno += 1; + readopts.set_iterate_upper_bound(ArchiveEntryId::Block(bound).to_vec().as_slice()); + + let mut iter = self + .db + .rocksdb() + .raw_iterator_cf_opt(&package_entries.cf(), readopts); + + iter.seek_to_first(); + loop { + let Some((key, value)) = iter.item() else { + iter.status()?; + return Ok(None); + }; + + let Some(ArchiveEntryIdKind::Block) = ArchiveEntryId::<()>::extract_kind(key) else { + continue; + }; + + return Ok(Some(BocRepr::decode::(value)?)); + } + } + pub async fn store_block_proof( &self, proof: &BlockProofStuffAug, @@ -531,89 +567,47 @@ impl BlockStorage { } } - pub async fn remove_boundary_blocks( - &self, - mc_block_handle: BlockHandle, - max_blocks_per_batch: Option, - ) -> Result<()> { - tracing::info!( - target_block_id = %mc_block_handle.id(), - "starting blocks GC", - ); - - let top_blocks = self - .load_block_data(&mc_block_handle) - .await - .context("Failed to load target key block data") - .and_then(|block_data| TopBlocks::from_mc_block(&block_data)) - .context("Failed to compute top blocks for target block")?; - - self.remove_outdated_block_internal(mc_block_handle.id(), top_blocks, max_blocks_per_batch) - .await?; - Ok(()) - } - + #[tracing::instrument(skip_all, fields(mc_seqno))] pub async fn remove_outdated_blocks( &self, - key_block_id: &BlockId, + mc_seqno: u32, max_blocks_per_batch: Option, - gc_type: BlocksGcKind, ) -> Result<()> { - // Find target block - let target_block = match gc_type { - BlocksGcKind::BeforePreviousKeyBlock => self - .block_handle_storage - .find_prev_key_block(key_block_id.seqno), - BlocksGcKind::BeforePreviousPersistentState => self - .block_handle_storage - .find_prev_persistent_key_block(key_block_id.seqno), + tracing::info!("started blocks GC"); + + let block = self + .find_mc_block_data(mc_seqno) + .context("failed to load target block data")? + .context("target block not found")?; + + let shard_heights = { + let extra = block.extra.load()?; + let custom = extra.custom.context("mc block extra not found")?.load()?; + custom + .shards + .latest_blocks() + .map(|id| id.map(|id| (id.shard, id.seqno))) + .collect::>()? }; - // Load target block data - let top_blocks = match target_block { - Some(handle) if handle.meta().has_data() => { - tracing::info!( - %key_block_id, - target_block_id = %handle.id(), - "starting blocks GC", - ); - self.load_block_data(&handle) - .await - .context("Failed to load target key block data") - .and_then(|block_data| TopBlocks::from_mc_block(&block_data)) - .context("Failed to compute top blocks for target block")? - } - _ => { - tracing::info!(%key_block_id, "Blocks GC skipped"); - return Ok(()); - } - }; - - self.remove_outdated_block_internal(key_block_id, top_blocks, max_blocks_per_batch) - .await?; - - // Done - Ok(()) - } - - async fn remove_outdated_block_internal( - &self, - target_block_id: &BlockId, - top_blocks: TopBlocks, - max_blocks_per_batch: Option, - ) -> Result<()> { // Remove all expired entries - let total_cached_handles_removed = self.block_handle_storage.gc_handles_cache(&top_blocks); + let total_cached_handles_removed = self + .block_handle_storage + .gc_handles_cache(mc_seqno, &shard_heights); + let span = tracing::Span::current(); let db = self.db.clone(); let BlockGcStats { mc_package_entries_removed, total_package_entries_removed, total_handles_removed, - } = rayon_run(move || remove_blocks(db, max_blocks_per_batch, &top_blocks)).await?; + } = rayon_run(move || { + let _span = span.enter(); + remove_blocks(db, max_blocks_per_batch, mc_seqno, shard_heights) + }) + .await?; tracing::info!( - %target_block_id, total_cached_handles_removed, mc_package_entries_removed, total_package_entries_removed, @@ -623,7 +617,10 @@ impl BlockStorage { Ok(()) } + #[tracing::instrument(skip_all, fields(until_id))] pub async fn remove_outdated_archives(&self, until_id: u32) -> Result<()> { + tracing::info!("started archives GC"); + let mut archive_ids = self.archive_ids.write(); let retained_ids = match archive_ids.iter().rev().find(|&id| *id < until_id).cloned() { @@ -631,7 +628,7 @@ impl BlockStorage { // `archive_ids` will now contain [..until_id] Some(until_id) => archive_ids.split_off(&until_id), None => { - tracing::info!("archives GC: nothing to remove"); + tracing::info!("nothing to remove"); return Ok(()); } }; @@ -639,27 +636,19 @@ impl BlockStorage { let removed_ids = std::mem::replace(&mut *archive_ids, retained_ids); // Print removed range bounds and compute real `until_id` - let until_id = match (removed_ids.first(), removed_ids.last()) { - (Some(first), Some(last)) => { - let len = removed_ids.len(); - tracing::info!( - archive_count = len, - first, - last, - "archives GC: removing archives" - ); - - match archive_ids.first() { - Some(until_id) => *until_id, - None => *last + 1, - } - } - _ => { - tracing::info!("archives GC: nothing to remove"); - return Ok(()); - } + let (Some(first), Some(last)) = (removed_ids.first(), removed_ids.last()) else { + tracing::info!("nothing to remove"); + return Ok(()); }; + let len = removed_ids.len(); + let until_id = match archive_ids.first() { + Some(until_id) => *until_id, + None => *last + 1, + }; + + drop(archive_ids); + // Remove archives let archives_cf = self.db.archives.cf(); let write_options = self.db.archives.write_config(); @@ -671,7 +660,7 @@ impl BlockStorage { write_options, )?; - tracing::info!("archives GC: done"); + tracing::info!(archive_count = len, first, last, "finished archives GC"); Ok(()) } @@ -768,13 +757,6 @@ impl BlockStorage { } } -#[derive(Debug, Copy, Clone, Eq, PartialEq, Serialize, Deserialize)] -#[serde(rename_all = "snake_case")] -pub enum BlocksGcKind { - BeforePreviousKeyBlock, - BeforePreviousPersistentState, -} - #[derive(Clone)] pub enum BlockProofHandle { Existing(BlockHandle), @@ -802,7 +784,8 @@ pub struct StoreBlockResult { fn remove_blocks( db: BaseDb, max_blocks_per_batch: Option, - top_blocks: &TopBlocks, + mc_seqno: u32, + shard_heights: ShardHeights, ) -> Result { let mut stats = BlockGcStats::default(); @@ -830,16 +813,19 @@ fn remove_blocks( // Read only prefix with shard ident and seqno let BlockIdShort { shard, seqno } = BlockIdShort::from_slice(key); + let is_masterchain = shard.is_masterchain(); // Don't gc latest blocks - if top_blocks.contains_shard_seqno(&shard, seqno) { + if is_masterchain && seqno >= mc_seqno + || !is_masterchain && shard_heights.contains_shard_seqno(&shard, seqno) + { blocks_iter.next(); continue; } // Additionally check whether this item is a key block if seqno == 0 - || shard.is_masterchain() + || is_masterchain && raw .get_pinned_cf_opt(&key_blocks_cf, seqno.to_be_bytes(), &key_blocks_readopts)? .is_some() diff --git a/storage/src/store/block_handle/mod.rs b/storage/src/store/block_handle/mod.rs index 82fd07a43..42f37d03c 100644 --- a/storage/src/store/block_handle/mod.rs +++ b/storage/src/store/block_handle/mod.rs @@ -1,7 +1,7 @@ use std::sync::Arc; use everscale_types::models::BlockId; -use tycho_block_util::block::TopBlocks; +use tycho_block_util::block::ShardHeights; use tycho_block_util::state::is_persistent_state; use crate::db::*; @@ -165,18 +165,6 @@ impl BlockHandleStorage { self.load_handle(&key_block_id) } - pub fn find_master_block(&self, seqno: u32) -> Option { - if seqno == 0 { - return None; - } - - let mut iter = self.db.key_blocks.raw_iterator(); - iter.seek(seqno.to_be_bytes()); - - let mc_block = BlockId::from_slice(iter.value()?); - self.load_handle(&mc_block) - } - pub fn find_prev_persistent_key_block(&self, seqno: u32) -> Option { if seqno == 0 { return None; @@ -242,7 +230,7 @@ impl BlockHandleStorage { } } - pub fn gc_handles_cache(&self, top_blocks: &TopBlocks) -> usize { + pub fn gc_handles_cache(&self, mc_seqno: u32, shard_heights: &ShardHeights) -> usize { let mut total_removed = 0; self.cache.retain(|block_id, value| { @@ -254,9 +242,12 @@ impl BlockHandleStorage { } }; + let is_masterchain = block_id.is_masterchain(); + if block_id.seqno == 0 - || block_id.is_masterchain() && value.is_key_block() - || top_blocks.contains(block_id) + || is_masterchain && (block_id.seqno >= mc_seqno || value.is_key_block()) + || !is_masterchain + && shard_heights.contains_shard_seqno(&block_id.shard, block_id.seqno) { // Keep zero state, key blocks and latest blocks true diff --git a/storage/src/store/internal_queue/mod.rs b/storage/src/store/internal_queue/mod.rs index d404caaea..9494796d8 100644 --- a/storage/src/store/internal_queue/mod.rs +++ b/storage/src/store/internal_queue/mod.rs @@ -326,8 +326,8 @@ impl InternalQueueStorage { let mut batch = WriteBatch::default(); while iter.valid() { - let (mut key, value) = match (iter.key(), iter.value()) { - (Some(key), Some(value)) => (key, value), + let mut key = match iter.key() { + Some(key) => key, _ => break, }; diff --git a/storage/src/store/persistent_state/mod.rs b/storage/src/store/persistent_state/mod.rs index fbd4c9ea0..48588596f 100644 --- a/storage/src/store/persistent_state/mod.rs +++ b/storage/src/store/persistent_state/mod.rs @@ -97,10 +97,12 @@ impl PersistentStateStorage { let root_hash = *root_hash; let is_cancelled = Some(self.is_cancelled.clone()); + let span = tracing::Span::current(); let db = self.db.clone(); let states_dir = self.prepare_persistent_states_dir(mc_seqno)?; rayon_run(move || { + let _span = span.enter(); let cell_writer = state_writer::StateWriter::new(&db, &states_dir, &block_id); match cell_writer.write(&root_hash, is_cancelled.as_deref()) { Ok(()) => tracing::info!(block_id = %block_id, "persistent state saved"), diff --git a/storage/src/store/runtime/mod.rs b/storage/src/store/runtime/mod.rs index c4f033f86..ce2b6f7d3 100644 --- a/storage/src/store/runtime/mod.rs +++ b/storage/src/store/runtime/mod.rs @@ -1,22 +1,18 @@ use std::sync::Arc; -use parking_lot::lock_api::RwLock; - pub use self::persistent_state_keeper::PersistentStateKeeper; -use super::BlockHandleStorage; +use crate::store::BlockHandleStorage; mod persistent_state_keeper; pub struct RuntimeStorage { persistent_state_keeper: PersistentStateKeeper, - last_gc_block_collection: parking_lot::RwLock>, } impl RuntimeStorage { pub fn new(block_handle_storage: Arc) -> Self { Self { persistent_state_keeper: PersistentStateKeeper::new(block_handle_storage), - last_gc_block_collection: RwLock::new(None), } } @@ -24,16 +20,4 @@ impl RuntimeStorage { pub fn persistent_state_keeper(&self) -> &PersistentStateKeeper { &self.persistent_state_keeper } - - #[inline(always)] - pub fn set_last_block_gc_collection(&self, mc_seqno: u32) { - let mut guard = self.last_gc_block_collection.write(); - *guard = Some(mc_seqno); - } - - #[inline(always)] - pub fn get_last_block_gc_collection(&self) -> Option { - let mut guard = self.last_gc_block_collection.read(); - *guard - } } diff --git a/storage/src/store/shard_state/mod.rs b/storage/src/store/shard_state/mod.rs index 5d5d5e475..f442fe4db 100644 --- a/storage/src/store/shard_state/mod.rs +++ b/storage/src/store/shard_state/mod.rs @@ -123,20 +123,7 @@ impl ShardStateStorage { let new_cell_count = cell_storage.store_cell(&mut batch, root_cell)?; metrics::histogram!("tycho_storage_cell_count").record(new_cell_count as f64); - let mut value = [0; 32 * 3]; - value[..32].copy_from_slice(root_hash.as_slice()); - value[32..64].copy_from_slice(block_id.root_hash.as_slice()); - value[64..96].copy_from_slice(block_id.file_hash.as_slice()); - - batch.put_cf( - &cf.bound(), - BlockIdShort { - shard: block_id.shard, - seqno: block_id.seqno, - } - .to_vec(), - value, - ); + batch.put_cf(&cf.bound(), block_id.to_vec(), root_hash.as_slice()); let hist = HistogramGuard::begin("tycho_storage_state_update_time"); raw_db.write(batch)?; @@ -175,24 +162,25 @@ impl ShardStateStorage { } pub async fn load_state(&self, block_id: &BlockId) -> Result { - let cell_id = self.load_state_root(block_id.as_short_id())?; + let cell_id = self.load_state_root(block_id)?; let cell = self.cell_storage.load_cell(cell_id)?; ShardStateStuff::from_root(block_id, Cell::from(cell as Arc<_>), &self.min_ref_mc_state) } - pub async fn remove_outdated_states(&self, mc_seqno: u32) -> Result { + #[tracing::instrument(skip_all, fields(mc_seqno))] + pub async fn remove_outdated_states(&self, mc_seqno: u32) -> Result<()> { // Compute recent block ids for the specified masterchain seqno - let top_blocks = self - .compute_recent_blocks(mc_seqno) - .await? - .context("Recent blocks edge not found")?; + let Some(top_blocks) = self.compute_recent_blocks(mc_seqno).await? else { + tracing::warn!("recent blocks edge not found"); + return Ok(()); + }; tracing::info!( - block_id = %top_blocks.mc_block, - "starting shard states GC", + target_block_id = %top_blocks.mc_block, + "started states GC", ); - let instant = Instant::now(); + let started_at = Instant::now(); let raw = self.db.rocksdb(); @@ -222,7 +210,7 @@ impl ShardStateStorage { }, }; - let block_id = BlockIdShort::from_slice(key); + let block_id = BlockId::from_slice(key); let root_hash = HashBytes::wrap(value.try_into().expect("invalid value")); // Skip blocks from zero state and top blocks @@ -247,10 +235,7 @@ impl ShardStateStorage { // drop(pending_op); removed_cells += total; - tracing::debug!( - removed_cells = total, - %block_id, - ); + tracing::debug!(removed_cells = total, %block_id); } removed_states += 1; @@ -262,10 +247,10 @@ impl ShardStateStorage { removed_states, removed_cells, block_id = %top_blocks.mc_block, - elapsed_sec = instant.elapsed().as_secs_f64(), - "finished shard states GC", + elapsed_sec = started_at.elapsed().as_secs_f64(), + "finished states GC", ); - Ok(top_blocks) + Ok(()) } /// Searches for an edge with the least referenced masterchain block @@ -279,11 +264,13 @@ impl ShardStateStorage { } } + let snapshot = self.db.rocksdb().snapshot(); + // 1. Find target block // Find block id using states table let mc_block_id = match self - .find_mc_block_id(mc_seqno) + .find_mc_block_id(mc_seqno, &snapshot) .context("Failed to find block id by seqno")? { Some(block_id) => block_id, @@ -307,7 +294,7 @@ impl ShardStateStorage { // Find full min masterchain reference id let min_ref_mc_seqno = block_info.min_ref_mc_seqno; - let min_ref_block_id = match self.find_mc_block_id(min_ref_mc_seqno)? { + let min_ref_block_id = match self.find_mc_block_id(min_ref_mc_seqno, &snapshot)? { Some(block_id) => block_id, None => return Ok(None), }; @@ -327,41 +314,42 @@ impl ShardStateStorage { .map(Some) } - fn load_state_root(&self, block_id_short: BlockIdShort) -> Result { + fn load_state_root(&self, block_id: &BlockId) -> Result { let shard_states = &self.db.shard_states; - let shard_state = shard_states.get(block_id_short.to_vec())?; + let shard_state = shard_states.get(block_id.to_vec())?; match shard_state { Some(root) => Ok(HashBytes::from_slice(&root[..32])), None => Err(ShardStateStorageError::NotFound.into()), } } - fn find_mc_block_id(&self, mc_seqno: u32) -> Result> { + fn find_mc_block_id( + &self, + mc_seqno: u32, + snapshot: &rocksdb::Snapshot<'_>, + ) -> Result> { let shard_states = &self.db.shard_states; - Ok(shard_states - .get( - BlockIdShort { - shard: ShardIdent::MASTERCHAIN, - seqno: mc_seqno, - } - .to_vec(), - )? - .and_then(|value| { - let value = value.as_ref(); - if value.len() < 96 { - return None; - } - - let root_hash: [u8; 32] = value[32..64].try_into().unwrap(); - let file_hash: [u8; 32] = value[64..96].try_into().unwrap(); - - Some(BlockId { - shard: ShardIdent::MASTERCHAIN, - seqno: mc_seqno, - root_hash: HashBytes(root_hash), - file_hash: HashBytes(file_hash), - }) - })) + + let mut bound = BlockId { + shard: ShardIdent::MASTERCHAIN, + seqno: mc_seqno, + root_hash: HashBytes::ZERO, + file_hash: HashBytes::ZERO, + }; + + let mut readopts = shard_states.new_read_config(); + readopts.set_snapshot(snapshot); + readopts.set_iterate_lower_bound(bound.to_vec().as_slice()); + bound.seqno += 1; + readopts.set_iterate_upper_bound(bound.to_vec().as_slice()); + + let mut iter = self + .db + .rocksdb() + .raw_iterator_cf_opt(&shard_states.cf(), readopts); + iter.seek_to_first(); + + Ok(iter.key().map(BlockId::from_slice)) } } diff --git a/util/src/time.rs b/util/src/time.rs index 65630e762..c36fda5f0 100644 --- a/util/src/time.rs +++ b/util/src/time.rs @@ -1,4 +1,4 @@ -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::Duration; use rand::Rng; @@ -25,21 +25,3 @@ pub fn shifted_interval_immediate(period: Duration, max_shift: Duration) -> toki let shift = rand::thread_rng().gen_range(Duration::ZERO..max_shift); tokio::time::interval(period + shift) } - -pub fn duration_between_unix_and_instant(unix_time: u64, instant: Instant) -> Duration { - // Convert Unix timestamp to SystemTime - let system_time = UNIX_EPOCH + Duration::from_secs(unix_time); - - // Convert SystemTime to Instant - let instant_from_unix = Instant::now() - - SystemTime::now() - .duration_since(system_time) - .unwrap_or(Duration::ZERO); - - // Calculate the duration - if instant_from_unix <= instant { - instant - instant_from_unix - } else { - instant_from_unix - instant - } -}