diff --git a/.clippy.toml b/.clippy.toml new file mode 100644 index 000000000..6c16172bd --- /dev/null +++ b/.clippy.toml @@ -0,0 +1,2 @@ +allow-print-in-tests = true +allow-dbg-in-tests = true \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index d8b77b731..0026fa469 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -648,7 +648,7 @@ dependencies = [ [[package]] name = "everscale-types" version = "0.1.0-rc.6" -source = "git+https://github.com/broxus/everscale-types.git?branch=0xdeafbeef/push-yntmntzvxrlu#9ef94cf9f1042d0605d2cc3325fdc570c1092bcd" +source = "git+https://github.com/broxus/everscale-types.git?branch=0xdeafbeef/push-xrvxlsnspsok#a4e7c1441ae58d61b51d60e120c7626593f6902c" dependencies = [ "ahash", "base64 0.21.7", @@ -657,7 +657,6 @@ dependencies = [ "everscale-crypto", "everscale-types-proc", "hex", - "itertools", "once_cell", "serde", "sha2", @@ -669,7 +668,7 @@ dependencies = [ [[package]] name = "everscale-types-proc" version = "0.1.4" -source = "git+https://github.com/broxus/everscale-types.git?branch=0xdeafbeef/push-yntmntzvxrlu#9ef94cf9f1042d0605d2cc3325fdc570c1092bcd" +source = "git+https://github.com/broxus/everscale-types.git?branch=0xdeafbeef/push-xrvxlsnspsok#a4e7c1441ae58d61b51d60e120c7626593f6902c" dependencies = [ "proc-macro2", "quote", @@ -2197,14 +2196,17 @@ name = "tycho-core" version = "0.0.1" dependencies = [ "anyhow", - "async-trait", "castaway", "everscale-types", "futures-util", "itertools", "parking_lot", + "sha2", + "tempfile", + "thiserror", "tokio", "tracing", + "tracing-test", "tycho-block-util", "tycho-network", "tycho-storage", diff --git a/Cargo.toml b/Cargo.toml index 773233388..9fd353a65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -22,6 +22,7 @@ members = [ [workspace.dependencies] # crates.io deps +aarc = "0.2" ahash = "0.8" anyhow = "1.0.79" arc-swap = "1.6.0" @@ -99,7 +100,7 @@ debug = true # NOTE: use crates.io dependency when it is released # https://github.com/sagebind/castaway/issues/18 castaway = { git = "https://github.com/sagebind/castaway.git" } -everscale-types = { git = "https://github.com/broxus/everscale-types.git", branch = "0xdeafbeef/push-yntmntzvxrlu" } +everscale-types = { git = "https://github.com/broxus/everscale-types.git", branch = "0xdeafbeef/push-xrvxlsnspsok" } [workspace.lints.rust] future_incompatible = "warn" @@ -158,6 +159,8 @@ needless_for_each = "warn" option_option = "warn" path_buf_push_overwrite = "warn" ptr_as_ptr = "warn" +print_stdout = "warn" +print_stderr = "warn" rc_mutex = "warn" ref_option_ref = "warn" rest_pat_in_fully_bound_structs = "warn" diff --git a/block-util/src/block/top_blocks.rs b/block-util/src/block/top_blocks.rs index e11c1d3f9..0cda19453 100644 --- a/block-util/src/block/top_blocks.rs +++ b/block-util/src/block/top_blocks.rs @@ -47,7 +47,7 @@ impl TopBlocks { self.contains_shard_seqno(&block_id.shard, block_id.seqno) } - /// Checks whether the given pair of [`ton_block::ShardIdent`] and seqno + /// Checks whether the given pair of [`ShardIdent`] and seqno /// is equal to or greater than the last block for the given shard. /// /// NOTE: Specified shard could be split or merged diff --git a/block-util/src/state/shard_state_stuff.rs b/block-util/src/state/shard_state_stuff.rs index 7d0a22df1..a358893e8 100644 --- a/block-util/src/state/shard_state_stuff.rs +++ b/block-util/src/state/shard_state_stuff.rs @@ -62,13 +62,18 @@ impl ShardStateStuff { let file_hash = sha2::Sha256::digest(bytes); anyhow::ensure!( id.file_hash.as_slice() == file_hash.as_slice(), - "file_hash mismatch for {id}" + "file_hash mismatch. Expected: {}, got: {}", + hex::encode(file_hash), + id.file_hash, ); let root = Boc::decode(bytes)?; anyhow::ensure!( &id.root_hash == root.repr_hash(), - "root_hash mismatch for {id}" + "root_hash mismatch for {id}. Expected: {expected}, got: {got}", + id = id, + expected = id.root_hash, + got = root.repr_hash(), ); Self::new( diff --git a/core/Cargo.toml b/core/Cargo.toml index d8b1d3bc3..335aa82e7 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -10,7 +10,6 @@ license.workspace = true [dependencies] anyhow = { workspace = true } -async-trait = { workspace = true } castaway = { workspace = true } everscale-types = { workspace = true } futures-util = { workspace = true } @@ -18,6 +17,8 @@ itertools = { workspace = true } parking_lot = { workspace = true } tokio = { workspace = true, features = ["rt"] } tracing = { workspace = true } +thiserror = { workspace = true } +sha2 = { workspace = true } # local deps tycho-block-util = { workspace = true } @@ -27,6 +28,8 @@ tycho-util = { workspace = true } [dev-dependencies] tycho-util = { workspace = true, features = ["test"] } +tempfile = { workspace = true } +tracing-test = { workspace = true } [lints] workspace = true diff --git a/core/src/block_strider/mod.rs b/core/src/block_strider/mod.rs index 241ff4774..4690ee5a4 100644 --- a/core/src/block_strider/mod.rs +++ b/core/src/block_strider/mod.rs @@ -4,18 +4,24 @@ use futures_util::future::BoxFuture; use futures_util::stream::FuturesOrdered; use futures_util::{FutureExt, TryStreamExt}; use itertools::Itertools; +use std::sync::Arc; pub mod provider; pub mod state; pub mod subscriber; +mod state_applier; + #[cfg(test)] -mod test_provider; +pub mod test_provider; +use crate::block_strider::state_applier::ShardStateUpdater; use provider::BlockProvider; use state::BlockStriderState; use subscriber::BlockSubscriber; use tycho_block_util::block::BlockStuff; +use tycho_block_util::state::MinRefMcStateTracker; +use tycho_storage::Storage; use tycho_util::FastDashMap; pub struct BlockStriderBuilder(BlockStrider); @@ -59,9 +65,25 @@ where P: BlockProvider, B: BlockSubscriber, { - pub fn build(self) -> BlockStrider { + pub(crate) fn build(self) -> BlockStrider { self.0 } + + pub(crate) fn build_with_state_applier( + self, + min_ref_mc_state_tracker: MinRefMcStateTracker, + storage: Arc, + ) -> BlockStrider> { + BlockStrider { + state: self.0.state, + provider: self.0.provider, + subscriber: ShardStateUpdater::new( + min_ref_mc_state_tracker, + storage, + self.0.subscriber, + ), + } + } } pub struct BlockStrider { @@ -133,61 +155,57 @@ where ) -> BoxFuture<'a, Result>> { async move { let mut prev_shard_block_id = shard_block_id; + let mut traversed_blocks = Vec::new(); - while !self.state.is_traversed(&shard_block_id) { - if shard_block_id.seqno == 0 { - break; - } - + tracing::debug!(id=?shard_block_id, "Finding prev shard blocks"); + while shard_block_id.seqno > 0 && !self.state.is_traversed(&shard_block_id) { prev_shard_block_id = shard_block_id; let block = self .fetch_block(&shard_block_id) .await .expect("provider failed to fetch shard block"); + tracing::debug!(id=?block.id(), "Fetched shard block"); let info = block.block().load_info()?; - shard_block_id = match info.load_prev_ref()? { + + match info.load_prev_ref()? { PrevBlockRef::Single(id) => { - let id = BlockId { - shard: info.shard, - seqno: id.seqno, - root_hash: id.root_hash, - file_hash: id.file_hash, + let shard = if info.after_split { + info.shard + .merge() + .expect("Merge should succeed after split") + } else { + info.shard }; - blocks.add_connection(id, shard_block_id); - id + shard_block_id = id.as_block_id(shard); + blocks.add_connection(shard_block_id, prev_shard_block_id); } PrevBlockRef::AfterMerge { left, right } => { let (left_shard, right_shard) = info.shard.split().expect("split on unsplitable shard"); - let left = BlockId { - shard: left_shard, - seqno: left.seqno, - root_hash: left.root_hash, - file_hash: left.file_hash, - }; - let right = BlockId { - shard: right_shard, - seqno: right.seqno, - root_hash: right.root_hash, - file_hash: right.file_hash, - }; - blocks.add_connection(left, shard_block_id); - blocks.add_connection(right, shard_block_id); - - return futures_util::try_join!( - self.find_prev_shard_blocks(left, blocks), - self.find_prev_shard_blocks(right, blocks) - ) - .map(|(mut left, right)| { - left.extend(right); - left - }); + let left_block_id = left.as_block_id(left_shard); + let right_block_id = right.as_block_id(right_shard); + blocks.add_connection(left_block_id, prev_shard_block_id); + blocks.add_connection(right_block_id, prev_shard_block_id); + + let left_blocks = + self.find_prev_shard_blocks(left_block_id, blocks).await?; + let right_blocks = + self.find_prev_shard_blocks(right_block_id, blocks).await?; + traversed_blocks.extend(left_blocks); + traversed_blocks.extend(right_blocks); + break; } - }; + } + blocks.store_block(block); } - Ok(vec![prev_shard_block_id]) + + if prev_shard_block_id.seqno > 0 { + traversed_blocks.push(prev_shard_block_id); + } + + Ok(traversed_blocks) } .boxed() } @@ -273,7 +291,7 @@ impl BlocksGraph { .get(block_id) .expect("should be in map"); subscriber - .handle_block(&block) + .handle_block(&block, None) .await .expect("subscriber failed"); state.commit_traversed(*block_id); @@ -296,9 +314,8 @@ mod test { use crate::block_strider::BlockStrider; #[tokio::test] + #[tracing_test::traced_test] async fn test_block_strider() { - tycho_util::test::init_logger("test_block_strider"); - let provider = TestBlockProvider::new(3); provider.validate(); diff --git a/core/src/block_strider/state.rs b/core/src/block_strider/state.rs index 6b3308097..c7fb47d71 100644 --- a/core/src/block_strider/state.rs +++ b/core/src/block_strider/state.rs @@ -1,5 +1,9 @@ +use std::sync::Arc; + use everscale_types::models::BlockId; +use tycho_storage::Storage; + pub trait BlockStriderState: Send + Sync + 'static { fn load_last_traversed_master_block_id(&self) -> BlockId; fn is_traversed(&self, block_id: &BlockId) -> bool; @@ -20,6 +24,30 @@ impl BlockStriderState for Box { } } +impl BlockStriderState for Arc { + fn load_last_traversed_master_block_id(&self) -> BlockId { + self.node_state() + .load_last_mc_block_id() + .expect("Db is not initialized") + } + + fn is_traversed(&self, block_id: &BlockId) -> bool { + self.block_handle_storage() + .load_handle(block_id) + .expect("db is dead") + .is_some() + } + + fn commit_traversed(&self, block_id: BlockId) { + if block_id.is_masterchain() { + self.node_state() + .store_last_mc_block_id(&block_id) + .expect("db is dead"); + } + // other blocks are stored with state applier: todo rework this? + } +} + #[cfg(test)] pub struct InMemoryBlockStriderState { last_traversed_master_block_id: parking_lot::Mutex, diff --git a/core/src/block_strider/state_applier.rs b/core/src/block_strider/state_applier.rs new file mode 100644 index 000000000..c750bc895 --- /dev/null +++ b/core/src/block_strider/state_applier.rs @@ -0,0 +1,257 @@ +use std::future::Future; +use std::pin::Pin; +use std::sync::Arc; + +use anyhow::{Context, Result}; +use futures_util::FutureExt; + +use tycho_block_util::block::BlockStuff; +use tycho_block_util::state::{MinRefMcStateTracker, ShardStateStuff}; +use tycho_storage::{BlockHandle, BlockMetaData, Storage}; + +use super::subscriber::BlockSubscriber; + +pub(crate) struct ShardStateUpdater { + min_ref_mc_state_tracker: MinRefMcStateTracker, + + storage: Arc, + state_subscriber: Arc, +} + +impl ShardStateUpdater +where + S: BlockSubscriber, +{ + pub(crate) fn new( + min_ref_mc_state_tracker: MinRefMcStateTracker, + storage: Arc, + state_subscriber: S, + ) -> Self { + Self { + min_ref_mc_state_tracker, + storage, + state_subscriber: Arc::new(state_subscriber), + } + } +} + +impl BlockSubscriber for ShardStateUpdater +where + S: BlockSubscriber, +{ + type HandleBlockFut = Pin> + Send + 'static>>; + + fn handle_block( + &self, + block: &BlockStuff, + _state: Option<&ShardStateStuff>, + ) -> Self::HandleBlockFut { + tracing::info!(id = ?block.id(), "applying block"); + let block = block.clone(); + let min_ref_mc_state_tracker = self.min_ref_mc_state_tracker.clone(); + let storage = self.storage.clone(); + let subscriber = self.state_subscriber.clone(); + + async move { + let block_h = Self::get_block_handle(&block, &storage)?; + + let (prev_id, _prev_id_2) = block //todo: handle merge + .construct_prev_id() + .context("Failed to construct prev id")?; + + let prev_state = storage + .shard_state_storage() + .load_state(&prev_id) + .await + .context("Prev state should exist")?; + let new_state = Self::compute_and_store_state_update( + &block, + &min_ref_mc_state_tracker, + storage, + &block_h, + prev_state, + ) + .await?; + + subscriber + .handle_block(&block, Some(&new_state)) + .await + .context("Failed to notify subscriber")?; + + Ok(()) + } + .boxed() + } +} + +impl ShardStateUpdater +where + S: BlockSubscriber, +{ + fn get_block_handle(block: &BlockStuff, storage: &Arc) -> Result> { + let info = block + .block() + .info + .load() + .context("Failed to load block info")?; + + let (block_h, _) = storage + .block_handle_storage() + .create_or_load_handle( + block.id(), + BlockMetaData { + is_key_block: info.key_block, + gen_utime: info.gen_utime, + mc_ref_seqno: info + .master_ref + .map(|r| { + r.load() + .context("Failed to load master ref") + .map(|mr| mr.seqno) + }) + .transpose() + .context("Failed to process master ref")?, + }, + ) + .context("Failed to create or load block handle")?; + + Ok(block_h) + } + + async fn compute_and_store_state_update( + block: &BlockStuff, + min_ref_mc_state_tracker: &MinRefMcStateTracker, + storage: Arc, + block_h: &Arc, + prev_state: Arc, + ) -> Result { + let update = block + .block() + .load_state_update() + .context("Failed to load state update")?; + + let new_state = + tokio::task::spawn_blocking(move || update.apply(&prev_state.root_cell().clone())) + .await + .context("Failed to join blocking task")? + .context("Failed to apply state update")?; + let new_state = ShardStateStuff::new(*block.id(), new_state, min_ref_mc_state_tracker) + .context("Failed to create new state")?; + + storage + .shard_state_storage() + .store_state(block_h, &new_state) + .await + .context("Failed to store new state")?; + + Ok(new_state) + } +} + +#[cfg(test)] +mod test { + use super::super::test_provider::archive_provider::ArchiveProvider; + use super::*; + + use crate::block_strider::subscriber::PrintSubscriber; + use crate::block_strider::BlockStrider; + use everscale_types::cell::HashBytes; + use everscale_types::models::BlockId; + use everscale_types::models::ShardIdent; + use std::str::FromStr; + use tracing_test::traced_test; + use tycho_storage::{BlockMetaData, Db, DbOptions, Storage}; + + #[tokio::test] + #[traced_test] + async fn test_state_apply() -> anyhow::Result<()> { + let data = include_bytes!("../../tests/00001"); + let provider = ArchiveProvider::new(data).unwrap(); + let temp = tempfile::tempdir().unwrap(); + let db = Db::open(temp.path().to_path_buf(), DbOptions::default()).unwrap(); + let storage = Storage::new(db, temp.path().join("file"), 1_000_000).unwrap(); + + let master = include_bytes!("../../tests/everscale_zerostate.boc"); + let shard = include_bytes!("../../tests/everscale_shard_zerostate.boc"); + + let master_id = BlockId { + root_hash: HashBytes::from_str( + "58ffca1a178daff705de54216e5433c9bd2e7d850070d334d38997847ab9e845", + ) + .unwrap(), + file_hash: HashBytes::from_str( + "d270b87b2952b5ba7daa70aaf0a8c361befcf4d8d2db92f9640d5443070838e4", + ) + .unwrap(), + shard: ShardIdent::MASTERCHAIN, + seqno: 0, + }; + let master = ShardStateStuff::deserialize_zerostate(master_id, master).unwrap(); + + // Parse block id + let block_id = BlockId::from_str("-1:8000000000000000:0:58ffca1a178daff705de54216e5433c9bd2e7d850070d334d38997847ab9e845:d270b87b2952b5ba7daa70aaf0a8c361befcf4d8d2db92f9640d5443070838e4")?; + + // Write zerostate to db + let (handle, _) = storage.block_handle_storage().create_or_load_handle( + &block_id, + BlockMetaData::zero_state(master.state().gen_utime), + )?; + + storage + .shard_state_storage() + .store_state(&handle, &master) + .await?; + + let shard_id = BlockId { + root_hash: HashBytes::from_str( + "95f042d1bf5b99840cad3aaa698f5d7be13d9819364faf9dd43df5b5d3c2950e", + ) + .unwrap(), + file_hash: HashBytes::from_str( + "97af4602a57fc884f68bb4659bab8875dc1f5e45a9fd4fbafd0c9bc10aa5067c", + ) + .unwrap(), + shard: ShardIdent::BASECHAIN, + seqno: 0, + }; + + //store workchain zerostate + let shard = ShardStateStuff::deserialize_zerostate(shard_id, shard).unwrap(); + let (handle, _) = storage.block_handle_storage().create_or_load_handle( + &shard_id, + BlockMetaData::zero_state(shard.state().gen_utime), + )?; + storage + .shard_state_storage() + .store_state(&handle, &shard) + .await?; + + let subscriber = ShardStateUpdater::new( + MinRefMcStateTracker::default(), + storage.clone(), + PrintSubscriber, + ); + + storage + .node_state() + .store_last_mc_block_id(&master_id) + .unwrap(); + + let last_mc = *provider.mc_block_ids.last_key_value().unwrap().1; + + let block_strider = BlockStrider::builder() + .with_provider(provider) + .with_subscriber(subscriber) + .with_state(storage.clone()) + .build(); + + block_strider.run().await?; + + assert_eq!( + storage.node_state().load_last_mc_block_id().unwrap(), + last_mc + ); + + Ok(()) + } +} diff --git a/core/src/block_strider/subscriber.rs b/core/src/block_strider/subscriber.rs index 261dfba08..539de6aea 100644 --- a/core/src/block_strider/subscriber.rs +++ b/core/src/block_strider/subscriber.rs @@ -1,18 +1,28 @@ use futures_util::future; use std::future::Future; + use tycho_block_util::block::BlockStuff; +use tycho_block_util::state::ShardStateStuff; pub trait BlockSubscriber: Send + Sync + 'static { type HandleBlockFut: Future> + Send + 'static; - fn handle_block(&self, block: &BlockStuff) -> Self::HandleBlockFut; + fn handle_block( + &self, + block: &BlockStuff, + state: Option<&ShardStateStuff>, + ) -> Self::HandleBlockFut; } impl BlockSubscriber for Box { type HandleBlockFut = T::HandleBlockFut; - fn handle_block(&self, block: &BlockStuff) -> Self::HandleBlockFut { - ::handle_block(self, block) + fn handle_block( + &self, + block: &BlockStuff, + state: Option<&ShardStateStuff>, + ) -> Self::HandleBlockFut { + ::handle_block(self, block, state) } } @@ -24,9 +34,13 @@ pub struct FanoutBlockSubscriber { impl BlockSubscriber for FanoutBlockSubscriber { type HandleBlockFut = future::BoxFuture<'static, anyhow::Result<()>>; - fn handle_block(&self, block: &BlockStuff) -> Self::HandleBlockFut { - let left = self.left.handle_block(block); - let right = self.right.handle_block(block); + fn handle_block( + &self, + block: &BlockStuff, + state: Option<&ShardStateStuff>, + ) -> Self::HandleBlockFut { + let left = self.left.handle_block(block, state); + let right = self.right.handle_block(block, state); Box::pin(async move { let (l, r) = future::join(left, right).await; @@ -42,7 +56,11 @@ pub struct PrintSubscriber; impl BlockSubscriber for PrintSubscriber { type HandleBlockFut = future::Ready>; - fn handle_block(&self, block: &BlockStuff) -> Self::HandleBlockFut { + fn handle_block( + &self, + block: &BlockStuff, + _state: Option<&ShardStateStuff>, + ) -> Self::HandleBlockFut { println!("Handling block: {:?}", block.id()); future::ready(Ok(())) } diff --git a/core/src/block_strider/test_provider/archive_provider.rs b/core/src/block_strider/test_provider/archive_provider.rs new file mode 100644 index 000000000..3310dcf32 --- /dev/null +++ b/core/src/block_strider/test_provider/archive_provider.rs @@ -0,0 +1,156 @@ +#![allow(clippy::map_err_ignore)] + +use std::collections::BTreeMap; + +use anyhow::Result; +use everscale_types::cell::Load; +use everscale_types::models::{Block, BlockId, BlockIdShort, BlockProof}; +use futures_util::future::BoxFuture; +use futures_util::FutureExt; +use sha2::Digest; + +use tycho_block_util::archive::{ArchiveEntryId, ArchiveReader}; +use tycho_block_util::block::BlockStuff; + +use crate::block_strider::provider::{BlockProvider, OptionalBlockStuff}; + +pub struct ArchiveProvider { + pub mc_block_ids: BTreeMap, + pub blocks: BTreeMap, +} + +impl BlockProvider for ArchiveProvider { + type GetNextBlockFut<'a> = BoxFuture<'a, OptionalBlockStuff>; + type GetBlockFut<'a> = BoxFuture<'a, OptionalBlockStuff>; + + fn get_next_block<'a>(&'a self, prev_block_id: &'a BlockId) -> Self::GetNextBlockFut<'a> { + let id = match self.mc_block_ids.get(&(prev_block_id.seqno + 1)) { + Some(id) => id, + None => return Box::pin(futures_util::future::ready(None)), + }; + + self.get_block(id) + } + + fn get_block<'a>(&'a self, block_id: &'a BlockId) -> Self::GetBlockFut<'a> { + futures_util::future::ready( + self.get_block_by_id(block_id) + .map(|b| (Ok(BlockStuff::with_block(*block_id, b)))), + ) + .boxed() + } +} + +impl ArchiveProvider { + pub(crate) fn new(data: &[u8]) -> Result { + let reader = ArchiveReader::new(data)?; + + let mut res = ArchiveProvider { + mc_block_ids: Default::default(), + blocks: Default::default(), + }; + + for data in reader { + let entry = data?; + match ArchiveEntryId::from_filename(entry.name)? { + ArchiveEntryId::Block(id) => { + let block = deserialize_block(&id, entry.data)?; + + res.blocks.entry(id).or_default().block = Some(block); + if id.shard.workchain() == -1 { + // todo: add is_masterchain() method + res.mc_block_ids.insert(id.seqno, id); + } + } + ArchiveEntryId::Proof(id) if id.shard.workchain() == -1 => { + let proof = deserialize_block_proof(&id, entry.data, false)?; + + res.blocks.entry(id).or_default().proof = Some(proof); + res.mc_block_ids.insert(id.seqno, id); + } + ArchiveEntryId::ProofLink(id) if id.shard.workchain() != -1 => { + let proof = deserialize_block_proof(&id, entry.data, true)?; + + res.blocks.entry(id).or_default().proof = Some(proof); + } + _ => continue, + } + } + Ok(res) + } + + pub fn lowest_mc_id(&self) -> Option<&BlockId> { + self.mc_block_ids.values().next() + } + + pub fn highest_mc_id(&self) -> Option<&BlockId> { + self.mc_block_ids.values().next_back() + } + + pub fn get_block_by_id(&self, id: &BlockId) -> Option { + self.blocks + .get(id) + .map(|entry| entry.block.as_ref().unwrap()) + .cloned() + } +} + +#[derive(Default)] +pub struct ArchiveDataEntry { + pub block: Option, + pub proof: Option, +} + +pub(crate) fn deserialize_block(id: &BlockId, data: &[u8]) -> Result { + let file_hash = sha2::Sha256::digest(data); + if id.file_hash.as_slice() != file_hash.as_slice() { + Err(ArchiveDataError::InvalidFileHash(id.as_short_id())) + } else { + let root = everscale_types::boc::Boc::decode(data) + .map_err(|_| ArchiveDataError::InvalidBlockData)?; + if &id.root_hash != root.repr_hash() { + return Err(ArchiveDataError::InvalidRootHash); + } + + Block::load_from(&mut root.as_slice()?).map_err(|_| ArchiveDataError::InvalidBlockData) + } +} + +pub(crate) fn deserialize_block_proof( + block_id: &BlockId, + data: &[u8], + is_link: bool, +) -> Result { + let root = + everscale_types::boc::Boc::decode(data).map_err(|_| ArchiveDataError::InvalidBlockProof)?; + let proof = everscale_types::models::BlockProof::load_from(&mut root.as_slice()?) + .map_err(|_| ArchiveDataError::InvalidBlockProof)?; + + if &proof.proof_for != block_id { + return Err(ArchiveDataError::ProofForAnotherBlock); + } + + if !block_id.shard.workchain() == -1 && !is_link { + Err(ArchiveDataError::ProofForNonMasterchainBlock) + } else { + Ok(proof) + } +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum ArchiveDataError { + #[error("Invalid file hash {0}")] + InvalidFileHash(BlockIdShort), + #[error("Invalid root hash")] + InvalidRootHash, + #[error("Invalid block data")] + InvalidBlockData, + #[error("Invalid block proof")] + InvalidBlockProof, + #[error("Proof for another block")] + ProofForAnotherBlock, + #[error("Proof for non-masterchain block")] + ProofForNonMasterchainBlock, + #[error(transparent)] + TypeError(#[from] everscale_types::error::Error), +} diff --git a/core/src/block_strider/test_provider.rs b/core/src/block_strider/test_provider/mod.rs similarity index 98% rename from core/src/block_strider/test_provider.rs rename to core/src/block_strider/test_provider/mod.rs index 6a6e0c405..94a4585c6 100644 --- a/core/src/block_strider/test_provider.rs +++ b/core/src/block_strider/test_provider/mod.rs @@ -11,6 +11,8 @@ use everscale_types::prelude::HashBytes; use std::collections::HashMap; use tycho_block_util::block::BlockStuff; +pub mod archive_provider; + const ZERO_HASH: HashBytes = HashBytes([0; 32]); impl BlockProvider for TestBlockProvider { @@ -97,7 +99,7 @@ fn master_block( let shard_block_ids = link_shard_blocks(prev_shard_block_ref, 2, blocks); let block_extra = McBlockExtra { - shards: ShardHashes::from_shards(shard_block_ids).unwrap(), + shards: ShardHashes::from_shards(shard_block_ids.iter().map(|x| (&x.0, &x.1))).unwrap(), fees: ShardFees { root: None, fees: Default::default(), @@ -142,7 +144,7 @@ fn insert_block( root_hash: block_ref.root_hash, file_hash: block_ref.file_hash, }; - blocks.insert(id.clone(), block); + blocks.insert(id, block); if let Some(master_ids) = master_ids { master_ids.push(id); } diff --git a/core/src/lib.rs b/core/src/lib.rs index 1f65cc9db..fe1f80a1e 100644 --- a/core/src/lib.rs +++ b/core/src/lib.rs @@ -1,2 +1,2 @@ -pub mod internal_queue; pub mod block_strider; +pub mod internal_queue; diff --git a/core/tests/00001 b/core/tests/00001 new file mode 100755 index 000000000..c5dce1c9b Binary files /dev/null and b/core/tests/00001 differ diff --git a/core/tests/everscale_shard_zerostate.boc b/core/tests/everscale_shard_zerostate.boc new file mode 100644 index 000000000..e675654f8 Binary files /dev/null and b/core/tests/everscale_shard_zerostate.boc differ diff --git a/core/tests/everscale_zerostate.boc b/core/tests/everscale_zerostate.boc new file mode 100644 index 000000000..6cea5582d Binary files /dev/null and b/core/tests/everscale_zerostate.boc differ diff --git a/storage/src/db/kv_db/tables.rs b/storage/src/db/kv_db/tables.rs index 0d588c972..a9bbe6065 100644 --- a/storage/src/db/kv_db/tables.rs +++ b/storage/src/db/kv_db/tables.rs @@ -51,7 +51,7 @@ impl ColumnFamily for BlockHandles { /// Maps seqno to key block id /// - Key: `u32 (BE)` -/// - Value: `ton_block::BlockIdExt` +/// - Value: `BlockIdExt` pub struct KeyBlocks; impl ColumnFamily for KeyBlocks { const NAME: &'static str = "key_blocks"; diff --git a/storage/src/store/shard_state/mod.rs b/storage/src/store/shard_state/mod.rs index f4a3bdeda..4cda70af8 100644 --- a/storage/src/store/shard_state/mod.rs +++ b/storage/src/store/shard_state/mod.rs @@ -108,6 +108,7 @@ impl ShardStateStorage { let _gc_lock = self.gc_lock.lock().await; + // todo: spawn_blocking let len = self .cell_storage .store_cell(&mut batch, state.root_cell().clone())?; diff --git a/util/src/test/logger.rs b/util/src/test/logger.rs index f7a0279a9..d9764ebd8 100644 --- a/util/src/test/logger.rs +++ b/util/src/test/logger.rs @@ -1,9 +1,4 @@ pub fn init_logger(test_name: &str) { - tracing_subscriber::fmt() - .with_env_filter(tracing_subscriber::EnvFilter::new("debug")) - .try_init() - .ok(); - tracing::info!("{test_name}"); std::panic::set_hook(Box::new(|info| {