From 8742baa78ddc071747e62f3905f2870d2082e346 Mon Sep 17 00:00:00 2001 From: Vladimir Petrzhikovskii Date: Wed, 8 May 2024 10:52:40 +0200 Subject: [PATCH] refactor: remove unnecessary test data Drop strider test using archives, cause we don't have working archives for tycho feature. Fix gc tests. Update test states for tycho feature. --- collator/src/test_utils.rs | 4 +- core/src/block_strider/state_applier.rs | 118 ------------------ justfile | 4 +- storage/src/db/kv_db/mod.rs | 11 +- .../src/store/shard_state/store_state_raw.rs | 23 +++- storage/tests/mod.rs | 23 ++-- .../tests/data => test}/test_state_2_0:80.boc | Bin .../data => test}/test_state_2_master.boc | Bin 8 files changed, 38 insertions(+), 145 deletions(-) rename {collator/src/state_node/tests/data => test}/test_state_2_0:80.boc (100%) rename {collator/src/state_node/tests/data => test}/test_state_2_master.boc (100%) diff --git a/collator/src/test_utils.rs b/collator/src/test_utils.rs index 224788571..b707dffb8 100644 --- a/collator/src/test_utils.rs +++ b/collator/src/test_utils.rs @@ -69,7 +69,7 @@ pub async fn prepare_test_storage() -> anyhow::Result { let tracker = MinRefMcStateTracker::default(); // master state - let master_bytes = include_bytes!("../src/state_node/tests/data/test_state_2_master.boc"); + let master_bytes = include_bytes!("../../test/test_state_2_master.boc"); let master_file_hash: HashBytes = sha2::Sha256::digest(master_bytes).into(); let master_root = Boc::decode(master_bytes)?; let master_root_hash = *master_root.repr_hash(); @@ -108,7 +108,7 @@ pub async fn prepare_test_storage() -> anyhow::Result { .await?; // shard state - let shard_bytes = include_bytes!("../src/state_node/tests/data/test_state_2_0:80.boc"); + let shard_bytes = include_bytes!("../../test/test_state_2_0:80.boc"); let shard_root = Boc::decode(shard_bytes)?; let shard_state = shard_root.parse::>()?; let shard_id = BlockId { diff --git a/core/src/block_strider/state_applier.rs b/core/src/block_strider/state_applier.rs index 600f8192a..c9ee36b45 100644 --- a/core/src/block_strider/state_applier.rs +++ b/core/src/block_strider/state_applier.rs @@ -210,121 +210,3 @@ struct Inner { storage: Storage, state_subscriber: S, } - -#[cfg(test)] -pub mod test { - use std::str::FromStr; - - use everscale_types::cell::HashBytes; - use everscale_types::models::*; - use tracing_test::traced_test; - use tycho_storage::{BlockMetaData, Storage}; - - use super::*; - use crate::block_strider::subscriber::test::PrintSubscriber; - use crate::block_strider::{ArchiveBlockProvider, BlockStrider, PersistentBlockStriderState}; - - #[traced_test] - #[tokio::test] - async fn test_state_apply() -> anyhow::Result<()> { - let (provider, storage) = prepare_state_apply().await?; - - let last_mc = *provider.mc_block_ids.last_key_value().unwrap().1; - let blocks = provider.blocks.keys().copied().collect::>(); - - let block_strider = BlockStrider::builder() - .with_provider(provider) - .with_state(PersistentBlockStriderState::new(last_mc, storage.clone())) - .with_state_subscriber(Default::default(), storage.clone(), PrintSubscriber) - .build(); - - block_strider.run().await?; - - assert_eq!( - storage.node_state().load_last_mc_block_id().unwrap(), - last_mc - ); - storage - .shard_state_storage() - .load_state(&last_mc) - .await - .unwrap(); - - for block in &blocks { - let handle = storage.block_handle_storage().load_handle(block).unwrap(); - assert!(handle.meta().is_applied()); - storage - .shard_state_storage() - .load_state(block) - .await - .unwrap(); - } - - Ok(()) - } - - pub async fn prepare_state_apply() -> Result<(ArchiveBlockProvider, Storage)> { - let data = include_bytes!("../../tests/data/00001"); - let provider = ArchiveBlockProvider::new(data).unwrap(); - - let (storage, _tmp_dir) = Storage::new_temp()?; - - let master = include_bytes!("../../tests/data/everscale_zerostate.boc"); - let shard = include_bytes!("../../tests/data/everscale_shard_zerostate.boc"); - - let master_id = BlockId { - root_hash: HashBytes::from_str( - "58ffca1a178daff705de54216e5433c9bd2e7d850070d334d38997847ab9e845", - ) - .unwrap(), - file_hash: HashBytes::from_str( - "d270b87b2952b5ba7daa70aaf0a8c361befcf4d8d2db92f9640d5443070838e4", - ) - .unwrap(), - shard: ShardIdent::MASTERCHAIN, - seqno: 0, - }; - let master = ShardStateStuff::deserialize_zerostate(&master_id, master).unwrap(); - - // Parse block id - let block_id = BlockId::from_str("-1:8000000000000000:0:58ffca1a178daff705de54216e5433c9bd2e7d850070d334d38997847ab9e845:d270b87b2952b5ba7daa70aaf0a8c361befcf4d8d2db92f9640d5443070838e4")?; - - // Write zerostate to db - let (handle, _) = storage.block_handle_storage().create_or_load_handle( - &block_id, - BlockMetaData::zero_state(master.state().gen_utime), - ); - - storage - .shard_state_storage() - .store_state(&handle, &master) - .await?; - - let shard_id = BlockId { - root_hash: HashBytes::from_str( - "95f042d1bf5b99840cad3aaa698f5d7be13d9819364faf9dd43df5b5d3c2950e", - ) - .unwrap(), - file_hash: HashBytes::from_str( - "97af4602a57fc884f68bb4659bab8875dc1f5e45a9fd4fbafd0c9bc10aa5067c", - ) - .unwrap(), - shard: ShardIdent::BASECHAIN, - seqno: 0, - }; - - // store workchain zerostate - let shard = ShardStateStuff::deserialize_zerostate(&shard_id, shard).unwrap(); - let (handle, _) = storage.block_handle_storage().create_or_load_handle( - &shard_id, - BlockMetaData::zero_state(shard.state().gen_utime), - ); - storage - .shard_state_storage() - .store_state(&handle, &shard) - .await?; - - storage.node_state().store_last_mc_block_id(&master_id); - Ok((provider, storage)) - } -} diff --git a/justfile b/justfile index 86b8df0f4..40ba2c285 100644 --- a/justfile +++ b/justfile @@ -62,11 +62,9 @@ test: # runs all tests including ignored. Will take a lot of time to run integration_test: prepare_integration_tests - export RUST_BACKTRACE=1 - export RUST_LIB_BACKTRACE=1 #cargo test -r --all-targets --all-features --workspace -- --ignored #uncomment this when all crates will compile ˙◠˙ # for now add tests one by one - RUST_LIB_BACKTRACE=1 RUST_BACKTRACE=1 cargo test -r --package tycho-storage --lib store::shard_state::replace_transaction::test::insert_and_delete_of_several_shards -- --ignored --exact --nocapture + RUST_LIB_BACKTRACE=1 RUST_BACKTRACE=1 cargo test -r --package tycho-storage --lib store::shard_state::store_state_raw::test::insert_and_delete_of_several_shards -- --ignored --exact --nocapture gen_network n: build_debug #!/usr/bin/env bash diff --git a/storage/src/db/kv_db/mod.rs b/storage/src/db/kv_db/mod.rs index 28957df09..5e9ff3112 100644 --- a/storage/src/db/kv_db/mod.rs +++ b/storage/src/db/kv_db/mod.rs @@ -154,15 +154,22 @@ impl Db { (self.archives.cf(), "archives"), (self.shard_states.cf(), "shard states"), (self.cells.cf(), "cells"), + (self.temp_cells.cf(), "temp cells"), ]; + let mut compaction_options = rocksdb::CompactOptions::default(); + compaction_options.set_exclusive_manual_compaction(true); + compaction_options + .set_bottommost_level_compaction(rocksdb::BottommostLevelCompaction::ForceOptimized); + for (cf, title) in tables { tracing::info!("{title} compaction started"); let instant = Instant::now(); - let bound = Option::<[u8; 0]>::None; - self.raw().compact_range_cf(&cf, bound, bound); + + self.raw() + .compact_range_cf_opt(&cf, bound, bound, &compaction_options); tracing::info!( elapsed = %humantime::format_duration(instant.elapsed()), diff --git a/storage/src/store/shard_state/store_state_raw.rs b/storage/src/store/shard_state/store_state_raw.rs index 673472431..ee31a0091 100644 --- a/storage/src/store/shard_state/store_state_raw.rs +++ b/storage/src/store/shard_state/store_state_raw.rs @@ -568,9 +568,9 @@ mod test { use super::*; - #[test] + #[tokio::test] #[ignore] - fn insert_and_delete_of_several_shards() -> anyhow::Result<()> { + async fn insert_and_delete_of_several_shards() -> Result<()> { tycho_util::test::init_logger("insert_and_delete_of_several_shards"); let project_root = project_root()?.join(".scratch"); let integration_test_path = project_root.join("integration_tests"); @@ -612,7 +612,7 @@ mod test { StoreStateRaw::new(&block_id, &db, &download_dir, &cells_storage, &tracker) .context("Failed to create ShardStateReplaceTransaction")?; - let file = std::fs::File::open(file.path())?; + let file = File::open(file.path())?; let mut file = BufReader::new(file); let chunk_size = 10_000_000; // size of each chunk in bytes let mut buffer = vec![0u8; chunk_size]; @@ -637,12 +637,19 @@ mod test { } tracing::info!("Finished processing all states"); tracing::info!("Starting gc"); - states_gc(&cells_storage, &db)?; + states_gc(&cells_storage, &db).await?; + + drop(db); + drop(cells_storage); + rocksdb::DB::destroy( + &rocksdb::Options::default(), + current_test_path.join("rocksdb"), + )?; Ok(()) } - fn states_gc(cell_storage: &Arc, db: &Db) -> anyhow::Result<()> { + async fn states_gc(cell_storage: &Arc, db: &Db) -> Result<()> { let states_iterator = db.shard_states.iterator(IteratorMode::Start); let bump = bumpalo::Bump::new(); @@ -659,9 +666,13 @@ mod test { // execute batch db.raw().write_opt(batch, db.cells.write_config())?; - tracing::info!("State deleted. Progress: {deleted}/{total_states}",); + tracing::info!("State deleted. Progress: {}/{total_states}", deleted + 1); } + // two compactions in row. First one run merge operators, second one will remove all tombstones + db.trigger_compaction().await; + db.trigger_compaction().await; + let cells_left = db.cells.iterator(IteratorMode::Start).count(); tracing::info!("States GC finished. Cells left: {cells_left}"); assert_eq!(cells_left, 0, "Gc is broken. Press F to pay respect"); diff --git a/storage/tests/mod.rs b/storage/tests/mod.rs index 094b48ace..29091c704 100644 --- a/storage/tests/mod.rs +++ b/storage/tests/mod.rs @@ -14,8 +14,7 @@ struct ShardStateCombined { } impl ShardStateCombined { - fn from_file(path: impl AsRef) -> Result { - let bytes = std::fs::read(path.as_ref())?; + fn from_bytes(bytes: &[u8]) -> Result { let cell = Boc::decode(&bytes)?; let state = cell.parse()?; Ok(Self { cell, state }) @@ -60,10 +59,11 @@ async fn persistent_storage_everscale() -> Result<()> { assert!(storage.node_state().load_init_mc_block_id().is_none()); // Read zerostate - let zero_state_raw = ShardStateCombined::from_file("tests/everscale_zerostate.boc")?; + let zero_state_raw = + ShardStateCombined::from_bytes(include_bytes!("../../test/test_state_2_master.boc"))?; // Parse block id - let block_id = BlockId::from_str("-1:8000000000000000:0:58ffca1a178daff705de54216e5433c9bd2e7d850070d334d38997847ab9e845:d270b87b2952b5ba7daa70aaf0a8c361befcf4d8d2db92f9640d5443070838e4")?; + let block_id = BlockId::from_str("-1:8000000000000000:2:4557702252a8fcec88387ab78407e5116e83222b213653911f86e6504cb7aa78:e2bc83d6be6975b9c68f56c5f6d4997d2a33226bfac6a431b47874e3ba18db75")?; // Write zerostate to db let (handle, _) = storage.block_handle_storage().create_or_load_handle( @@ -107,13 +107,13 @@ async fn persistent_storage_everscale() -> Result<()> { storage .persistent_state_storage() - .prepare_persistent_states_dir(&zerostate.block_id())?; + .prepare_persistent_states_dir(zerostate.block_id())?; storage .persistent_state_storage() .save_state( - &zerostate.block_id(), - &zerostate.block_id(), + zerostate.block_id(), + zerostate.block_id(), zero_state_raw.cell.repr_hash(), ) .await?; @@ -121,7 +121,7 @@ async fn persistent_storage_everscale() -> Result<()> { // Check if state exists let exist = storage .persistent_state_storage() - .state_exists(&zerostate.block_id(), &zerostate.block_id()); + .state_exists(zerostate.block_id(), zerostate.block_id()); assert_eq!(exist, true); // Read persistent state @@ -130,12 +130,7 @@ async fn persistent_storage_everscale() -> Result<()> { let persistent_state_storage = storage.persistent_state_storage(); let persistent_state_data = persistent_state_storage - .read_state_part( - &zerostate.block_id(), - &zerostate.block_id(), - offset, - max_size, - ) + .read_state_part(zerostate.block_id(), zerostate.block_id(), offset, max_size) .await .unwrap(); diff --git a/collator/src/state_node/tests/data/test_state_2_0:80.boc b/test/test_state_2_0:80.boc similarity index 100% rename from collator/src/state_node/tests/data/test_state_2_0:80.boc rename to test/test_state_2_0:80.boc diff --git a/collator/src/state_node/tests/data/test_state_2_master.boc b/test/test_state_2_master.boc similarity index 100% rename from collator/src/state_node/tests/data/test_state_2_master.boc rename to test/test_state_2_master.boc