From bfa9305da3a3416a7d76b70d94468bc19df0f1e8 Mon Sep 17 00:00:00 2001 From: Alexey Pashinov Date: Fri, 8 Mar 2024 17:48:24 +0100 Subject: [PATCH] refactor(storage): add tests (WIP) --- .gitignore | 5 +- Cargo.lock | 5 + storage/Cargo.toml | 7 + storage/src/db/kv_db/mod.rs | 55 +--- storage/src/lib.rs | 54 ++++ storage/src/store/block_connection/mod.rs | 4 +- storage/src/store/block_handle/mod.rs | 6 +- storage/src/store/node_state/mod.rs | 6 +- storage/src/store/persistent_state/mod.rs | 1 - storage/src/store/shard_state/cell_storage.rs | 8 +- storage/src/store/shard_state/mod.rs | 2 +- storage/tests/everscale_zerostate.boc | Bin 0 -> 31818 bytes storage/tests/global-config.json | 22 ++ storage/tests/mod.rs | 270 ++++++++++++++++++ 14 files changed, 377 insertions(+), 68 deletions(-) create mode 100644 storage/tests/everscale_zerostate.boc create mode 100644 storage/tests/global-config.json create mode 100644 storage/tests/mod.rs diff --git a/.gitignore b/.gitignore index ca739ab20..628f21143 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,7 @@ target/ .fleet/ perf.data* -.scratch \ No newline at end of file +.scratch + +.DS_Store +storage/tmp/ \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 218743560..a40911494 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2240,6 +2240,7 @@ version = "0.0.1" dependencies = [ "anyhow", "arc-swap", + "base64 0.22.0", "bumpalo", "bytes", "bytesize", @@ -2256,12 +2257,16 @@ dependencies = [ "quick_cache", "rlimit", "serde", + "serde_json", "sha2", "smallvec", "sysinfo", "thiserror", "tokio", "tracing", + "tracing-appender", + "tracing-subscriber", + "tracing-test", "triomphe", "tycho-block-util", "tycho-util", diff --git a/storage/Cargo.toml b/storage/Cargo.toml index aea2cc4e8..b8760f1e6 100644 --- a/storage/Cargo.toml +++ b/storage/Cargo.toml @@ -36,5 +36,12 @@ weedb = "0.1.1" tycho-block-util = { path = "../block-util" } tycho-util = { path = "../util" } +[dev-dependencies] +base64 = "0.22.0" +serde_json = "1.0.114" +tracing-appender = "0.2.3" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +tracing-test = "0.2" + [lints] workspace = true diff --git a/storage/src/db/kv_db/mod.rs b/storage/src/db/kv_db/mod.rs index 7e9c434a8..6e2faf8ab 100644 --- a/storage/src/db/kv_db/mod.rs +++ b/storage/src/db/kv_db/mod.rs @@ -10,6 +10,8 @@ use weedb::{Caches, WeeDb}; pub use weedb::Stats as RocksdbStats; pub use weedb::{rocksdb, BoundedCfHandle, ColumnFamily, Table}; +pub use self::config::DbOptions; + pub mod refcount; pub mod tables; @@ -247,56 +249,3 @@ impl Drop for Db { self.raw().cancel_all_background_work(true); } } - -#[derive(Debug, Copy, Clone, Serialize, Deserialize)] -#[serde(deny_unknown_fields, default)] -pub struct DbOptions { - pub rocksdb_lru_capacity: ByteSize, - pub cells_cache_size: ByteSize, -} - -impl Default for DbOptions { - fn default() -> Self { - // Fetch the currently available memory in bytes - let available = { - let mut sys = sysinfo::System::new(); - sys.refresh_memory(); - sys.available_memory() - }; - - // Estimated memory usage of components other than cache: - // - 2 GiBs for write buffers(4 if we are out of luck and all memtables are being flushed at the same time) - // - 2 GiBs for indexer logic - // - 10 bits per cell for bloom filter. Realistic case is 100M cells, so 0.25 GiBs - // - 1/3 of all available memory is reserved for kernel buffers - const WRITE_BUFFERS: ByteSize = ByteSize::gib(2); - const INDEXER_LOGIC: ByteSize = ByteSize::gib(2); - const BLOOM_FILTER: ByteSize = ByteSize::mib(256); - let estimated_memory_usage = WRITE_BUFFERS + INDEXER_LOGIC + BLOOM_FILTER + available / 3; - - // Reduce the available memory by the fixed offset - let available = available - .checked_sub(estimated_memory_usage.as_u64()) - .unwrap_or_else(|| { - tracing::error!( - "Not enough memory for cache, using 1/4 of all available memory. \ - Tweak `db_options` in config to improve performance." - ); - available / 4 - }); - - // We will use 3/4 of available memory for the cells cache (at most 4 GB). - let cells_cache_size = std::cmp::min(ByteSize(available * 4 / 3), ByteSize::gib(4)); - - // The reset of the memory is used for LRU cache (at least 128 MB) - let rocksdb_lru_capacity = std::cmp::max( - ByteSize(available.saturating_sub(cells_cache_size.as_u64())), - ByteSize::mib(128), - ); - - Self { - rocksdb_lru_capacity, - cells_cache_size, - } - } -} diff --git a/storage/src/lib.rs b/storage/src/lib.rs index b1b8d4f48..73b31cfdc 100644 --- a/storage/src/lib.rs +++ b/storage/src/lib.rs @@ -23,15 +23,69 @@ pub struct Storage { } impl Storage { + pub fn new( + db: Arc, + file_db_path: PathBuf, + max_cell_cache_size_bytes: u64, + ) -> anyhow::Result> { + let block_handle_storage = Arc::new(BlockHandleStorage::new(db.clone())); + let runtime_storage = Arc::new(RuntimeStorage::new(block_handle_storage.clone())); + let block_storage = Arc::new(BlockStorage::new(db.clone(), block_handle_storage.clone())?); + let shard_state_storage = ShardStateStorage::new( + db.clone(), + block_handle_storage.clone(), + block_storage.clone(), + file_db_path.clone(), + max_cell_cache_size_bytes, + )?; + let persistent_state_storage = PersistentStateStorage::new( + file_db_path.clone(), + db.clone(), + block_handle_storage.clone(), + )?; + let node_state_storage = NodeStateStorage::new(db.clone()); + let block_connection_storage = BlockConnectionStorage::new(db); + + Ok(Arc::new(Self { + file_db_path, + + block_handle_storage, + block_storage, + shard_state_storage, + persistent_state_storage, + block_connection_storage, + node_state_storage, + runtime_storage, + })) + } + + #[inline(always)] + pub fn runtime_storage(&self) -> &RuntimeStorage { + &self.runtime_storage + } + + #[inline(always)] + pub fn persistent_state_storage(&self) -> &PersistentStateStorage { + &self.persistent_state_storage + } + + #[inline(always)] pub fn block_handle_storage(&self) -> &BlockHandleStorage { &self.block_handle_storage } + #[inline(always)] pub fn block_connection_storage(&self) -> &BlockConnectionStorage { &self.block_connection_storage } + #[inline(always)] pub fn shard_state_storage(&self) -> &ShardStateStorage { &self.shard_state_storage } + + #[inline(always)] + pub fn node_state(&self) -> &NodeStateStorage { + &self.node_state_storage + } } diff --git a/storage/src/store/block_connection/mod.rs b/storage/src/store/block_connection/mod.rs index 50ad26ece..3a6dd2f55 100644 --- a/storage/src/store/block_connection/mod.rs +++ b/storage/src/store/block_connection/mod.rs @@ -13,8 +13,8 @@ pub struct BlockConnectionStorage { } impl BlockConnectionStorage { - pub fn new(db: Arc) -> Result { - Ok(Self { db }) + pub fn new(db: Arc) -> Self { + Self { db } } pub fn store_connection( diff --git a/storage/src/store/block_handle/mod.rs b/storage/src/store/block_handle/mod.rs index 535a62623..1790047c9 100644 --- a/storage/src/store/block_handle/mod.rs +++ b/storage/src/store/block_handle/mod.rs @@ -16,11 +16,11 @@ pub struct BlockHandleStorage { } impl BlockHandleStorage { - pub fn new(db: Arc) -> Result { - Ok(Self { + pub fn new(db: Arc) -> Self { + Self { db, cache: Arc::new(Default::default()), - }) + } } pub fn store_block_applied(&self, handle: &Arc) -> Result { diff --git a/storage/src/store/node_state/mod.rs b/storage/src/store/node_state/mod.rs index 02c45eed5..a9ae5926a 100644 --- a/storage/src/store/node_state/mod.rs +++ b/storage/src/store/node_state/mod.rs @@ -15,13 +15,13 @@ pub struct NodeStateStorage { } impl NodeStateStorage { - pub fn new(db: Arc) -> Result { - Ok(Self { + pub fn new(db: Arc) -> Self { + Self { db, last_mc_block_id: (Default::default(), LAST_MC_BLOCK_ID), init_mc_block_id: (Default::default(), INIT_MC_BLOCK_ID), shards_client_mc_block_id: (Default::default(), SHARDS_CLIENT_MC_BLOCK_ID), - }) + } } pub fn store_historical_sync_start(&self, id: &BlockId) -> Result<()> { diff --git a/storage/src/store/persistent_state/mod.rs b/storage/src/store/persistent_state/mod.rs index 7c1d43da2..9c0e74477 100644 --- a/storage/src/store/persistent_state/mod.rs +++ b/storage/src/store/persistent_state/mod.rs @@ -139,7 +139,6 @@ impl PersistentStateStorage { self.storage_path .clone() .join(mc_block_id.seqno.to_string()) - .join(block_id.root_hash.to_string()) } pub fn cancel(&self) { diff --git a/storage/src/store/shard_state/cell_storage.rs b/storage/src/store/shard_state/cell_storage.rs index 16112cc95..99b211685 100644 --- a/storage/src/store/shard_state/cell_storage.rs +++ b/storage/src/store/shard_state/cell_storage.rs @@ -20,20 +20,20 @@ pub struct CellStorage { } impl CellStorage { - pub fn new(db: Arc, cache_size_bytes: u64) -> Result> { + pub fn new(db: Arc, cache_size_bytes: u64) -> Arc { let cells_cache = Default::default(); let raw_cells_cache = RawCellsCache::new(cache_size_bytes); - Ok(Arc::new(Self { + Arc::new(Self { db, cells_cache, raw_cells_cache, - })) + }) } pub fn store_cell( &self, - batch: &mut weedb::rocksdb::WriteBatch, + batch: &mut rocksdb::WriteBatch, root: Cell, ) -> Result { struct CellWithRefs<'a> { diff --git a/storage/src/store/shard_state/mod.rs b/storage/src/store/shard_state/mod.rs index cde8dedda..5603b4443 100644 --- a/storage/src/store/shard_state/mod.rs +++ b/storage/src/store/shard_state/mod.rs @@ -46,7 +46,7 @@ impl ShardStateStorage { cache_size_bytes: u64, ) -> Result { let downloads_dir = prepare_file_db_dir(file_db_path, "downloads")?; - let cell_storage = CellStorage::new(db.clone(), cache_size_bytes)?; + let cell_storage = CellStorage::new(db.clone(), cache_size_bytes); let res = Self { db, diff --git a/storage/tests/everscale_zerostate.boc b/storage/tests/everscale_zerostate.boc new file mode 100644 index 0000000000000000000000000000000000000000..6cea5582d5fa249d97848bbe894ea66160a878f9 GIT binary patch literal 31818 zcmc$H2_RJ4|M)p~24f$)ESW(iZDUOmBa)=Dwoz142{V>28MB$bMkN(d=J zS<4n7m3B!rN#=jJP#u=ShuB0c2W!Im2ucJa!eT-WQH8ieV4^^VK#o9}AX$(mIA8FRkgSldP#npW zlq5_M))qDvW(g;XSczN^DHF97?Gn=$n<*9{_F6nnLRMn5#AJy;iBFP1k}SDUa)7Kt zP9&$2v&sEZ9BC`*Qt3_^nvAK8t&D@Lp=^k3ne1cPSF$~F{&GQb5pp$h-SQIh*W@26 zELN0PTscZ`RJxM3l9f`N(g~#uWkF?GWl!ZmCYXtYiHeE6Nr1_b8QL?}&&ZuoHshVChiRf|p=qmWzga5Xj$T85M1O6r zYkt{0-~9ee*_pJNF&4&_nwBY+7cBE<5od+YN}AO@Yrx9RYWZw^Yb9&nIg~m1HfwD* z+JxK0*d*GVwYg?<$7aA5*lOD{=h5fSTwrNOvD;yH+&;j;-BH>x!SSw>p0k$oV&`kl zcNQ8gOj?+}NMMoDqNR&U7QI_cU2MAe*%J0rqovoD7A>uDQFJ-tlHzjNrO$P%Yq)Ei zYm;l=GNon3%Q}~PE@v)hEx+TY?N;Z$(B0jg>7Kn}qYaTd+qXy^Gfx) z>{aAdwf?@hiMN%HjZYr~FbbF)<^{iN{$BoD16Bqw1L+$iHayz6XVcEjhFeUw6l^Wt z+8-piU2F$=hgGm0%PC}O$hwe?A>ko0A)Jtmkb=;nunXarc4q9d*{!wPaQB5hm-c4t zE!Y>dpA|uhl!!bLnGu=K&SzH~xO|{EDl~dejBzYImKnD>epEc`VEDnJLzfcN6Dkfn z9%denIAVCD?dZy*-baIu7A0CH&QDyPm~zbaSZ`8b(vBo{Qr&T<09N>UcLN zA(@_BkX)ABbb@k%bK>lYoRi>WYKm=2MasjJXQw8fNQ1Uys&=Y%YR~D!w9#o= zX+~)~&VVx_XOzykpD8-iaW>+d{<-XP1?Q^L<3;549fZKBPZf`Y`Ze=)o4lKjG^I6VHx)P4 zG_^IoZ0cm+-Hu*N!wy3tGw)D1~w$iqTZJljB?Vw$v-M9UI z$K(!nM`Fj>Cn8V0o|JWJcN%qCbvkvf>@0e!{xtMy$J6eo1J9N}%YQ!l`Htri&yPG$ zd!GHg_<2p2ZdXiK&kKha?k{{_+q#>EaK z!LSp!x=TFRn;zqlqqaot`Zm^QE}<1v?&qH!j5AcxbvpD=6K5T zR38R~_@-}>Bx7M#*YYJf$@yPPrf)y>z~zy<9a!LT=i$Np<_`XhK`}FqhaTL3qS&L zKpv9 zzvYH~kKnx?-E1HO+UUkB4fW6sRG=;`ttNfQr4?4nJB}knE(qN@D@Y_iO>&L)B(dJ{ ze)2&ApMlI;3Bga5)}@L7*n)Y|QkS|1#y@>BMZ8nhXoE>gqQjx}mJ$IkWE6}=^Y$D) ztOcEM-QL|Y#xWb+HZ|0Tp6lJOQkL`d=|0FYSQ#2Jpt>(i%vN1 zv-Ll*-f`xcmy2wO7vuIIUT9lp`bqx;x*-7`#wXwC348Y=1W_5F`y*>x7%m$PA0N}6 zCUE9(9#qc&fanmGe-of_-R~yUMo&#pf z1Gj~xD7#m1$c$v(j+;3Z|-%3RJ zgz1DacXID6;k*b3mtr`Reuf1L7J9y*cjNQ+0*uDzO^rWq6nx$u-`_r=!S?ArVXIZX z9dZdJs&UTdn_}kge|uF>VfvnOp<+yuZeV_6wQm6iG=L_;fvOclBxKT!8Lzqyb``SL z8M%xBr=F9Vy%+3Ou9P}vOdBx{XwdC3eR-sf{m-2qcAp5wV3MHjMJ*hK=29C>Aox{w z4|@jym~_Zt`cHM>R)g`DK&AkU1>=xdOyB`Z3ndiV9?CgHdt+HuFRIp-(&HY2fqF#> zIzXpt)oOXw_R={$$?9LA6WQ_$x#t8Iz*;q(TA{jX_!Xp-maCQX?!IvX);J(tvoUbb ziJ~0@RrhsgR(zP;qY>#By?3eHEcW3E1MyD{aH(kl#>1I^D++;?w4^hWR4FRHA=mDM z{1%qF?6FNx_sEXg5~iwG0>)pr?I;9fy1rWgwF+queP95O=PwRvI!=ySwCT9*j*x+hy_%3d(&U;eE8hjE%nyN1{)DV zW0`Al^Xa5Lhb~V@kP|7cEQ}U!lfqqF5{N$?K?E^Vw+TWx!gF-im60E}c22K}!i1Brw8h}j+@4j79)Eee z$9tJxSnf`btX*e?j@F%+6>_x2(V;GWwY#ja-IS=G=WMur2u$Fy+|L_J)Gb$~zu05z zd9h`luBOg{6;;u1#*SMvTeKB!-!n>~f#Pa3Tr)pkgTNZ+-oMfHMT#MxSU=qc>G9YX zpwJCw08?NFWqF`CMW#2mr&JWsfwj~-=d(Xm{^jl<_gP|)0?ffofYpGkZ$mt<8f^}i@xdk&fr24;c*T9 z`$>;A_x2J_PB@xkP9mItw(lFi?yQ#<6IQL-vVw49TkI;T*qZ(L&FFT-`F~?C$$(jZ zFCJC^DO*`0bK3NKS6oi<7}ZCku8%yW^W z1#9BAGv@YDZUcZRcR0t+86tCK`vhe>Ux(VAsVAGl+{5jY`%@j~zwb8R;_$vnHHLE|8y5$4M)l!ovhZ)C+!U?Kj;B?12Nw`ArK@L!n+`PG+KC24D2o zi}Icj|G7q>)Y}|HS^@hCTC4w{7S_=7DzA=F4YHFs`pSIY!L{SOU5=hQ@Z90j-i<4h zkNP_zmTf#d+?FsxmUTuf+avp8g^adMZp!@5=6g~P-;iw64bSacqL4C6n$8g z(#D}@x|-R1{xIO2})=`seCYZ_9bsluxs-09{ zv2W2~)hZS|rx(cC{8w}Fr}Jd61}5h{i>EU;7+j60Tx2d^)H`{hV)J!d@A!msQ^c1> z7#0l(-y8ACqkimK!F)Pk&nwEX!+8M)(7GuuLTGNq}7*l>=mSfYGXqyijIpjYf zX0YdceYqA#%sM_|yl})c!qlr#wu=lDkBo8L`CPZAWhQp&;(TH>(X<5HxSl2Q8W$nn zHE9wOsoC!jzwAkgdRTR`3_G_~sP#V|8w)cTzV`lYI(+*#-_pDE>&WGFSNU+PVgK;h2* z(jvx!ijB}~br7Jy9l9sv&7z*)N4q|Fgy zQFGoHvuMUmRQR}RywlN>tIzFseTl!4kfR)wF^Xs zxdR>j)6lO z4)4^UH7>@jL^J!9oLMF(Ex?ZyRjb2RuRBgcW930NQm><6J3rP)M`<18TBY~n<5MA)$ac_0xIk-$kI zrZk(}#hOj*v%-YTo-E~|~=I0)s&xrwp9wAs7dh;tKVaoc>D zjzZ3;S-~&BN*LYg80RL!`rK}1N1jR^2ezR0_aX_T&&n+=6H+Ya$ydmht4Ws&mGfX9 zhW^Uc2RU5xh(^;Eu`hdNC+LJe7vXPMLFnarU{+W{33ei^fDzYA<^ zY_myeFIZeDkDN0KI}GoE3rGXOkP=G%5NK_?WN;{``(QuzP=|_>ESpUWC8LPk09|Hq z^7s_jOSIEN5Iq^m>;rDZ-&JTg6zBuh5CM|w1t=thV1<&uFak80EXC4t0Xt%)ydAL* zlP>p?Mt-8%T^9W-Kr_D|_%gYA6yI1kBC`$oojgR`Fa^uYjCCV0+mM4i%vbMUc@`(68IHk zh6zUNcEC__&zewl;bOs1RJqZUpF^(?QXNh`*cZEM> zV#?CIOsa1t>OfnwiMPFjTplq?79A4s-bK5xkM$CJiG9FAzuAOYS_Wi#K`)w9L_>TW zbfEb^NjO13{HRDlyeY}|qX~TIb~!z#sLdv{S|JuCOFE;#vFA*!8tN%KWZb?Sw9l(G zqs{XCBGq%k8IgY|`C*^~oioHt;4@kh+YeRPd&AX+PPMxcz<#1#vx#Hupm>=s66+?i zqg}We`F!5$bEYabo9I#tVXUGXsg2@`C^--8273tkjv9mH80SXx32sLnf?ogv=qQ(W z*p1+W1)Qk^Sd6}Yhoc>fj5DB4oSTpjnj8%zSm>VUoP`sGm~5|iNz+abaV%XQCrVhv zi`DV*YlHrvA6y1|G3^+dK6|eP>Yt7r$Ih~2mW-tpqPTrX?y0aQm6i+1R!H}Pa$cDk zcBBe*Szq6A)?mM4-2|EKI5zq~4XAdIM+`5|aj?rK**hm=rRBVGGsLgbWq^*?x6n9; zVP`REbX70e67lX5Br?Cy*ZCkVuJg_8 zg8-O&al-LX17Iu`k@+#=y$3*5ENSC)47yFnk1ZNKj(cIH;0p~Q@ix`05-ysCV<3g_ znL;!b>(#&2d#fVwRWfs2R7}O1Pbn0ckgidP zJH&cTS@;L_!KuM_wZbiQ8jevY>PJONeNmzjsaV>{Hcqu)m_!4liDiZ1^2O5O5{{cT zR+65Ci@C^Nnt_~E5ODj_HfMBe5WzN|RIqk@rif3dJ3epNOdyE3^LOZ`1tI5hKSD4 z!8xZV)6UXD1|$G|CdQUQ%~Kg&gsu%rBP!L99-I1fk z5&ixDx#hinCyF6zM=Kr0W+tIx#~*F|V|oELYA8Q|-fK z`Aj8UZdP&E56N~dK_njJ%GJ=A(A|8U+6%xz4b*H9KEEbdtywM4Q7(r)S)YJQ&_1Ab z4zoX;Y&V`mEG;)f8V(1`zY*y!CyD^SLrjQ7ZwD^Yvv9uTfMnQWp@=OAGkh`Ts)T}C z*bH&zOWu1t{7Ig4G0bY^soWR>c9jBps-{_fC2+n+wtiv4~ZaWj*Q( z9vRW|pscMsX$IkFmq&e}ijh0npmakKuZ@&1R=Tm0B5W%5(6Dmis_;j;x-4SeZ(JZ8 zp~h7xc!X2_32=h2#jsNCNyT0YX$GI+OA9WSRG5<9*4>`st#RAdS-{?~Qb0>iHT;1J zY7W%Nsenx>d=5XV=#t)sq{vAmNENBRF)qw_B)?%!2f-2|j3HeeLu2An-CK`R1lZwZ zq^Q_CX=IINlMZR50#1Qcpe2CjS$1%qaRY-N$At-qD}vcq$0D-h&sZ9U3t=6;6pque z*W<9Goi>rqgxh;^927WnGY};aIuIoh`LpwkW22fXHtUM=QY1K zm-MIRb~`EVK_)Y5#DYT@YD5p_^xQxKT-Q;0kkR6h(Z@lIU5ZEp!#M(I4^f&OVTj7g zKpD7;Q^z@wju(Z1FUrq@3;0nsxQaPuTf3}@I{qfJ_N=;{`@n&Fy(v=FESEL=W{S7j z0G~Fx!&q0k;Y!zg;WUs7_>^`LWMC^YP8e1a#TOO2mo%?RleOfa**c2E^T~qjN`*pF zRdI8nENCuZ@n@#^`caZQu4QqcncxyOE?t84CdoXk4|y^r<~~Vt6b;|g`OPo8ICs$3 zDLjx}vOsGuoxIg-YE9H^s}P?gnX?&F!?obm5NLha4>ax#FeKy7OCDy^Y~mB9#+KGZ zvSJ70lrA;ifxy79R+#mSrbj?3ltCgeh)DNd0i$b$V8NuGy>-3FN2oEY29s;hiQJyO z$X7seRj8JSvtO-RH3q%FHE)18%^X z=O*GNk^luKI%FJ?$GQm)&QUCyBSdw`_<@AecR?<$Q}e-1Obb;)&I%$+WJbi$d=OcB zufPX`ddT-&2IW96u^MPx+fg@9lJ0<(77-nf!RRtfzD|0tUC=0 zxE=o1wBXs#`W#5@tSKC%iB*vzMj!d^SyaL9F~G2!GG1mfMcH%}!3go22}>NeISt4mjuf?qykp1n2I>I-eTaXGv*clBjV<^-<0g)Y7CjWu3{M zMT+1S1{Z^3a2rsDjAOj_xSFblJ{hOB+3U}M*2*an%9|0*2p=juy29+qu2K%t=!j&- z(eqhivf;`FaX;wgON5v^>YQFy+jRU^vF%WdQuqxQD+4Ss%?K4E4xr%6Flu-&dxn`@;ZaC~O*IFCQ53Qk)S<9r40pexdrR@NcLGL2U-M&khpA=VKIS8G zOVIn9TlrOTl#xh9i`HoHPd5ydqek;Se>yFBdFHD^-Bpnqj`d&HU44;%{_!5Ob5Lv^ zx!~Z)8?8ibpRIlNA$|#|fsN8JB**AjI zEBYh7_ov;`Nk>a|x4iAXI-{9+03J$9#m>nxeKgIm4Z5$)ADF^9Jv-5hr6UMv#baj1 zJ#AT)iZ9>liiX5#qvcmoD2A4_E=Jl-eCaBru%zY1#0?iuP1tqg^bLbY0#|CS`kXFs z#iMA=Fli(GpU-5y)Uh7nzAi;ifeE=cmdUJ6lv{N8oQ~Mu9!k-@_%N5xk#7}%RDQhk zu$!HhG{=qd&w)0hnAXQB!;b_Z{x8?j8gRTf^6}n;*b5iZEGVyL<$A z+u+6fa=MV4Hpe|&?r6&>-)jXS`E+NZYQCjOT0Vg5wC4*wP&F5(|J<@Ep;+8r zZeu06;12+wq6AJ`E}4oH#CFE1XqR>eO}TLx=$eB%wuL=hJP#Ao}Vhk|kgY&BY&hNH9jQibOfTf%%E>eQ&!RnujVYLkO)=qw&M&tc2x;PHz$7T8ar3;e3x z%DL2>cyB7|$zbJo0y(id-J)%@9(O!o{{p;3Bm^PEFnABV09y>`2%WB1jgBoA)`+7W)VS^$P}~#7L&mU>$hmO?++r zzF?xqp+X~YxumzUloKu!ANOGL$!M`i1AW&WVx>r5sXf&4c8x`Xr% zT$~6tu~vmsgh3FXyN6Fd7F*LsHo=U<0RNzQ@SxN^Tsb~Y_QJJ-ND>9S2R&d6PS3Jp zZKN3j{(6J{9|fA~5SmSff;5SEkOl>;7TP)RBQ<1=ZW8!_6{83VBXD)j0bAN{p%Y#; zs)4Thbovv9!%Fo^$U4W1gY`2v2tz# zZ8}tzC|3`UdKfsct|zz&hbkEFK5(ebE+Csk2owHdt{W!IQAiU^#iY~ZIF2w4^_7FH z4%~XWN!B2bX*RJW{O(E|*~(3@qXT?F z=G9kX(_ciz#v*BS#7%TiW+B1jsF=r&(9uPLYEU4_#Y~;rF~;|ZoA7T{LIy{G;$|W~ zI|k!sb}_hzKhAD5IIqO zht6;P{dqgW^;j2F-Dw38)s$O;C@qv4(myKam^YQ*ZX6!p>_Ab#XqPs0K{1Cdt5fw0 zQ3XT#6>?&$xLS zqA0FU+bK#R;P2qk4)nQO1qI++wwklXFNgt2Yra?K@@r|n?w;?&>xWrZEtdQ9&@NB*~t@89B*DQ_mr?ma{nPdKEv zLJX4NvK*)7b@RjnOF@PrCTW#CFY8Y6tTA)(fW&|RajrW{i*^JWvy9!T+9BFNX%7^@ zK?*`4;Dk>?2#C(x56@KzVPkN0S?y10-T3$>r>_Gkso+hzP$sSp?;!pxvwI`;; zzIK(Jr2pXZG-i6)>4~t6+%o;zz?UH9Xz_iVE|lRgBC8fEpf-bx;EjekRh^CLxu&HB zn}S=H=Gkzx1eFGK%TBG@u1*j=eo4r}=Z4MXj;Q6sX{M#jU(~ zL)=Q?2?Zm~-+(f>C49Ucw7k~r^7nqYT+PFM`ptKfmY=edC5Q))TJ4aTA1(dYmM|nE zOBhPwmN1m&dm!&G7k8{Bu6n--zTLsToSf%%nG1Asj$BqjF3-pGrzasC z*jHj&v&`q~Q|gOj5(=#SXPPcPeUnqBLyYej#>axPI6yf*KzY88^*YDyo0OvD>?3vN zAFCdPIx|hCwCHyExE#8*@LjJ_RzE(#j^U?QcbEYx;_ycC;VI$pTK(UZ+c8h?UNgsI zUfk%*j#u-{)Rr_hP8yT`c=y9PFPR8T27=ZVH5yk@2pX!w(U^=KU0KcB@1r2HmBvGD zG>biCM@{a8&)0)fMpn zmrnTvTHm{zsm~A>pq$Za?f*1OzDf{~5ne>s8zbisWyeE(goos6O;qrvgwJ5r>?D4pYu3@L}=ZhGefN+zx zb_UNz?||s?@KY2AV=Qj7pw@;*YKQ$6U2t|iHEA_%#JHJ)^3A{eOQyFH5EtHuJYTNI>5Ik_uY^Ff}1lP*dl6J*t zHteN_-2ltCDOBr7+Rq3dcAp3hehP0vGe}2xM38so)uXaakA!Bk{#QtIavLAE{~`>q8KdIh)) zw9DhGSY+y;)n5Uw57(RP&-LRnxxU=>T(4m63a%%26?Y|DuUCM{^(7KNdap!J9ehfxYttKr^N2 z-cq65{WwQ`LTl!@=!^4@0}%?G18t-a1#o;-6J?n7dBx;~8(1 z%~XFf%D*u;Cnveuw?PmMKYX)i=9ioOwR`e1d2S(qvoO(dDRY| zyQeEihqWhb`0l9a*gLG)nTb!l1s^q*IBFVXHd)C|lr4L*tB(?ETbky+Y2!Gd(xjy) z9&9(ABs+Ok5DqZs_+uNMpOtdZu@Bwx+p8?Xn{GXNA>wzg&EmzzwSDdgS zkLSVpc;O5eVpt#_RR8i5@byY4Z-62CE z&miofKu56dG^WB7u9+k>1FfuOh=EbU5|c4iX7v{dIUN-sEFnhL)uKwGe3It!Q4&B6 zsD*EQA>IXH8nlP~!*GZjmC4j#sOumvMxy5u&L>{#Xtnkz%8jAK)6`j%>#b~kN8-;l zDBO7=5cEn=qIFuc*s_j#DF#&TJZFfUk0tm83$9NqEv1X*D19TE8m-;|9Y8wQpX!f} z4pC2IDFJ7Q9b7*u*Vj#uI}=oV2DQhbWnTRJSlq4 zm>Z!qlq}N7VTY1g2;-TRqH0ve1ZR;62(UIf7+)zWq=o*+Rf_V)qH;{7yk@0nutue* zqLrIb|>G?fv>oux;Ak&X6rCx6nxA?)*lz6Ou4hMFq`#xJtqlQqxbxItKQb=m; zY1;yq0x?8D2)WoJG)nGVN7`ofwDTc=mvKc%M`y$Jo;E)&aNOnZmq|TF^A7>6Ozt=a z`AV=}t0U%2bt7;cC*QBfTDg-|1_CPS_t9D@%d3(;zN+*-q*m%sD-CkvD#I$J z;wnQbWjgL^d)#l1tAr#A^U#50su}v1>=H(2n!6jj zpLRd#ZtrgCuJ5knb>Hf~*Im+mJ1;f(WcNOLi>7;fu6g%SG=MG@b1lLJc}ey5VvDB* zr7KrKHv~qLAoafnP=32kFXc)wdkWLdJrb=v|IJ2ZGE46=8KTS% zH$;4n8aG-gSurRS$py^Aj#BY73QOyHD_jZBvK^@m1$BnJ21Dsj1f|Xx^%o?i&QSac zxO3GRa#X*~8fodtIy&6->iq7z(74{v14+m2Dcrf5zpJ46qr%``(0_NN*i-{OM|CvX zSvOjB;VQuD`cDM_U&Ol!!hW_hl_5n%lSoZh&(x)%lpuRuM=3OY3`bowTn)yA(hIzxR4(s{jriY}7eNQ3xb{4;d?5*5$~_H)gB*;A1% zkH$S>Bfhm>oEKkLjqZI+DS@$Quk{D=#D(}*gdSHjhfxM_9h}Tp2$~FX5uhBRp^`

d+ewxCtPQPF0_N;{QaW--SLvWKiuQ8-oe` za8n~3?!=JmJDQom5Q@9cK{VU4qGsnrNkN&t`JeW^N!H?_3 z0KSJF$gBTBTy@pl`3112&HFozacy!CJ)!*L4WP>@VCJ91GWD;7GSc$`{|&S6-!W7* z6G@7nl^RZayV%OOQv0=Z2|mJf!*4?3h1n5Jft%nb_6A3dT=sNF^AQj|M5&x9geAzE zuwY;Ytwum-HXv{NMXM zLK&r_UO*Pi%77VJ|H`mD(bm)|z&Bi9h-)LeRf0FRt*!mGx;YH}mwKzN8ewg_ekGf*CNsX9{XNU`&Dbbd!U`s#yD zdr&-Xj|9gunaqwsX2V@$7tnwnQQ=Sge>CYZ3`G6699~=x?1H=TA`Y7XKVQ9|VqyAk z+~3Q~P@0cLI3<)&6=f=$FY8YUABEbagi{cqPYft3xtmo~csBt3>lIa&ekgieijGUs zaSsLVLFt-sFWd(@f8qT9wb$z=_}@?gLK!ACTi||pC37wJ;=kg6)ZjDUUAVmO9-R8B zyr%_s;Q=^+E7-vQi!f{}*D9t*4@SVVv3^|7j_)>Ktx)w3H9Qkn!|34vzEOwDA)^ji z%zq!Zf{}fdP9`3KG?j0MKS#2DQK;PdR_{g3`#;1{C+{6lLq);lI3v1XJKCv?$o;GDYX|Exq^3y58r{M;SYxkvJ(q&I)?M4tqWW z=I7H#yl|X|8f}VF_BnT^Mw&VOx&NvxX6h4Fs~L8Wj@%TrE#`exFzg<2qa3t4(HEG# zeAT*LYdQ=)8eLo{f#)1ltyHLbMlI+4U*ojA^4DLTF=eT{j+ zwWk4RqPOQloq-vimyM*>5?XV}@%Ac#-|$Dq`>s$7n-@8zXtr&--&x|>KJSik#Mb2N z20P>r@8!Lm>@#dqK{5{T1Rvl@9AN7lR%Gc+{T+E$%#-Vjr3|u#BPL&)5M%0gqE6iP zkuQ7T);wAnsqb_wr*sUwG)CP^P5z->0sPUI^aVGP}}Z+IsP69?KbNo7o6!-wqp{h>cw4vC->ur_qs% zUYUWuEfe=R^v_P}`25j$?zWl3U@Z&B+GRf0uHaZ}={!hE+3%S)`wZ{H94(KTH+nW$ zbF7y{FK+kV(P{DWfHw~ByzXq{>m@Fe)vtx!E}%qej*(`pU7FDt!aliWT50~>nqlBw z#o=A!!@JI(rGq6wFT(CR&0pAi=jzJ}wWd!OpN6jQm3_FS;EnfraJ3G-{3JhO+vfXT zcei<7GHm|*b-hujkC3x^2+eBNmQ}ZS?evl6SLLE6ucdukR(f?!VxdTh^!`P9Ir?Xq&!jk(N_pW;R!ocn|R4>|{{mwvDds5o_ zHV2dC86$4A0tx#8X4;AqmcHz$6LWf|zK3!04VK;!rx#7^Ed{Gx<#Z1tnfBLNC3kVe zmGKc*jw9}|z|x3y>lFIx$IMH760!Z1%YJRRXt(FF2K6@6NFi4>JY{X5^lX0D`2&e> zq)Q&ucqKPrvvx}Qw0apkx!6A`NZFi+gxX&NyoUq4&j(nE1AOF`&h?#5Xws^k6JM#} z6Khi;pBQk??a=LBfqlnLkx%qXOC`P$s8L8goRkuApzX|e)ZD)pAY;(tTU+I# zDsdrklUHZeo{U?Pi=T}=a{_-g+Xsbi>sJwiebru3ifaPyitGr9Z+%GPtdh>1%iEN( z8HQWUt4ly0<3HQTumPc#m-)V{>2r9_lEw=wJ#UwlCuS>runIRR>UoRvj;hdkM6^NLQm%nmA zSk;g9c8@7uFD&Duiv1<2PC8};N+P2zU&p&Kl@ZCA!>~2 zQu9~$w8rkyDSamo-&Kt`ldBbH-S7FU9qJG;3rZYImslPtcH-SnEf^!ex)=>AcvP0ssAtRbbG%L-bZ%dEE%;E=rf**= zzP-J1e7XDcQ6o=8?L>;zq4IWOIVte^9**}S8C6O1k83oqUgJEO(902wxSsAj($^)z zE*vv%5si^zAEvl1Gq5k$?!NHIto+UNf&OPZJ};}hNDGMQ7>>N2;+T2H$INpaGY!YG zd#z8~_dWk$e$$~u_0evfSG6StFDnctz0}eQ2_ja`fF3@7Ym@8N*L>wh6!U@Iz9CTwl|B8 zUVgH3>-7z<6R&dw_uf6Stugl0q9`h!o%IoZf~7wvk&PQ%f8wvdp+NVK+%d)<^TxFQ zlsCpid1J$*i-Buw)xp7o5JYz23zy`F!d6dz#l;a$L}(n60oE5UMEX0VVhuauMz@GA4zi z@C*JpKLvh+1Hh}9Ni6}#)T;vUw-tZ_7!}pgMy(fDi)we2DgaS!g#`3g7ZkSqmYW&X z_Pryp`Fozzz8l~3ob>T`V*H%mq&)E7OKtj!f;9ifNj&`^>91pXcoNUVpCc!DjAJlG zX#qby$!;jX^F1+%pS6?sJ!@wqSR6^B4iBGnK}-m~MsXzM?Elvjb%q-v9=`(rqwE|E z{-;?v3;x5b984JHw1}`-Pl)KTR~U%$f!z5C6&yUF;!g|06s88#ZNfqs$0G}+7s@e2 z-KdUHL%Adv@qd)dfeC=z|5+LYfhy@H3C~|~|`sTd|dlA-$ z-hkL|*l!p#wSQ`mLBD}nKdoOcNT*+ivF>`XNQwHTuGD+ zH1I#;f?ZE<=@yCNtOTG<#UVz{Qu(##ITK+2BwJ%5N)(x&R;af#-P&v zMm452m_D`q$Lj9?#6KEb$NkkBE;bhSqbK}Zx&LMz_g}5yV&i~6dIo{Z_~r8ke@7Yo z*XbhIco6ri z=L+~4W+UhHd*2r_M=XFxwswh41j1KcPwP_tvn5Py5}vU6UtPi+shY+nV^d%ap8ZBg z*>5AvFaBNL4W0srXR8mJvW3wA5m^)7BVX{vQ+P1KJvyd`>4PxQdMEhp=7ah?5=1oD)fIj?d-}UqWeQ0ohS)j-kW`TFW5}O6& z@&O>^W~~t~bhZO>H&MJJH%}?<1R&!Ttg$(GVS~+;hMwm|-x<+0+oCt6P4}5_hUMFw(>{Kz zZJ_2895*Ylm3YIp*gU+j#~kqC&&R*)@ZK!|SF0SSEQMcL7?Fw86m{+6^A$yc3y(SL zmt4r}PO1<)I*Vyn1P}EQ~W3Z^TN=o0=gYP zT94t6=wUvCP5WWK04~N)3|;oa{Lo8`0eV*t?8jgoej1>wSTs_Mn}D_1;@p=nK zDgZKj@Gjv55s9(!;YDLH_{|6KuP89{CBJ(^y&dZNAbiLIxWM>cWvEj_orSwN=*FQc zc(bus97LpX2yZqX|2haXiy*3l;~M?*?-K&1^^gF7P!#~CEOY_w^E*!j6tsu_aRyDm z+dGUM!Hc6<(U>LGOA- z#Z~MYUYx`5_kCiS*d@F;k6plvi&zG5k$u9M(~%@N+h3qd;9YyHjnoBr`+f252%j}^ zcMg$1ExMFh{?da!2$ASMR6o`_H{Y6^yUro_+f3hyVj;fM6b07Yq{>}9Zikr!oV`C- z9sH<&&3GkmT*|j8-3A631*N{%_gThu3QS2`s6JJgZTv1Jso)Ro=k2$%GE!LdX8FDD z<+1OhyhiyE?}_&eM7qz_3EYz19z_3MA0I{*b{Q|OVA(^xODIXvAl-SME0pN6GI3!- zY-4`2`=^{SjTgcSns}mFB|q*Ss=x2;4&hd%sjo3$d^!KCb?2^#?juWXf4eS3e(_}1 zSaQoCu-`^=v{pd5;s-Ps=(>nhX4?Kz7wW6N&qkViB*U_)|qT(i2fEPC~^omdlmWSmJ z^)-H>!nV6L7o?u}tw|`T5cX6liIzt{e7VjvBg5iWQ0+zfU}HaE#~eGd^Dr$ZD&S4@ zY>ao08|e`#GjK0^{^q%Ii!l=?^C93r>Stdr^4(jRdsy1bJto4kIm~RbxzWVO&n(#w zv-K2%D=B}hAHC(n=#xOHa1_x|9+D8!2J z;ucmsG@5fO7w=@q-P+00ey?(RPR1$&*2Z#m(WR|#L!?sLKh%Dsf4_UE{)vuPUGi18 z&sPYo%ek_*@69@U?y4Mt{PWwy8LCGD62*T&8>*j8P5aW>qODQjU(nTY_Oozo!`+Gc z@_u)MLWJ5C=WPi5WBajMSHA6@Xmw|ORe8X%=~Gl1*0zY2#O@4wn{UgT@l1;TqkViB zC0Hq5Jis2}#a*lnFDkJryeP*i@ZuhJf2b!{pKUv)Gyl4^Ud)z=2@^@mqYiJow1++; zEOMgs{SV$uF&5x&Kjz`!Sg}l>C3nxY3sm={6Q5+>SafB}>Ujq%oJK!5XVA5GSHU0Z zAG#6qU`5i{lbrK8oPCAw!_MzmoA;Vl>g#lKdehkUCgLCK$6r1;%RtxsRh796e0Hzr zvtv|Xz;pTr7r9oq^@^%b5(k?{G>i|U8mqyJTC8qpG>5~a?grCqW4w2_vfY)0*7?8m z42nJeeDp{43%a|Hj#B+2v|Y;gJ3Z6WjAqcv-a0lZt1mlSByE!;eK1LSdz7hPOeTF0 z+>g;`AD&UM*+0O@#MpZIc(oi)vkeEyRUae9?z%E+lXw~f{IUJWmF2OP&mWJg-7|k? z?ztMh{rj879$qXqL#x{CX3wXaT0h#yhf$9;0GFD?Kw%T-) -> Result { + let data = std::fs::read_to_string(path.as_ref())?; + Ok(serde_json::from_str::(&data)?.try_into()?) + } +} + +impl TryFrom for GlobalConfig { + type Error = anyhow::Error; + + fn try_from(value: GlobalConfigJson) -> Result { + Ok(Self { + block_id: value.validator.zero_state.try_into()?, + }) + } +} + +#[tokio::test] +async fn storage_init() { + tracing_subscriber::fmt::try_init().ok(); + tracing::info!("connect_new_node_to_bootstrap"); + + let root_path = Path::new("tmp"); + let db_options = DbOptions { + rocksdb_lru_capacity: ByteSize::kb(1024), + cells_cache_size: ByteSize::kb(1024), + }; + let db = Db::open(root_path.join("db_storage"), db_options).unwrap(); + + let storage = Storage::new( + db, + root_path.join("file_storage"), + db_options.cells_cache_size.as_u64(), + ) + .unwrap(); + assert!(storage.node_state().load_init_mc_block_id().is_err()); + + // Read zerostate + let zero_state = ShardStateCombined::from_file("tests/everscale_zerostate.boc").unwrap(); + + // Read global config + let global_config = GlobalConfig::from_file("tests/global-config.json").unwrap(); + + // Write zerostate to db + let (handle, _) = storage + .block_handle_storage() + .create_or_load_handle( + &global_config.block_id, + BlockMetaData::zero_state(zero_state.gen_utime().unwrap()), + ) + .unwrap(); + + let state = ShardStateStuff::new( + global_config.block_id, + zero_state.cell.clone(), + storage.shard_state_storage().min_ref_mc_state(), + ) + .unwrap(); + + storage + .shard_state_storage() + .store_state(&handle, &state) + .await + .unwrap(); + + let min_ref_mc_state = storage.shard_state_storage().min_ref_mc_state(); + assert_eq!(min_ref_mc_state.seqno(), zero_state.min_ref_mc_seqno()); + + // Write persistent state + let persistent_state_keeper = storage.runtime_storage().persistent_state_keeper(); + assert!(persistent_state_keeper.current().is_none()); + + storage + .persistent_state_storage() + .prepare_persistent_states_dir(&state.block_id()) + .unwrap(); + + storage + .persistent_state_storage() + .save_state( + &state.block_id(), + &state.block_id(), + zero_state.cell.repr_hash(), + ) + .await + .unwrap(); + + tokio::time::sleep(Duration::from_secs(10)).await; + + //println!("{:?}", zero_state.state); + //println!("{:?}", global_config); + + //std::fs::remove_dir_all(root_path).unwrap() +}