From 2fefd76856665f8f35704b246eb8148f3150ccc5 Mon Sep 17 00:00:00 2001 From: Alex Kouprin Date: Mon, 1 Jun 2020 20:05:06 +0300 Subject: [PATCH] feat(GC): remove Chunks by height_created (#2716) * feat(GC): remove chunks with zero height_included * cache update * WIP * feat(store validator): block_chunks_exist function (#2718) * WIP * moving Store Validator module into Chain * block_chunks_exist relies on "do I care about the shard" * ++ protocol version --- Cargo.lock | 14 +- Cargo.toml | 1 - Makefile | 4 +- chain/chain/Cargo.toml | 1 - chain/chain/src/chain.rs | 4 +- chain/chain/src/lib.rs | 4 +- chain/chain/src/store.rs | 241 ++++++++++++------ chain/chain/src/store_validator.rs | 135 ++++++++++ .../chain/src/store_validator}/validate.rs | 170 ++++++++---- chain/chain/src/types.rs | 32 +-- chain/chain/tests/gc.rs | 18 +- chain/client/Cargo.toml | 3 +- chain/client/src/client.rs | 4 +- chain/client/src/client_actor.rs | 5 +- chain/client/src/info.rs | 2 +- chain/client/src/sync.rs | 3 +- chain/client/src/view_client.rs | 4 +- core/chain-configs/src/lib.rs | 2 +- core/primitives/src/block.rs | 30 +++ core/primitives/src/sharding.rs | 12 +- core/store/src/db.rs | 5 +- core/store/src/lib.rs | 3 +- genesis-tools/genesis-populate/src/lib.rs | 4 +- neard/res/genesis_config.json | 2 +- neard/src/runtime.rs | 5 +- scripts/binary-release.sh | 2 +- scripts/migrations/19-col-chunks-height.py | 23 ++ test-utils/store-validator-bin/Cargo.toml | 22 -- test-utils/store-validator/Cargo.toml | 15 +- test-utils/store-validator/src/lib.rs | 108 -------- .../src/main.rs | 7 +- 31 files changed, 542 insertions(+), 343 deletions(-) create mode 100644 chain/chain/src/store_validator.rs rename {test-utils/store-validator/src => chain/chain/src/store_validator}/validate.rs (51%) create mode 100644 scripts/migrations/19-col-chunks-height.py delete mode 100644 test-utils/store-validator-bin/Cargo.toml delete mode 100644 test-utils/store-validator/src/lib.rs rename test-utils/{store-validator-bin => store-validator}/src/main.rs (91%) diff --git a/Cargo.lock b/Cargo.lock index 561a67f4d79..613ff00c408 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2037,7 +2037,6 @@ dependencies = [ "near-pool", "near-primitives", "near-store", - "near-store-validator", "num-rational", "rand 0.7.3", "rocksdb", @@ -2103,7 +2102,6 @@ dependencies = [ "near-pool", "near-primitives", "near-store", - "near-store-validator", "near-telemetry", "neard", "num-rational", @@ -2360,15 +2358,6 @@ dependencies = [ "tempfile", ] -[[package]] -name = "near-store-validator" -version = "0.1.0" -dependencies = [ - "near-chain-configs", - "near-primitives", - "near-store", -] - [[package]] name = "near-telemetry" version = "0.1.0" @@ -3560,7 +3549,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] -name = "store-validator-bin" +name = "store-validator" version = "0.1.0" dependencies = [ "ansi_term", @@ -3571,7 +3560,6 @@ dependencies = [ "near-logger-utils", "near-primitives", "near-store", - "near-store-validator", "neard", "serde_json", "testlib", diff --git a/Cargo.toml b/Cargo.toml index 7e97d715515..5e94fc8a225 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,7 +31,6 @@ members = [ "test-utils/testlib", "test-utils/loadtester", "test-utils/state-viewer", - "test-utils/store-validator-bin", "test-utils/store-validator", "neard/", "tools/rpctypegen/core", diff --git a/Makefile b/Makefile index 5833903587d..6a36eca8160 100644 --- a/Makefile +++ b/Makefile @@ -7,7 +7,7 @@ release: cargo build -p genesis-csv-to-json --release cargo build -p near-vm-runner-standalone --release cargo build -p state-viewer --release - cargo build -p store-validator-bin --release + cargo build -p store-validator --release debug: cargo build -p neard @@ -15,4 +15,4 @@ debug: cargo build -p genesis-csv-to-json cargo build -p near-vm-runner-standalone cargo build -p state-viewer - cargo build -p store-validator-bin + cargo build -p store-validator diff --git a/chain/chain/Cargo.toml b/chain/chain/Cargo.toml index 3cda43ea229..cb71eb3ccfb 100644 --- a/chain/chain/Cargo.toml +++ b/chain/chain/Cargo.toml @@ -28,7 +28,6 @@ near-pool = { path = "../pool" } [dev-dependencies] near-logger-utils = {path = "../../test-utils/logger"} -near-store-validator = { path = "../../test-utils/store-validator" } [features] # if enabled, we assert in most situations that are impossible unless some byzantine behavior is observed. diff --git a/chain/chain/src/chain.rs b/chain/chain/src/chain.rs index 5a925cb7af5..eda028eb374 100644 --- a/chain/chain/src/chain.rs +++ b/chain/chain/src/chain.rs @@ -9,7 +9,7 @@ use chrono::Duration; use log::{debug, error, info}; use near_chain_configs::GenesisConfig; -use near_primitives::block::genesis_chunks; +use near_primitives::block::{genesis_chunks, Tip}; use near_primitives::challenge::{ BlockDoubleSign, Challenge, ChallengeBody, ChallengesResult, ChunkProofs, ChunkState, MaybeEncodedShardChunk, SlashedValidator, @@ -40,7 +40,7 @@ use crate::store::{ use crate::types::{ AcceptedBlock, ApplyTransactionResult, Block, BlockHeader, BlockStatus, BlockSyncResponse, Provenance, ReceiptList, ReceiptProofResponse, ReceiptResponse, RootProof, RuntimeAdapter, - ShardStateSyncResponseHeader, StatePartKey, Tip, + ShardStateSyncResponseHeader, StatePartKey, }; use crate::validate::{ validate_challenge, validate_chunk_proofs, validate_chunk_with_chunk_extra, diff --git a/chain/chain/src/lib.rs b/chain/chain/src/lib.rs index 21e6f8395c9..a3c632c370d 100644 --- a/chain/chain/src/lib.rs +++ b/chain/chain/src/lib.rs @@ -6,7 +6,8 @@ pub use doomslug::{Doomslug, DoomslugBlockProductionReadiness, DoomslugThreshold pub use error::{Error, ErrorKind}; pub use lightclient::create_light_client_block_view; pub use store::{ChainStore, ChainStoreAccess, ChainStoreUpdate}; -pub use types::{Block, BlockHeader, BlockStatus, Provenance, ReceiptResult, RuntimeAdapter, Tip}; +pub use store_validator::{ErrorMessage, StoreValidator}; +pub use types::{Block, BlockHeader, BlockStatus, Provenance, ReceiptResult, RuntimeAdapter}; pub mod chain; mod doomslug; @@ -14,6 +15,7 @@ mod error; mod lightclient; mod metrics; mod store; +pub mod store_validator; pub mod test_utils; pub mod types; pub mod validate; diff --git a/chain/chain/src/store.rs b/chain/chain/src/store.rs index b680e114078..a8f89f07960 100644 --- a/chain/chain/src/store.rs +++ b/chain/chain/src/store.rs @@ -1,3 +1,4 @@ +use std::collections::hash_map::Entry; use std::collections::{HashMap, HashSet}; use std::convert::TryFrom; use std::io; @@ -9,7 +10,7 @@ use chrono::Utc; use serde::Serialize; use tracing::debug; -use near_primitives::block::Approval; +use near_primitives::block::{Approval, Tip}; use near_primitives::errors::InvalidTxError; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{MerklePath, PartialMerkleTree}; @@ -30,18 +31,18 @@ use near_primitives::views::LightClientBlockView; use near_store::{ read_with_cache, ColBlock, ColBlockExtra, ColBlockHeader, ColBlockHeight, ColBlockMerkleTree, ColBlockMisc, ColBlockPerHeight, ColBlockRefCount, ColBlocksToCatchup, ColChallengedBlocks, - ColChunkExtra, ColChunkPerHeightShard, ColChunks, ColEpochLightClientBlocks, - ColIncomingReceipts, ColInvalidChunks, ColLastBlockWithNewChunk, ColNextBlockHashes, - ColNextBlockWithNewChunk, ColOutgoingReceipts, ColPartialChunks, ColReceiptIdToShardId, - ColState, ColStateChanges, ColStateDlInfos, ColStateHeaders, ColTransactionResult, - ColTransactions, ColTrieChanges, KeyForStateChanges, ShardTries, Store, StoreUpdate, - TrieChanges, WrappedTrieChanges, HEADER_HEAD_KEY, HEAD_KEY, LARGEST_TARGET_HEIGHT_KEY, - LATEST_KNOWN_KEY, SYNC_HEAD_KEY, TAIL_KEY, + ColChunkExtra, ColChunkHashesByHeight, ColChunkPerHeightShard, ColChunks, + ColEpochLightClientBlocks, ColIncomingReceipts, ColInvalidChunks, ColLastBlockWithNewChunk, + ColNextBlockHashes, ColNextBlockWithNewChunk, ColOutgoingReceipts, ColPartialChunks, + ColReceiptIdToShardId, ColState, ColStateChanges, ColStateDlInfos, ColStateHeaders, + ColTransactionResult, ColTransactions, ColTrieChanges, KeyForStateChanges, ShardTries, Store, + StoreUpdate, TrieChanges, WrappedTrieChanges, CHUNK_TAIL_KEY, HEADER_HEAD_KEY, HEAD_KEY, + LARGEST_TARGET_HEIGHT_KEY, LATEST_KNOWN_KEY, SYNC_HEAD_KEY, TAIL_KEY, }; use crate::byzantine_assert; use crate::error::{Error, ErrorKind}; -use crate::types::{Block, BlockHeader, LatestKnown, ReceiptProofResponse, ReceiptResponse, Tip}; +use crate::types::{Block, BlockHeader, LatestKnown, ReceiptProofResponse, ReceiptResponse}; /// lru cache size const CACHE_SIZE: usize = 100; @@ -79,8 +80,10 @@ pub trait ChainStoreAccess { fn store(&self) -> &Store; /// The chain head. fn head(&self) -> Result; - /// The chain tail height. + /// The chain Blocks Tail height. fn tail(&self) -> Result; + /// The chain Chunks Tail height. + fn chunk_tail(&self) -> Result; /// Head of the header chain (not the same thing as head_header). fn header_head(&self) -> Result; /// The "sync" head: last header we received from syncing. @@ -151,6 +154,11 @@ pub trait ChainStoreAccess { &mut self, height: BlockHeight, ) -> Result<&HashMap>, Error>; + /// Returns a HashSet of Chunk Hashes for current Height + fn get_all_chunk_hashes_by_height( + &mut self, + height: BlockHeight, + ) -> Result, Error>; /// Returns a number of references for Block with `block_hash` fn get_block_refcount(&mut self, block_hash: &CryptoHash) -> Result<&u64, Error>; /// Check if we saw chunk hash at given height and shard id. @@ -269,7 +277,7 @@ pub struct ChainStore { chunk_extras: SizedCache, ChunkExtra>, /// Cache with height to hash on the main chain. height: SizedCache, CryptoHash>, - /// Cache with height to hash on any chain. + /// Cache with height to block hash on any chain. block_hash_per_height: SizedCache, HashMap>>, /// Cache with height and shard_id to any chunk hash. chunk_hash_per_height_shard: SizedCache, ChunkHash>, @@ -471,7 +479,7 @@ impl ChainStoreAccess for ChainStore { option_to_not_found(self.store.get_ser(ColBlockMisc, HEAD_KEY), "HEAD") } - /// The chain tail height, used by GC. + /// The chain Blocks Tail height, used by GC. fn tail(&self) -> Result { self.store .get_ser(ColBlockMisc, TAIL_KEY) @@ -479,6 +487,14 @@ impl ChainStoreAccess for ChainStore { .map_err(|e| e.into()) } + /// The chain Chunks Tail height, used by GC. + fn chunk_tail(&self) -> Result { + self.store + .get_ser(ColBlockMisc, CHUNK_TAIL_KEY) + .map(|option| option.unwrap_or_else(|| self.genesis_height)) + .map_err(|e| e.into()) + } + /// The "sync" head: last header we received from syncing. fn sync_head(&self) -> Result { option_to_not_found(self.store.get_ser(ColBlockMisc, SYNC_HEAD_KEY), "SYNC_HEAD") @@ -674,6 +690,17 @@ impl ChainStoreAccess for ChainStore { ) } + fn get_all_chunk_hashes_by_height( + &mut self, + height: BlockHeight, + ) -> Result, Error> { + match self.store.get_ser(ColChunkHashesByHeight, &index_to_bytes(height)) { + Ok(Some(hash_set)) => Ok(hash_set), + Ok(None) => Ok(HashSet::new()), + Err(e) => Err(e.into()), + } + } + fn get_block_refcount(&mut self, block_hash: &CryptoHash) -> Result<&u64, Error> { option_to_not_found( read_with_cache( @@ -1040,6 +1067,7 @@ pub struct ChainStoreUpdate<'a> { chain_store_cache_update: ChainStoreCacheUpdate, head: Option, tail: Option, + chunks_tail: Option, header_head: Option, sync_head: Option, largest_target_height: Option, @@ -1062,6 +1090,7 @@ impl<'a> ChainStoreUpdate<'a> { chain_store_cache_update: ChainStoreCacheUpdate::new(), head: None, tail: None, + chunks_tail: None, header_head: None, sync_head: None, largest_target_height: None, @@ -1132,7 +1161,7 @@ impl<'a> ChainStoreAccess for ChainStoreUpdate<'a> { } } - /// The chain tail height, used by GC. + /// The chain Block Tail height, used by GC. fn tail(&self) -> Result { if let Some(tail) = &self.tail { Ok(tail.clone()) @@ -1141,6 +1170,15 @@ impl<'a> ChainStoreAccess for ChainStoreUpdate<'a> { } } + /// The chain Chunks Tail height, used by GC. + fn chunk_tail(&self) -> Result { + if let Some(chunks_tail) = &self.chunks_tail { + Ok(chunks_tail.clone()) + } else { + self.chain_store.chunk_tail() + } + } + /// The "sync" head: last header we received from syncing. fn sync_head(&self) -> Result { if let Some(sync_head) = &self.sync_head { @@ -1236,6 +1274,13 @@ impl<'a> ChainStoreAccess for ChainStoreUpdate<'a> { self.chain_store.get_all_block_hashes_by_height(height) } + fn get_all_chunk_hashes_by_height( + &mut self, + height: BlockHeight, + ) -> Result, Error> { + self.chain_store.get_all_chunk_hashes_by_height(height) + } + fn get_block_refcount(&mut self, block_hash: &CryptoHash) -> Result<&u64, Error> { if let Some(refcount) = self.chain_store_cache_update.block_refcounts.get(block_hash) { Ok(refcount) @@ -1769,6 +1814,10 @@ impl<'a> ChainStoreUpdate<'a> { self.tail = Some(height); } + pub fn update_chunks_tail(&mut self, height: BlockHeight) { + self.chunks_tail = Some(height); + } + pub fn clear_state_data(&mut self) { let mut store_update = self.store().store_update(); @@ -1779,6 +1828,54 @@ impl<'a> ChainStoreUpdate<'a> { self.merge(store_update); } + pub fn clear_chunk_data(&mut self, min_chunk_height: BlockHeight) -> Result<(), Error> { + let mut store_update = self.store().store_update(); + + let chunk_tail = self.chunk_tail()?; + for height in chunk_tail..min_chunk_height { + let chunk_hashes = self.get_all_chunk_hashes_by_height(height)?; + for chunk_hash in chunk_hashes { + // 1. Delete chunk-related data + let chunk = self.get_chunk(&chunk_hash)?.clone(); + debug_assert_eq!(chunk.header.inner.height_created, height); + // 1a. Delete from receipt_id_to_shard_id (ColReceiptIdToShardId) + for receipt in chunk.receipts { + store_update.delete(ColReceiptIdToShardId, receipt.receipt_id.as_ref()); + self.chain_store + .receipt_id_to_shard_id + .cache_remove(&receipt.receipt_id.into()); + } + // 1b. Delete from ColTransactions + for transaction in chunk.transactions { + store_update.delete(ColTransactions, transaction.get_hash().as_ref()); + self.chain_store.transactions.cache_remove(&transaction.get_hash().into()); + } + + // 2. Delete chunk_hash-indexed data + let chunk_header_hash = chunk_hash.clone().into(); + let chunk_header_hash_ref = chunk_hash.as_ref(); + // 2a. Delete chunks (ColChunks) + store_update.delete(ColChunks, chunk_header_hash_ref); + self.chain_store.chunks.cache_remove(&chunk_header_hash); + // 2b. Delete chunk extras (ColChunkExtra) + store_update.delete(ColChunkExtra, chunk_header_hash_ref); + self.chain_store.chunk_extras.cache_remove(&chunk_header_hash); + // 2c. Delete partial_chunks (ColPartialChunks) + store_update.delete(ColPartialChunks, chunk_header_hash_ref); + self.chain_store.partial_chunks.cache_remove(&chunk_header_hash); + // 2d. Delete invalid chunks (ColInvalidChunks) + store_update.delete(ColInvalidChunks, chunk_header_hash_ref); + self.chain_store.invalid_chunks.cache_remove(&chunk_header_hash); + } + // 3. Delete chunks_tail-related data + // 3a. Delete from ColChunkHashesByHeight + store_update.delete(ColChunkHashesByHeight, &index_to_bytes(height)); + } + self.update_chunks_tail(min_chunk_height); + self.merge(store_update); + Ok(()) + } + // Clearing block data of `block_hash`, if on a fork. // Clearing block data of `block_hash.prev`, if on the Canonical Chain. pub fn clear_block_data( @@ -1868,75 +1965,41 @@ impl<'a> ChainStoreUpdate<'a> { // 2f. Delete from ColStateParts // Already done, check chain.clear_downloaded_parts() } - for chunk_header in block.chunks { - if let Ok(chunk) = self.get_chunk_clone_from_header(&chunk_header) { - // 2g. Delete from receipt_id_to_shard_id (ColReceiptIdToShardId) - for receipt in chunk.receipts { - store_update.delete(ColReceiptIdToShardId, receipt.receipt_id.as_ref()); - self.chain_store - .receipt_id_to_shard_id - .cache_remove(&receipt.receipt_id.into()); - } - // 2h. Delete from ColTransactions - for transaction in chunk.transactions { - store_update.delete(ColTransactions, transaction.get_hash().as_ref()); - self.chain_store.transactions.cache_remove(&transaction.get_hash().into()); - } - } - // 3. Delete chunk_hash-indexed data - let chunk_header_hash = chunk_header.hash.clone().into(); - let chunk_header_hash_ref = chunk_header.hash.as_ref(); - // 3a. Delete chunks (ColChunks) - store_update.delete(ColChunks, chunk_header_hash_ref); - self.chain_store.chunks.cache_remove(&chunk_header_hash); - // 3b. Delete chunk extras (ColChunkExtra) - store_update.delete(ColChunkExtra, chunk_header_hash_ref); - self.chain_store.chunk_extras.cache_remove(&chunk_header_hash); - // 3c. Delete partial_chunks (ColPartialChunks) - store_update.delete(ColPartialChunks, chunk_header_hash_ref); - self.chain_store.partial_chunks.cache_remove(&chunk_header_hash); - // 3d. Delete invalid chunks (ColInvalidChunks) - store_update.delete(ColInvalidChunks, chunk_header_hash_ref); - self.chain_store.invalid_chunks.cache_remove(&chunk_header_hash); - } - - // 4. Delete block_hash-indexed data - //let chunk_header_hash = chunk_header.hash.clone().into(); + // 3. Delete block_hash-indexed data let block_hash_ref = block_hash.as_ref(); - // 4a. Delete block (ColBlock) + // 3a. Delete block (ColBlock) store_update.delete(ColBlock, block_hash_ref); self.chain_store.blocks.cache_remove(&block_hash.into()); - // 4b. Delete block header (ColBlockHeader) - don't do because header sync needs headers - // 4c. Delete block extras (ColBlockExtra) + // 3b. Delete block header (ColBlockHeader) - don't do because header sync needs headers + // 3c. Delete block extras (ColBlockExtra) store_update.delete(ColBlockExtra, block_hash_ref); self.chain_store.block_extras.cache_remove(&block_hash.into()); - // 4d. Delete from next_block_hashes (ColNextBlockHashes) + // 3d. Delete from next_block_hashes (ColNextBlockHashes) store_update.delete(ColNextBlockHashes, block_hash_ref); self.chain_store.next_block_hashes.cache_remove(&block_hash.into()); - // 4e. Delete from ColChallengedBlocks + // 3e. Delete from ColChallengedBlocks store_update.delete(ColChallengedBlocks, block_hash_ref); - // 4f. Delete from ColBlocksToCatchup + // 3f. Delete from ColBlocksToCatchup store_update.delete(ColBlocksToCatchup, block_hash_ref); - // 4g. Delete from KV state changes + // 3g. Delete from KV state changes let storage_key = KeyForStateChanges::get_prefix(&block_hash); - // 4g1. We should collect all the keys which key prefix equals to `block_hash` + // 3g1. We should collect all the keys which key prefix equals to `block_hash` let stored_state_changes = self.chain_store.store().iter_prefix(ColStateChanges, storage_key.as_ref()); - // 4g2. Remove from ColStateChanges all found State Changes + // 3g2. Remove from ColStateChanges all found State Changes for (key, _) in stored_state_changes { store_update.delete(ColStateChanges, key.as_ref()); } - // 4h. Delete from ColBlockRefCount + // 3h. Delete from ColBlockRefCount store_update.delete(ColBlockRefCount, block_hash_ref); self.chain_store.block_refcounts.cache_remove(&block_hash.into()); match gc_mode { GCMode::Fork(_) => { - // 5. Forks only clearing - // 5a. Update block_hash_per_height - let epoch_to_hashes_ref = - self.get_all_block_hashes_by_height(height).expect("current height exists"); + // 4. Forks only clearing + // 4a. Update block_hash_per_height + let epoch_to_hashes_ref = self.get_all_block_hashes_by_height(height)?; let mut epoch_to_hashes = epoch_to_hashes_ref.clone(); let hashes = epoch_to_hashes .get_mut(&block.header.inner_lite.epoch_id) @@ -1950,21 +2013,24 @@ impl<'a> ChainStoreUpdate<'a> { self.chain_store .block_hash_per_height .cache_set(index_to_bytes(height), epoch_to_hashes); - // 5b. Decreasing block refcount + // 4b. Decreasing block refcount self.dec_block_refcount(&block.header.prev_hash)?; } - GCMode::Canonical(_) => { - // 6. Canonical Chain only clearing - // 6a. Delete blocks with current height (ColBlockPerHeight) - store_update.delete(ColBlockPerHeight, &index_to_bytes(height)); - self.chain_store.block_hash_per_height.cache_remove(&index_to_bytes(height)); - // 6b. Delete from ColBlockHeight - don't do because: block sync needs it + genesis should be accessible - } - GCMode::StateSync => { - // 7. Post State Sync clearing - // 7a. Delete blocks with current height (ColBlockPerHeight) - that's fine to do it multiple times + GCMode::Canonical(_) | GCMode::StateSync => { + // 5. Canonical Chain and Post State Sync clearing + // 5a. Delete blocks with current height (ColBlockPerHeight) store_update.delete(ColBlockPerHeight, &index_to_bytes(height)); self.chain_store.block_hash_per_height.cache_remove(&index_to_bytes(height)); + // 5b. Delete from ColBlockHeight - don't do because: block sync needs it + genesis should be accessible + + // 6. Delete chunks and chunk-indexed data + let mut min_chunk_height = self.tail()?; + for chunk_header in block.chunks { + if min_chunk_height > chunk_header.inner.height_created { + min_chunk_height = chunk_header.inner.height_created; + } + } + self.clear_chunk_data(min_chunk_height)?; } }; self.merge(store_update); @@ -2045,11 +2111,33 @@ impl<'a> ChainStoreUpdate<'a> { .set_ser(ColChunkPerHeightShard, &key, chunk_hash) .map_err::(|e| e.into())?; } + let mut chunk_hashes_by_height: HashMap> = HashMap::new(); for (chunk_hash, chunk) in self.chain_store_cache_update.chunks.iter() { + match chunk_hashes_by_height.entry(chunk.header.inner.height_created) { + Entry::Occupied(mut entry) => { + entry.get_mut().insert(chunk_hash.clone()); + } + Entry::Vacant(entry) => { + let mut hash_set = match self + .chain_store + .get_all_chunk_hashes_by_height(chunk.header.inner.height_created) + { + Ok(hash_set) => hash_set.clone(), + Err(_) => HashSet::new(), + }; + hash_set.insert(chunk_hash.clone()); + entry.insert(hash_set); + } + }; store_update .set_ser(ColChunks, chunk_hash.as_ref(), chunk) .map_err::(|e| e.into())?; } + for (height, hash_set) in chunk_hashes_by_height { + store_update + .set_ser(ColChunkHashesByHeight, &index_to_bytes(height), &hash_set) + .map_err::(|e| e.into())?; + } for (chunk_hash, partial_chunk) in self.chain_store_cache_update.partial_chunks.iter() { store_update .set_ser(ColPartialChunks, chunk_hash.as_ref(), partial_chunk) @@ -2340,19 +2428,19 @@ mod tests { use near_chain_configs::GenesisConfig; use near_crypto::KeyType; - use near_primitives::block::Block; + use near_primitives::block::{Block, Tip}; use near_primitives::errors::InvalidTxError; use near_primitives::hash::hash; use near_primitives::types::{BlockHeight, EpochId, NumBlocks}; use near_primitives::utils::index_to_bytes; use near_primitives::validator_signer::{InMemoryValidatorSigner, ValidatorSigner}; use near_store::test_utils::create_test_store; - use near_store_validator::StoreValidator; use crate::chain::{check_refcount_map, MAX_HEIGHTS_TO_CLEAR}; use crate::store::{ChainStoreAccess, GCMode}; + use crate::store_validator::StoreValidator; use crate::test_utils::KeyValueRuntime; - use crate::{Chain, ChainGenesis, DoomslugThresholdMode, Tip}; + use crate::{Chain, ChainGenesis, DoomslugThresholdMode}; fn get_chain() -> Chain { get_chain_with_epoch_length(10) @@ -2741,8 +2829,9 @@ mod tests { let mut genesis = GenesisConfig::default(); genesis.genesis_height = 0; let mut store_validator = StoreValidator::new( + None, genesis.clone(), - chain.runtime_adapter.get_tries(), + chain.runtime_adapter.clone(), chain.store().owned_store(), ); store_validator.validate(); diff --git a/chain/chain/src/store_validator.rs b/chain/chain/src/store_validator.rs new file mode 100644 index 00000000000..2f254bd60d9 --- /dev/null +++ b/chain/chain/src/store_validator.rs @@ -0,0 +1,135 @@ +use std::convert::TryFrom; +use std::sync::Arc; + +use borsh::BorshDeserialize; + +use near_chain_configs::GenesisConfig; +use near_primitives::borsh; +use near_primitives::hash::CryptoHash; +use near_primitives::sharding::ChunkHash; +use near_primitives::types::AccountId; +use near_store::{DBCol, Store}; + +use crate::RuntimeAdapter; + +mod validate; + +#[derive(Debug)] +pub struct ErrorMessage { + pub col: Option, + pub key: Option, + pub func: String, + pub reason: String, +} + +impl ErrorMessage { + fn new(func: String, reason: String) -> Self { + Self { col: None, key: None, func, reason } + } +} + +pub struct StoreValidator { + me: Option, + config: GenesisConfig, + runtime_adapter: Arc, + store: Arc, + + pub errors: Vec, + tests: u64, +} + +impl StoreValidator { + pub fn new( + me: Option, + config: GenesisConfig, + runtime_adapter: Arc, + store: Arc, + ) -> Self { + StoreValidator { + me, + config, + runtime_adapter, + store: store.clone(), + errors: vec![], + tests: 0, + } + } + pub fn is_failed(&self) -> bool { + self.tests == 0 || self.errors.len() > 0 + } + pub fn num_failed(&self) -> u64 { + self.errors.len() as u64 + } + pub fn tests_done(&self) -> u64 { + self.tests + } + fn col_to_key(col: DBCol, key: &[u8]) -> String { + match col { + DBCol::ColBlockHeader | DBCol::ColBlock => { + format!("{:?}", CryptoHash::try_from(key.as_ref())) + } + DBCol::ColChunks => format!("{:?}", ChunkHash::try_from_slice(key.as_ref())), + _ => format!("{:?}", key), + } + } + fn validate_col(&mut self, col: DBCol) { + for (key, value) in self.store.clone().iter(col) { + match col { + DBCol::ColBlockHeader => { + // Block Header Hash is valid + self.check(&validate::block_header_validity, &key, &value, col); + } + DBCol::ColBlock => { + // Block Hash is valid + self.check(&validate::block_hash_validity, &key, &value, col); + // Block Header for current Block exists + self.check(&validate::block_header_exists, &key, &value, col); + // Block Height is greater or equal to tail, or to Genesis Height + self.check(&validate::block_height_cmp_tail, &key, &value, col); + // Chunks for current Block exist + self.check(&validate::block_chunks_exist, &key, &value, col); + } + DBCol::ColChunks => { + // Chunk Hash is valid + self.check(&validate::chunk_basic_validity, &key, &value, col); + // There is a State Root in the Trie + self.check(&validate::chunks_state_roots_in_trie, &key, &value, col); + // ShardChunk can be indexed by Height + self.check(&validate::chunks_indexed_by_height_created, &key, &value, col); + } + DBCol::ColChunkHashesByHeight => { + // ShardChunk which can be indexed by Height exists + self.check(&validate::chunk_of_height_exists, &key, &value, col); + } + _ => unimplemented!(), + } + } + } + pub fn validate(&mut self) { + self.check(&validate::head_tail_validity, &[0], &[0], DBCol::ColBlockMisc); + self.validate_col(DBCol::ColBlockHeader); + self.validate_col(DBCol::ColBlock); + self.validate_col(DBCol::ColChunks); + self.validate_col(DBCol::ColChunkHashesByHeight); + } + + fn check( + &mut self, + f: &dyn Fn(&StoreValidator, &[u8], &[u8]) -> Result<(), ErrorMessage>, + key: &[u8], + value: &[u8], + col: DBCol, + ) { + let result = f(self, key, value); + self.tests += 1; + match result { + Ok(_) => {} + Err(e) => { + let mut e = e; + e.col = Some(col); + e.key = Some(Self::col_to_key(col, key)); + self.errors.push(e) + } + } + } +} diff --git a/test-utils/store-validator/src/validate.rs b/chain/chain/src/store_validator/validate.rs similarity index 51% rename from test-utils/store-validator/src/validate.rs rename to chain/chain/src/store_validator/validate.rs index e58e817cf77..e3daa6b351e 100644 --- a/test-utils/store-validator/src/validate.rs +++ b/chain/chain/src/store_validator/validate.rs @@ -1,24 +1,17 @@ -use std::collections::{HashMap, HashSet}; +use std::collections::HashSet; use std::convert::TryFrom; use borsh::BorshDeserialize; -use near_primitives::block::{Block, BlockHeader}; +use near_primitives::block::{Block, BlockHeader, Tip}; use near_primitives::borsh; use near_primitives::hash::CryptoHash; use near_primitives::sharding::{ChunkHash, ShardChunk}; -use near_primitives::types::{BlockHeight, EpochId}; +use near_primitives::types::BlockHeight; use near_primitives::utils::index_to_bytes; -#[allow(unused)] use near_store::{ - read_with_cache, ColBlock, ColBlockExtra, ColBlockHeader, ColBlockHeight, ColBlockMisc, - ColBlockPerHeight, ColBlockRefCount, ColBlocksToCatchup, ColChallengedBlocks, ColChunkExtra, - ColChunkPerHeightShard, ColChunks, ColEpochLightClientBlocks, ColIncomingReceipts, - ColInvalidChunks, ColLastBlockWithNewChunk, ColNextBlockHashes, ColNextBlockWithNewChunk, - ColOutgoingReceipts, ColPartialChunks, ColReceiptIdToShardId, ColState, ColStateChanges, - ColStateDlInfos, ColStateHeaders, ColTransactionResult, ColTransactions, ColTrieChanges, DBCol, - KeyForStateChanges, ShardTries, Store, StoreUpdate, Trie, TrieChanges, TrieIterator, - WrappedTrieChanges, TAIL_KEY, + ColBlockHeader, ColBlockMisc, ColChunkHashesByHeight, ColChunks, TrieIterator, CHUNK_TAIL_KEY, + HEAD_KEY, TAIL_KEY, }; use crate::{ErrorMessage, StoreValidator}; @@ -47,18 +40,51 @@ macro_rules! unwrap_or_err { Err(e) => { return Err(ErrorMessage::new(get_parent_function_name!(), format!("{}, error: {}", format!($($x),*), e))) } - } + }; + }; +} + +macro_rules! unwrap_or_err_db { + ($obj: expr, $($x: tt),*) => { + match $obj { + Ok(Some(value)) => value, + Err(e) => { + return Err(ErrorMessage::new(get_parent_function_name!(), format!("{}, error: {}", format!($($x),*), e))) + } + _ => { + return Err(ErrorMessage::new(get_parent_function_name!(), format!($($x),*))) + } + }; }; } // All validations start here -pub(crate) fn nothing( - _sv: &StoreValidator, +pub(crate) fn head_tail_validity( + sv: &StoreValidator, _key: &[u8], _value: &[u8], ) -> Result<(), ErrorMessage> { - // Make sure that validation is executed + let tail = unwrap_or_err!( + sv.store.get_ser::(ColBlockMisc, TAIL_KEY), + "Can't get Tail from storage" + ) + .unwrap_or(sv.config.genesis_height); + let chunk_tail = unwrap_or_err!( + sv.store.get_ser::(ColBlockMisc, CHUNK_TAIL_KEY), + "Can't get Chunk Tail from storage" + ) + .unwrap_or(sv.config.genesis_height); + let head = unwrap_or_err_db!( + sv.store.get_ser::(ColBlockMisc, HEAD_KEY), + "Can't get Head from storage" + ); + if chunk_tail > tail { + return err!("chunk_tail > tail, {:?} > {:?}", chunk_tail, tail); + } + if tail > head.height { + return err!("tail > head.height, {:?} > {:?}", tail, head); + } Ok(()) } @@ -98,17 +124,14 @@ pub(crate) fn block_header_exists( ) -> Result<(), ErrorMessage> { let block_hash = unwrap_or_err!(CryptoHash::try_from(key.as_ref()), "Can't deserialize Block Hash"); - let header = unwrap_or_err!( + unwrap_or_err_db!( sv.store.get_ser::(ColBlockHeader, block_hash.as_ref()), "Can't get Block Header from storage" ); - match header { - Some(_) => Ok(()), - None => err!("Block Header not found"), - } + Ok(()) } -pub(crate) fn chunk_hash_validity( +pub(crate) fn chunk_basic_validity( _sv: &StoreValidator, key: &[u8], value: &[u8], @@ -118,51 +141,43 @@ pub(crate) fn chunk_hash_validity( let shard_chunk = unwrap_or_err!(ShardChunk::try_from_slice(value), "Can't deserialize ShardChunk"); if shard_chunk.chunk_hash != chunk_hash { - return err!("Invalid ShardChunk stored"); + return err!("Invalid ShardChunk {:?} stored", shard_chunk); } Ok(()) } -pub(crate) fn block_of_chunk_exists( +pub(crate) fn block_chunks_exist( sv: &StoreValidator, _key: &[u8], value: &[u8], ) -> Result<(), ErrorMessage> { - let shard_chunk = - unwrap_or_err!(ShardChunk::try_from_slice(value), "Can't deserialize ShardChunk"); - let height = shard_chunk.header.height_included; - let map = unwrap_or_err!( - sv.store.get_ser::>>( - ColBlockPerHeight, - &index_to_bytes(height), - ), - "Can't get Map from storage on height {:?}, no one is responsible for ShardChunk {:?}", - height, - shard_chunk - ); - match map { - Some(map) => { - for (_, set) in map { - for block_hash in set { - match sv.store.get_ser::(ColBlock, block_hash.as_ref()) { - Ok(Some(block)) => { - if block.chunks.contains(&shard_chunk.header) { - // Block for ShardChunk is found - return Ok(()); - } - } - _ => {} - } + let block = unwrap_or_err!(Block::try_from_slice(value), "Can't deserialize Block"); + for chunk_header in block.chunks { + match &sv.me { + Some(me) => { + if sv.runtime_adapter.cares_about_shard( + Some(&me), + &block.header.prev_hash, + chunk_header.inner.shard_id, + true, + ) || sv.runtime_adapter.will_care_about_shard( + Some(&me), + &block.header.prev_hash, + chunk_header.inner.shard_id, + true, + ) { + unwrap_or_err_db!( + sv.store + .get_ser::(ColChunks, chunk_header.chunk_hash().as_ref()), + "Can't get Chunk {:?} from storage", + chunk_header + ); } } - err!("No Block on height {:?} accepts ShardChunk {:?}", height, shard_chunk) + _ => {} } - None => err!( - "Map is empty on height {:?}, no one is responsible for ShardChunk {:?}", - height, - shard_chunk - ), } + Ok(()) } pub(crate) fn block_height_cmp_tail( @@ -197,10 +212,53 @@ pub(crate) fn chunks_state_roots_in_trie( unwrap_or_err!(ShardChunk::try_from_slice(value), "Can't deserialize ShardChunk"); let shard_id = shard_chunk.header.inner.shard_id; let state_root = shard_chunk.header.inner.prev_state_root; - let trie = sv.shard_tries.get_trie_for_shard(shard_id); + let trie = sv.runtime_adapter.get_trie_for_shard(shard_id); let trie = TrieIterator::new(&trie, &state_root).unwrap(); for item in trie { unwrap_or_err!(item, "Can't find ShardChunk {:?} in Trie", shard_chunk); } Ok(()) } + +pub(crate) fn chunks_indexed_by_height_created( + sv: &StoreValidator, + _key: &[u8], + value: &[u8], +) -> Result<(), ErrorMessage> { + let shard_chunk: ShardChunk = + unwrap_or_err!(ShardChunk::try_from_slice(value), "Can't deserialize ShardChunk"); + let height = shard_chunk.header.inner.height_created; + let chunk_hashes = unwrap_or_err_db!( + sv.store.get_ser::>(ColChunkHashesByHeight, &index_to_bytes(height)), + "Can't get Chunks Set from storage on Height {:?}, no one is responsible for ShardChunk {:?}", + height, + shard_chunk + ); + if !chunk_hashes.contains(&shard_chunk.chunk_hash) { + err!("Can't find ShardChunk {:?} on Height {:?}", shard_chunk, height) + } else { + Ok(()) + } +} + +pub(crate) fn chunk_of_height_exists( + sv: &StoreValidator, + key: &[u8], + value: &[u8], +) -> Result<(), ErrorMessage> { + let height: BlockHeight = + unwrap_or_err!(BlockHeight::try_from_slice(key), "Can't deserialize Height"); + let chunk_hashes: HashSet = + unwrap_or_err!(HashSet::::try_from_slice(value), "Can't deserialize Set"); + for chunk_hash in chunk_hashes { + let shard_chunk = unwrap_or_err_db!( + sv.store.get_ser::(ColChunks, chunk_hash.as_ref()), + "Can't get Chunk from storage with ChunkHash {:?}", + chunk_hash + ); + if shard_chunk.header.inner.height_created != height { + return err!("Invalid ShardChunk {:?} stored at Height {:?}", shard_chunk, height); + } + } + Ok(()) +} diff --git a/chain/chain/src/types.rs b/chain/chain/src/types.rs index d6c099e4c4b..e5a0895f068 100644 --- a/chain/chain/src/types.rs +++ b/chain/chain/src/types.rs @@ -7,7 +7,7 @@ use serde::Serialize; use near_crypto::Signature; use near_pool::types::PoolIterator; -pub use near_primitives::block::{Block, BlockHeader}; +pub use near_primitives::block::{Block, BlockHeader, Tip}; use near_primitives::challenge::{ChallengesResult, SlashedValidator}; use near_primitives::errors::InvalidTxError; use near_primitives::hash::{hash, CryptoHash}; @@ -475,36 +475,6 @@ pub struct LatestKnown { pub seen: u64, } -/// The tip of a fork. A handle to the fork ancestry from its leaf in the -/// blockchain tree. References the max height and the latest and previous -/// blocks for convenience -#[derive(BorshSerialize, BorshDeserialize, Serialize, Debug, Clone, PartialEq)] -pub struct Tip { - /// Height of the tip (max height of the fork) - pub height: BlockHeight, - /// Last block pushed to the fork - pub last_block_hash: CryptoHash, - /// Previous block - pub prev_block_hash: CryptoHash, - /// Current epoch id. Used for getting validator info. - pub epoch_id: EpochId, - /// Next epoch id. - pub next_epoch_id: EpochId, -} - -impl Tip { - /// Creates a new tip based on provided header. - pub fn from_header(header: &BlockHeader) -> Tip { - Tip { - height: header.inner_lite.height, - last_block_hash: header.hash(), - prev_block_hash: header.prev_hash, - epoch_id: header.inner_lite.epoch_id.clone(), - next_epoch_id: header.inner_lite.next_epoch_id.clone(), - } - } -} - #[derive(Debug, Clone, PartialEq, Eq, BorshSerialize, BorshDeserialize, Serialize)] pub struct ShardStateSyncResponseHeader { pub chunk: ShardChunk, diff --git a/chain/chain/tests/gc.rs b/chain/chain/tests/gc.rs index 85ca6b0cc8f..1e0c30764be 100644 --- a/chain/chain/tests/gc.rs +++ b/chain/chain/tests/gc.rs @@ -3,6 +3,7 @@ mod tests { use std::sync::Arc; use near_chain::chain::{check_refcount_map, Chain, ChainGenesis}; + use near_chain::store_validator::StoreValidator; use near_chain::test_utils::KeyValueRuntime; use near_chain::types::Tip; use near_chain::DoomslugThresholdMode; @@ -14,7 +15,6 @@ mod tests { use near_primitives::validator_signer::InMemoryValidatorSigner; use near_store::test_utils::{create_test_store, gen_changes}; use near_store::{ShardTries, StoreUpdate, Trie, WrappedTrieChanges}; - use near_store_validator::StoreValidator; use rand::Rng; fn get_chain(num_shards: NumShards) -> Chain { @@ -254,12 +254,20 @@ mod tests { } let mut genesis = GenesisConfig::default(); genesis.genesis_height = 0; - let mut store_validator = - StoreValidator::new(genesis.clone(), tries1, chain1.store().owned_store()); + let mut store_validator = StoreValidator::new( + None, + genesis.clone(), + chain1.runtime_adapter.clone(), + chain1.store().owned_store(), + ); store_validator.validate(); assert!(!store_validator.is_failed()); - let mut store_validator = - StoreValidator::new(genesis, tries2, chain2.store().owned_store()); + let mut store_validator = StoreValidator::new( + None, + genesis, + chain2.runtime_adapter.clone(), + chain2.store().owned_store(), + ); store_validator.validate(); assert!(!store_validator.is_failed()); } diff --git a/chain/client/Cargo.toml b/chain/client/Cargo.toml index 35234ef4930..2e45a9ba8cb 100644 --- a/chain/client/Cargo.toml +++ b/chain/client/Cargo.toml @@ -33,7 +33,6 @@ near-network = { path = "../network" } near-pool = { path = "../pool" } near-chunks = { path = "../chunks" } near-telemetry = { path = "../telemetry" } -near-store-validator = { path = "../../test-utils/store-validator", optional = true } [dev-dependencies] near-logger-utils = { path = "../../test-utils/logger" } @@ -44,5 +43,5 @@ neard = { path = "../../neard" } # if enabled, we assert in most situations that are impossible unless some byzantine behavior is observed. byzantine_asserts = ["near-chain/byzantine_asserts"] expensive_tests = [] -adversarial = ["near-network/adversarial", "near-chain/adversarial", "near-store-validator"] +adversarial = ["near-network/adversarial", "near-chain/adversarial"] metric_recorder = [] diff --git a/chain/client/src/client.rs b/chain/client/src/client.rs index 6b4e2021983..212818e9402 100644 --- a/chain/client/src/client.rs +++ b/chain/client/src/client.rs @@ -15,12 +15,12 @@ use near_chain::test_utils::format_hash; use near_chain::types::{AcceptedBlock, LatestKnown, ReceiptResponse}; use near_chain::{ BlockStatus, Chain, ChainGenesis, ChainStoreAccess, Doomslug, DoomslugThresholdMode, ErrorKind, - Provenance, RuntimeAdapter, Tip, + Provenance, RuntimeAdapter, }; use near_chain_configs::ClientConfig; use near_chunks::{ProcessPartialEncodedChunkResult, ShardsManager}; use near_network::{FullPeerInfo, NetworkAdapter, NetworkClientResponses, NetworkRequests}; -use near_primitives::block::{Approval, ApprovalInner, ApprovalMessage, Block, BlockHeader}; +use near_primitives::block::{Approval, ApprovalInner, ApprovalMessage, Block, BlockHeader, Tip}; use near_primitives::challenge::{Challenge, ChallengeBody}; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{merklize, MerklePath}; diff --git a/chain/client/src/client_actor.rs b/chain/client/src/client_actor.rs index 4310d2ded15..ee0f60260bd 100644 --- a/chain/client/src/client_actor.rs +++ b/chain/client/src/client_actor.rs @@ -49,7 +49,7 @@ use crate::types::{ }; use crate::StatusResponse; #[cfg(feature = "adversarial")] -use near_store_validator::StoreValidator; +use near_chain::store_validator::StoreValidator; /// Multiplier on `max_block_time` to wait until deciding that chain stalled. const STATUS_WAIT_TIME_MULTIPLIER: u64 = 10; @@ -261,8 +261,9 @@ impl Handler for ClientActor { let mut genesis = GenesisConfig::default(); genesis.genesis_height = self.client.chain.store().get_genesis_height(); let mut store_validator = StoreValidator::new( + self.client.validator_signer.as_ref().map(|x| x.validator_id().clone()), genesis, - self.client.runtime_adapter.get_tries(), + self.client.runtime_adapter.clone(), self.client.chain.store().owned_store(), ); store_validator.validate(); diff --git a/chain/client/src/info.rs b/chain/client/src/info.rs index 3b1913daf9f..09c6b1354cb 100644 --- a/chain/client/src/info.rs +++ b/chain/client/src/info.rs @@ -7,9 +7,9 @@ use ansi_term::Color::{Blue, Cyan, Green, White, Yellow}; use log::info; use sysinfo::{get_current_pid, set_open_files_limit, Pid, ProcessExt, System, SystemExt}; -use near_chain::Tip; use near_chain_configs::ClientConfig; use near_network::types::NetworkInfo; +use near_primitives::block::Tip; use near_primitives::network::PeerId; use near_primitives::serialize::to_base; use near_primitives::telemetry::{ diff --git a/chain/client/src/sync.rs b/chain/client/src/sync.rs index 6e622bef7e0..dc0038dd10d 100644 --- a/chain/client/src/sync.rs +++ b/chain/client/src/sync.rs @@ -11,9 +11,10 @@ use log::{debug, error, info, warn}; use rand::{thread_rng, Rng}; use near_chain::types::BlockSyncResponse; -use near_chain::{Chain, RuntimeAdapter, Tip}; +use near_chain::{Chain, RuntimeAdapter}; use near_network::types::{AccountOrPeerIdOrHash, NetworkResponses, ReasonForBan}; use near_network::{FullPeerInfo, NetworkAdapter, NetworkRequests}; +use near_primitives::block::Tip; use near_primitives::hash::CryptoHash; use near_primitives::types::{AccountId, BlockHeight, BlockHeightDelta, NumBlocks, ShardId}; use near_primitives::unwrap_or_return; diff --git a/chain/client/src/view_client.rs b/chain/client/src/view_client.rs index cf6c141785c..3db5115fc7b 100644 --- a/chain/client/src/view_client.rs +++ b/chain/client/src/view_client.rs @@ -12,7 +12,7 @@ use log::{debug, error, info, warn}; use near_chain::types::ShardStateSyncResponse; use near_chain::{ - Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode, ErrorKind, RuntimeAdapter, Tip, + Chain, ChainGenesis, ChainStoreAccess, DoomslugThresholdMode, ErrorKind, RuntimeAdapter, }; use near_chain_configs::ClientConfig; #[cfg(feature = "adversarial")] @@ -21,7 +21,7 @@ use near_network::types::{ NetworkViewClientMessages, NetworkViewClientResponses, ReasonForBan, StateResponseInfo, }; use near_network::{NetworkAdapter, NetworkRequests}; -use near_primitives::block::{BlockHeader, GenesisId}; +use near_primitives::block::{BlockHeader, GenesisId, Tip}; use near_primitives::hash::CryptoHash; use near_primitives::merkle::{verify_path, PartialMerkleTree}; use near_primitives::network::AnnounceAccount; diff --git a/core/chain-configs/src/lib.rs b/core/chain-configs/src/lib.rs index aff2368212d..9821091fd53 100644 --- a/core/chain-configs/src/lib.rs +++ b/core/chain-configs/src/lib.rs @@ -7,4 +7,4 @@ pub use genesis_config::{ }; /// Current latest version of the protocol -pub const PROTOCOL_VERSION: u32 = 18; +pub const PROTOCOL_VERSION: u32 = 19; diff --git a/core/primitives/src/block.rs b/core/primitives/src/block.rs index bd7698033b7..412658b301e 100644 --- a/core/primitives/src/block.rs +++ b/core/primitives/src/block.rs @@ -714,3 +714,33 @@ pub struct GenesisId { /// Hash of genesis block pub hash: CryptoHash, } + +/// The tip of a fork. A handle to the fork ancestry from its leaf in the +/// blockchain tree. References the max height and the latest and previous +/// blocks for convenience +#[derive(BorshSerialize, BorshDeserialize, Serialize, Debug, Clone, PartialEq)] +pub struct Tip { + /// Height of the tip (max height of the fork) + pub height: BlockHeight, + /// Last block pushed to the fork + pub last_block_hash: CryptoHash, + /// Previous block + pub prev_block_hash: CryptoHash, + /// Current epoch id. Used for getting validator info. + pub epoch_id: EpochId, + /// Next epoch id. + pub next_epoch_id: EpochId, +} + +impl Tip { + /// Creates a new tip based on provided header. + pub fn from_header(header: &BlockHeader) -> Tip { + Tip { + height: header.inner_lite.height, + last_block_hash: header.hash(), + prev_block_hash: header.prev_hash, + epoch_id: header.inner_lite.epoch_id.clone(), + next_epoch_id: header.inner_lite.next_epoch_id.clone(), + } + } +} diff --git a/core/primitives/src/sharding.rs b/core/primitives/src/sharding.rs index 3e13c9ca163..65995f60b6f 100644 --- a/core/primitives/src/sharding.rs +++ b/core/primitives/src/sharding.rs @@ -13,7 +13,17 @@ use crate::validator_signer::ValidatorSigner; use reed_solomon_erasure::ReconstructShard; #[derive( - BorshSerialize, BorshDeserialize, Serialize, Hash, Eq, PartialEq, Clone, Debug, Default, + BorshSerialize, + BorshDeserialize, + Serialize, + Hash, + Eq, + PartialEq, + Ord, + PartialOrd, + Clone, + Debug, + Default, )] pub struct ChunkHash(pub CryptoHash); diff --git a/core/store/src/db.rs b/core/store/src/db.rs index be0a5f563b5..8e74b3cbeeb 100644 --- a/core/store/src/db.rs +++ b/core/store/src/db.rs @@ -82,10 +82,11 @@ pub enum DBCol { ColTrieChanges = 36, /// Merkle tree of block hashes ColBlockMerkleTree = 37, + ColChunkHashesByHeight = 38, } // Do not move this line from enum DBCol -const NUM_COLS: usize = 38; +const NUM_COLS: usize = 39; impl std::fmt::Display for DBCol { fn fmt(&self, formatter: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { @@ -128,6 +129,7 @@ impl std::fmt::Display for DBCol { Self::ColBlockRefCount => "refcount per block", Self::ColTrieChanges => "trie changes", Self::ColBlockMerkleTree => "block merkle tree", + Self::ColChunkHashesByHeight => "chunk hashes indexed by height_created", }; write!(formatter, "{}", desc) } @@ -135,6 +137,7 @@ impl std::fmt::Display for DBCol { pub const HEAD_KEY: &[u8; 4] = b"HEAD"; pub const TAIL_KEY: &[u8; 4] = b"TAIL"; +pub const CHUNK_TAIL_KEY: &[u8; 10] = b"CHUNK_TAIL"; pub const SYNC_HEAD_KEY: &[u8; 9] = b"SYNC_HEAD"; pub const HEADER_HEAD_KEY: &[u8; 11] = b"HEADER_HEAD"; pub const LATEST_KNOWN_KEY: &[u8; 12] = b"LATEST_KNOWN"; diff --git a/core/store/src/lib.rs b/core/store/src/lib.rs index cdd7e5b8077..de0d46c745a 100644 --- a/core/store/src/lib.rs +++ b/core/store/src/lib.rs @@ -10,7 +10,8 @@ use cached::{Cached, SizedCache}; pub use db::DBCol::{self, *}; pub use db::{ - HEADER_HEAD_KEY, HEAD_KEY, LARGEST_TARGET_HEIGHT_KEY, LATEST_KNOWN_KEY, SYNC_HEAD_KEY, TAIL_KEY, + CHUNK_TAIL_KEY, HEADER_HEAD_KEY, HEAD_KEY, LARGEST_TARGET_HEIGHT_KEY, LATEST_KNOWN_KEY, + SYNC_HEAD_KEY, TAIL_KEY, }; use near_crypto::PublicKey; use near_primitives::account::{AccessKey, Account}; diff --git a/genesis-tools/genesis-populate/src/lib.rs b/genesis-tools/genesis-populate/src/lib.rs index fb6921a75c3..7d3bc3a3fa4 100644 --- a/genesis-tools/genesis-populate/src/lib.rs +++ b/genesis-tools/genesis-populate/src/lib.rs @@ -9,11 +9,11 @@ use std::sync::Arc; use borsh::BorshSerialize; use indicatif::{ProgressBar, ProgressStyle}; -use near_chain::{Block, Chain, ChainStore, RuntimeAdapter, Tip}; +use near_chain::{Block, Chain, ChainStore, RuntimeAdapter}; use near_chain_configs::Genesis; use near_crypto::{InMemorySigner, KeyType}; use near_primitives::account::{AccessKey, Account}; -use near_primitives::block::genesis_chunks; +use near_primitives::block::{genesis_chunks, Tip}; use near_primitives::contract::ContractCode; use near_primitives::hash::{hash, CryptoHash}; use near_primitives::state_record::StateRecord; diff --git a/neard/res/genesis_config.json b/neard/res/genesis_config.json index ed957d86f5e..3ffbc681033 100644 --- a/neard/res/genesis_config.json +++ b/neard/res/genesis_config.json @@ -1,6 +1,6 @@ { "config_version": 1, - "protocol_version": 18, + "protocol_version": 19, "genesis_time": "1970-01-01T00:00:00.000000000Z", "chain_id": "sample", "genesis_height": 0, diff --git a/neard/src/runtime.rs b/neard/src/runtime.rs index a454bf53716..33c6f8bfb31 100644 --- a/neard/src/runtime.rs +++ b/neard/src/runtime.rs @@ -107,7 +107,7 @@ pub struct NightshadeRuntime { home_dir: PathBuf, store: Arc, - pub tries: ShardTries, + tries: ShardTries, trie_viewer: TrieViewer, pub runtime: Runtime, epoch_manager: SafeEpochManager, @@ -1322,9 +1322,10 @@ mod test { use num_rational::Rational; - use near_chain::{ReceiptResult, Tip}; + use near_chain::ReceiptResult; use near_crypto::{InMemorySigner, KeyType, Signer}; use near_logger_utils::init_test_logger; + use near_primitives::block::Tip; use near_primitives::transaction::{ Action, CreateAccountAction, DeleteAccountAction, StakeAction, }; diff --git a/scripts/binary-release.sh b/scripts/binary-release.sh index 2ff7475e14b..51c8ca075be 100755 --- a/scripts/binary-release.sh +++ b/scripts/binary-release.sh @@ -20,4 +20,4 @@ upload_binary keypair-generator upload_binary genesis-csv-to-json upload_binary near-vm-runner-standalone upload_binary state-viewer -upload_binary store-validator-bin +upload_binary store-validator diff --git a/scripts/migrations/19-col-chunks-height.py b/scripts/migrations/19-col-chunks-height.py new file mode 100644 index 00000000000..cec4370396a --- /dev/null +++ b/scripts/migrations/19-col-chunks-height.py @@ -0,0 +1,23 @@ +""" +Getting data from ColTrieChanges is changed. + +https://github.com/nearprotocol/nearcore/pull/2592 + +""" + +import sys +import os +import json +from collections import OrderedDict + +home = sys.argv[1] +output_home = sys.argv[2] + +config = json.load(open(os.path.join(home, 'output.json')), + object_pairs_hook=OrderedDict) + +assert config['protocol_version'] == 18 + +config['protocol_version'] = 19 + +json.dump(config, open(os.path.join(output_home, 'output.json'), 'w'), indent=2) diff --git a/test-utils/store-validator-bin/Cargo.toml b/test-utils/store-validator-bin/Cargo.toml deleted file mode 100644 index f8af4b57608..00000000000 --- a/test-utils/store-validator-bin/Cargo.toml +++ /dev/null @@ -1,22 +0,0 @@ -[package] -name = "store-validator-bin" -version = "0.1.0" -authors = ["Near Inc "] -edition = "2018" - -[dependencies] -ansi_term = "0.11" -clap = "2.33" - -near-chain-configs = { path = "../../core/chain-configs" } -near-logger-utils = { path = "../../test-utils/logger" } -near-primitives = { path = "../../core/primitives" } -near-store = { path = "../../core/store" } -near-chain = { path = "../../chain/chain" } -neard = { path = "../../neard" } -near-store-validator = { path = "../store-validator" } - -[dev-dependencies] -testlib = { path = "../../test-utils/testlib" } -serde_json = "1" -near-client = { path = "../../chain/client" } diff --git a/test-utils/store-validator/Cargo.toml b/test-utils/store-validator/Cargo.toml index 89e3d1b955b..b191e5e49b9 100644 --- a/test-utils/store-validator/Cargo.toml +++ b/test-utils/store-validator/Cargo.toml @@ -1,10 +1,21 @@ [package] -name = "near-store-validator" +name = "store-validator" version = "0.1.0" authors = ["Near Inc "] edition = "2018" [dependencies] +ansi_term = "0.11" +clap = "2.33" + near-chain-configs = { path = "../../core/chain-configs" } +near-logger-utils = { path = "../../test-utils/logger" } near-primitives = { path = "../../core/primitives" } -near-store = { path = "../../core/store" } \ No newline at end of file +near-store = { path = "../../core/store" } +near-chain = { path = "../../chain/chain" } +neard = { path = "../../neard" } + +[dev-dependencies] +testlib = { path = "../../test-utils/testlib" } +serde_json = "1" +near-client = { path = "../../chain/client" } diff --git a/test-utils/store-validator/src/lib.rs b/test-utils/store-validator/src/lib.rs deleted file mode 100644 index 8c2875ce84d..00000000000 --- a/test-utils/store-validator/src/lib.rs +++ /dev/null @@ -1,108 +0,0 @@ -use std::convert::TryFrom; -use std::sync::Arc; - -use borsh::BorshDeserialize; - -use near_chain_configs::GenesisConfig; -use near_primitives::borsh; -use near_primitives::hash::CryptoHash; -use near_primitives::sharding::ChunkHash; -use near_store::{DBCol, ShardTries, Store}; - -mod validate; - -#[derive(Debug)] -pub struct ErrorMessage { - pub col: Option, - pub key: Option, - pub func: String, - pub reason: String, -} - -impl ErrorMessage { - fn new(func: String, reason: String) -> Self { - Self { col: None, key: None, func, reason } - } -} - -pub struct StoreValidator { - config: GenesisConfig, - shard_tries: ShardTries, - store: Arc, - - pub errors: Vec, - tests: u64, -} - -impl StoreValidator { - pub fn new(config: GenesisConfig, shard_tries: ShardTries, store: Arc) -> Self { - StoreValidator { - config, - shard_tries: shard_tries.clone(), - store: store.clone(), - errors: vec![], - tests: 0, - } - } - pub fn is_failed(&self) -> bool { - self.tests == 0 || self.errors.len() > 0 - } - pub fn num_failed(&self) -> u64 { - self.errors.len() as u64 - } - pub fn tests_done(&self) -> u64 { - self.tests - } - fn col_to_key(col: DBCol, key: &[u8]) -> String { - match col { - DBCol::ColBlockHeader | DBCol::ColBlock => { - format!("{:?}", CryptoHash::try_from(key.as_ref())) - } - DBCol::ColChunks => format!("{:?}", ChunkHash::try_from_slice(key.as_ref())), - _ => format!("{:?}", key), - } - } - pub fn validate(&mut self) { - self.check(&validate::nothing, &[0], &[0], DBCol::ColBlockMisc); - for (key, value) in self.store.clone().iter(DBCol::ColBlockHeader) { - // Block Header Hash is valid - self.check(&validate::block_header_validity, &key, &value, DBCol::ColBlockHeader); - } - for (key, value) in self.store.clone().iter(DBCol::ColBlock) { - // Block Hash is valid - self.check(&validate::block_hash_validity, &key, &value, DBCol::ColBlock); - // Block Header for current Block exists - self.check(&validate::block_header_exists, &key, &value, DBCol::ColBlock); - // Block Height is greater or equal to tail, or to Genesis Height - self.check(&validate::block_height_cmp_tail, &key, &value, DBCol::ColBlock); - } - for (key, value) in self.store.clone().iter(DBCol::ColChunks) { - // Chunk Hash is valid - self.check(&validate::chunk_hash_validity, &key, &value, DBCol::ColChunks); - // Block for current Chunk exists - self.check(&validate::block_of_chunk_exists, &key, &value, DBCol::ColChunks); - // There is a State Root in the Trie - self.check(&validate::chunks_state_roots_in_trie, &key, &value, DBCol::ColChunks); - } - } - - fn check( - &mut self, - f: &dyn Fn(&StoreValidator, &[u8], &[u8]) -> Result<(), ErrorMessage>, - key: &[u8], - value: &[u8], - col: DBCol, - ) { - let result = f(self, key, value); - self.tests += 1; - match result { - Ok(_) => {} - Err(e) => { - let mut e = e; - e.col = Some(col); - e.key = Some(Self::col_to_key(col, key)); - self.errors.push(e) - } - } - } -} diff --git a/test-utils/store-validator-bin/src/main.rs b/test-utils/store-validator/src/main.rs similarity index 91% rename from test-utils/store-validator-bin/src/main.rs rename to test-utils/store-validator/src/main.rs index 8b1a201aab2..f25f0c41103 100644 --- a/test-utils/store-validator-bin/src/main.rs +++ b/test-utils/store-validator/src/main.rs @@ -5,17 +5,17 @@ use std::sync::Arc; use ansi_term::Color::{Green, Red, White, Yellow}; use clap::{App, Arg, SubCommand}; +use near_chain::store_validator::StoreValidator; use near_chain::RuntimeAdapter; use near_logger_utils::init_integration_logger; use near_store::create_store; -use near_store_validator::StoreValidator; use neard::{get_default_home, get_store_path, load_config}; fn main() { init_integration_logger(); let default_home = get_default_home(); - let matches = App::new("store-validator-bin") + let matches = App::new("store-validator") .arg( Arg::with_name("home") .long("home") @@ -40,8 +40,9 @@ fn main() { )); let mut store_validator = StoreValidator::new( + near_config.validator_signer.as_ref().map(|x| x.validator_id().clone()), near_config.genesis.config.clone(), - runtime_adapter.get_tries(), + runtime_adapter.clone(), store.clone(), ); store_validator.validate();