From cac1b376af4170f656a506cca2efd15b0e3b765e Mon Sep 17 00:00:00 2001 From: Quantum Explorer Date: Wed, 8 Jan 2025 09:40:52 +0700 Subject: [PATCH] fmt --- grovedb-version/src/version/merk_versions.rs | 11 +- grovedb-version/src/version/mod.rs | 8 +- grovedb-version/src/version/v1.rs | 8 +- grovedb-version/src/version/v2.rs | 10 +- grovedb/src/batch/batch_structure.rs | 6 +- .../estimated_costs/average_case_costs.rs | 93 +++--- .../batch/estimated_costs/worst_case_costs.rs | 46 +-- .../batch/just_in_time_reference_update.rs | 8 +- grovedb/src/batch/mod.rs | 124 ++++--- .../src/batch/single_deletion_cost_tests.rs | 3 +- grovedb/src/element/constructor.rs | 2 +- grovedb/src/element/delete.rs | 26 +- grovedb/src/element/get.rs | 53 +-- grovedb/src/element/helpers.rs | 59 ++-- grovedb/src/element/insert.rs | 27 +- grovedb/src/element/mod.rs | 5 +- grovedb/src/element/query.rs | 4 +- .../src/estimated_costs/average_case_costs.rs | 104 +++--- .../src/estimated_costs/worst_case_costs.rs | 16 +- grovedb/src/lib.rs | 36 ++- grovedb/src/operations/delete/average_case.rs | 15 +- grovedb/src/operations/delete/mod.rs | 5 +- grovedb/src/operations/delete/worst_case.rs | 34 +- grovedb/src/operations/get/average_case.rs | 29 +- grovedb/src/operations/get/query.rs | 29 +- grovedb/src/operations/get/worst_case.rs | 13 +- grovedb/src/operations/is_empty_tree.rs | 3 +- grovedb/src/operations/proof/verify.rs | 5 +- grovedb/src/replication.rs | 6 +- grovedb/src/replication/state_sync_session.rs | 2 +- grovedb/src/tests/sum_tree_tests.rs | 59 +++- grovedb/src/util.rs | 14 +- .../src/estimated_costs/average_case_costs.rs | 306 +++++++++++++++++- merk/src/estimated_costs/mod.rs | 2 +- merk/src/estimated_costs/worst_case_costs.rs | 2 +- merk/src/merk/apply.rs | 4 +- merk/src/merk/mod.rs | 61 ++-- merk/src/merk/open.rs | 7 +- merk/src/merk/restore.rs | 13 +- merk/src/merk/source.rs | 2 +- merk/src/proofs/tree.rs | 18 +- merk/src/test_utils/mod.rs | 2 +- merk/src/test_utils/temp_merk.rs | 3 +- merk/src/tree/encoding.rs | 6 +- merk/src/tree/kv.rs | 12 +- merk/src/tree/link.rs | 48 ++- merk/src/tree/mod.rs | 39 ++- merk/src/tree/ops.rs | 2 +- merk/src/tree/tree_feature_type.rs | 73 +++-- merk/src/tree/walk/mod.rs | 4 +- node-grove/src/converter.rs | 4 + 51 files changed, 1022 insertions(+), 449 deletions(-) diff --git a/grovedb-version/src/version/merk_versions.rs b/grovedb-version/src/version/merk_versions.rs index fac25f913..f92d90132 100644 --- a/grovedb-version/src/version/merk_versions.rs +++ b/grovedb-version/src/version/merk_versions.rs @@ -1,2 +1,11 @@ +use versioned_feature_core::FeatureVersion; + #[derive(Clone, Debug, Default)] -pub struct MerkVersions {} +pub struct MerkVersions { + pub average_case_costs: MerkAverageCaseCostsVersions, +} + +#[derive(Clone, Debug, Default)] +pub struct MerkAverageCaseCostsVersions { + pub add_average_case_merk_propagate: FeatureVersion, +} diff --git a/grovedb-version/src/version/mod.rs b/grovedb-version/src/version/mod.rs index 8f6268419..d795176c1 100644 --- a/grovedb-version/src/version/mod.rs +++ b/grovedb-version/src/version/mod.rs @@ -6,7 +6,7 @@ pub mod v2; pub use versioned_feature_core::*; use crate::version::{ - grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2 + grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2, }; #[derive(Clone, Debug, Default)] @@ -17,6 +17,12 @@ pub struct GroveVersion { } impl GroveVersion { + pub fn first<'a>() -> &'a Self { + GROVE_VERSIONS + .first() + .expect("expected to have a platform version") + } + pub fn latest<'a>() -> &'a Self { GROVE_VERSIONS .last() diff --git a/grovedb-version/src/version/v1.rs b/grovedb-version/src/version/v1.rs index 97cfb38b3..71a1c71f3 100644 --- a/grovedb-version/src/version/v1.rs +++ b/grovedb-version/src/version/v1.rs @@ -8,7 +8,7 @@ use crate::version::{ GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBReplicationVersions, GroveDBVersions, }, - merk_versions::MerkVersions, + merk_versions::{MerkAverageCaseCostsVersions, MerkVersions}, GroveVersion, }; @@ -184,5 +184,9 @@ pub const GROVE_V1: GroveVersion = GroveVersion { apply_chunk: 0, }, }, - merk_versions: MerkVersions {}, + merk_versions: MerkVersions { + average_case_costs: MerkAverageCaseCostsVersions { + add_average_case_merk_propagate: 0, + }, + }, }; diff --git a/grovedb-version/src/version/v2.rs b/grovedb-version/src/version/v2.rs index 46ec8142d..304b034c1 100644 --- a/grovedb-version/src/version/v2.rs +++ b/grovedb-version/src/version/v2.rs @@ -8,7 +8,7 @@ use crate::version::{ GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions, GroveDBReplicationVersions, GroveDBVersions, }, - merk_versions::MerkVersions, + merk_versions::{MerkAverageCaseCostsVersions, MerkVersions}, GroveVersion, }; @@ -144,7 +144,7 @@ pub const GROVE_V2: GroveVersion = GroveVersion { }, average_case: GroveDBOperationsAverageCaseVersions { add_average_case_get_merk_at_path: 0, - average_case_merk_replace_tree: 1, //changed + average_case_merk_replace_tree: 1, // changed average_case_merk_insert_tree: 0, average_case_merk_delete_tree: 0, average_case_merk_insert_element: 0, @@ -184,5 +184,9 @@ pub const GROVE_V2: GroveVersion = GroveVersion { apply_chunk: 0, }, }, - merk_versions: MerkVersions {}, + merk_versions: MerkVersions { + average_case_costs: MerkAverageCaseCostsVersions { + add_average_case_merk_propagate: 1, + }, + }, }; diff --git a/grovedb/src/batch/batch_structure.rs b/grovedb/src/batch/batch_structure.rs index bd50c796a..8ed802f99 100644 --- a/grovedb/src/batch/batch_structure.rs +++ b/grovedb/src/batch/batch_structure.rs @@ -129,9 +129,9 @@ where } Ok(()) } - GroveOp::RefreshReference { .. } - | GroveOp::Delete - | GroveOp::DeleteTree(_) => Ok(()), + GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree(_) => { + Ok(()) + } GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => { Err(Error::InvalidBatchOperation( "replace and insert tree hash are internal operations only", diff --git a/grovedb/src/batch/estimated_costs/average_case_costs.rs b/grovedb/src/batch/estimated_costs/average_case_costs.rs index abd699302..93c6dccda 100644 --- a/grovedb/src/batch/estimated_costs/average_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/average_case_costs.rs @@ -10,18 +10,18 @@ use std::{ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::RootHashKeyAndAggregateData; #[cfg(feature = "full")] use grovedb_merk::{ estimated_costs::average_case_costs::{average_case_merk_propagate, EstimatedLayerInformation}, IsSumTree, }; +use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use grovedb_merk::merk::TreeType; + use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -53,28 +53,32 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::average_case_merk_replace_tree( - key, - layer_element_estimates, - aggregate_data, - propagate, - grove_version, - ), - GroveOp::InsertTreeWithRootHash { flags, sum, .. } => { - GroveDb::average_case_merk_insert_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => { + GroveDb::average_case_merk_replace_tree( key, - flags, - sum.is_some(), - in_tree_using_sums, - propagate_if_input(), + layer_element_estimates, + aggregate_data.parent_tree_type(), + propagate, grove_version, ) } + GroveOp::InsertTreeWithRootHash { + flags, + aggregate_data, + .. + } => GroveDb::average_case_merk_insert_tree( + key, + flags, + aggregate_data.parent_tree_type(), + in_tree_type, + propagate_if_input(), + grove_version, + ), GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { GroveDb::average_case_merk_insert_element( key, element, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ) @@ -91,14 +95,14 @@ impl GroveOp { *max_reference_hop, flags.clone(), ), - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), GroveOp::Replace { element } => GroveDb::average_case_merk_replace_element( key, element, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), @@ -109,7 +113,7 @@ impl GroveOp { key, element, *change_in_bytes, - in_tree_using_sums, + in_tree_type, propagate_if_input(), grove_version, ), @@ -121,7 +125,7 @@ impl GroveOp { ), GroveOp::DeleteTree(tree_type) => GroveDb::average_case_merk_delete_tree( key, - tree_type, + *tree_type, layer_element_estimates, propagate, grove_version, @@ -231,12 +235,11 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { &mut cost, path, layer_should_be_empty, - layer_info.is_sum_tree, + layer_info.tree_type, grove_version, ) ); - self.cached_merks - .insert(path.clone(), layer_info.is_sum_tree); + self.cached_merks.insert(path.clone(), layer_info.tree_type); } for (key, op) in ops_at_path_by_key.into_iter() { @@ -248,9 +251,10 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { cost_return_on_error!( &mut cost, - average_case_merk_propagate(layer_element_estimates).map_err(Error::MerkError) + average_case_merk_propagate(layer_element_estimates, grove_version) + .map_err(Error::MerkError) ); - Ok(([0u8; 32], None, None)).wrap_with_cost(cost) + Ok(([0u8; 32], None, AggregateData::NoAggregateData)).wrap_with_cost(cost) } fn update_base_merk_root_key( @@ -272,12 +276,12 @@ impl TreeCache for AverageCaseTreeCacheKnownPaths { estimated_layer_info .estimated_layer_count .estimated_to_be_empty(), - estimated_layer_info.is_sum_tree, + estimated_layer_info.tree_type, grove_version ) ); self.cached_merks - .insert(base_path, estimated_layer_info.is_sum_tree); + .insert(base_path, estimated_layer_info.tree_type); } } Ok(()).wrap_with_cost(cost) @@ -293,11 +297,14 @@ mod tests { storage_cost::{removal::StorageRemovedBytes::NoStorageRemoval, StorageCost}, OperationCost, }; - use grovedb_merk::estimated_costs::average_case_costs::{ - EstimatedLayerCount::{ApproximateElements, EstimatedLevel, PotentiallyAtMaxElements}, - EstimatedLayerInformation, - EstimatedLayerSizes::{AllItems, AllSubtrees}, - EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, + use grovedb_merk::{ + estimated_costs::average_case_costs::{ + EstimatedLayerCount::{ApproximateElements, EstimatedLevel, PotentiallyAtMaxElements}, + EstimatedLayerInformation, + EstimatedLayerSizes::{AllItems, AllSubtrees}, + EstimatedSumTrees::{NoSumTrees, SomeSumTrees}, + }, + merk::TreeType, }; use grovedb_version::version::GroveVersion; @@ -325,7 +332,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: ApproximateElements(0), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -394,7 +401,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, Some(3)), }, @@ -402,7 +409,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"key1".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -461,7 +468,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllItems(4, 3, None), }, @@ -534,7 +541,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), }, @@ -620,7 +627,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, false), estimated_layer_sizes: AllSubtrees(1, NoSumTrees, None), }, @@ -629,7 +636,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"0".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -700,7 +707,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees( 1, @@ -715,7 +722,7 @@ mod tests { paths.insert( KeyInfoPath::from_known_owned_path(vec![vec![7]]), EstimatedLayerInformation { - is_sum_tree: true, + tree_type: TreeType::SumTree, estimated_layer_count: PotentiallyAtMaxElements, estimated_layer_sizes: AllItems(32, 8, None), }, @@ -778,7 +785,7 @@ mod tests { paths.insert( KeyInfoPath(vec![]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(1, false), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, @@ -787,7 +794,7 @@ mod tests { paths.insert( KeyInfoPath(vec![KeyInfo::KnownKey(b"0".to_vec())]), EstimatedLayerInformation { - is_sum_tree: false, + tree_type: TreeType::NormalTree, estimated_layer_count: EstimatedLevel(0, true), estimated_layer_sizes: AllSubtrees(4, NoSumTrees, None), }, diff --git a/grovedb/src/batch/estimated_costs/worst_case_costs.rs b/grovedb/src/batch/estimated_costs/worst_case_costs.rs index 0eb4efe06..b22368731 100644 --- a/grovedb/src/batch/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/batch/estimated_costs/worst_case_costs.rs @@ -14,13 +14,13 @@ use grovedb_costs::{ use grovedb_merk::estimated_costs::worst_case_costs::{ worst_case_merk_propagate, WorstCaseLayerInformation, }; -use grovedb_merk::RootHashKeyAndAggregateData; +use grovedb_merk::{merk::TreeType, tree::AggregateData, RootHashKeyAndAggregateData}; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::version::GroveVersion; #[cfg(feature = "full")] use itertools::Itertools; -use grovedb_merk::merk::TreeType; + use crate::Element; #[cfg(feature = "full")] use crate::{ @@ -49,24 +49,28 @@ impl GroveOp { } }; match self { - GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => GroveDb::worst_case_merk_replace_tree( - key, - aggregate_data, - in_parent_tree_type, - worst_case_layer_element_estimates, - propagate, - grove_version, - ), - GroveOp::InsertTreeWithRootHash { flags, aggregate_data, .. } => { - GroveDb::worst_case_merk_insert_tree( + GroveOp::ReplaceTreeRootKey { aggregate_data, .. } => { + GroveDb::worst_case_merk_replace_tree( key, - flags, - aggregate_data, + aggregate_data.parent_tree_type(), in_parent_tree_type, - propagate_if_input(), + worst_case_layer_element_estimates, + propagate, grove_version, ) } + GroveOp::InsertTreeWithRootHash { + flags, + aggregate_data, + .. + } => GroveDb::worst_case_merk_insert_tree( + key, + flags, + aggregate_data.parent_tree_type(), + in_parent_tree_type, + propagate_if_input(), + grove_version, + ), GroveOp::InsertOrReplace { element } | GroveOp::InsertOnly { element } => { GroveDb::worst_case_merk_insert_element( key, @@ -117,7 +121,7 @@ impl GroveOp { ), GroveOp::DeleteTree(tree_type) => GroveDb::worst_case_merk_delete_tree( key, - tree_type, + *tree_type, worst_case_layer_element_estimates, propagate, grove_version, @@ -156,7 +160,7 @@ impl fmt::Debug for WorstCaseTreeCacheKnownPaths { #[cfg(feature = "full")] impl TreeCache for WorstCaseTreeCacheKnownPaths { - fn insert(&mut self, op: &QualifiedGroveDbOp, _is_sum_tree: bool) -> CostResult<(), Error> { + fn insert(&mut self, op: &QualifiedGroveDbOp, _tree_type: TreeType) -> CostResult<(), Error> { let mut worst_case_cost = OperationCost::default(); let mut inserted_path = op.path.clone(); inserted_path.push(op.key.clone()); @@ -201,7 +205,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, - false, + TreeType::NormalTree, grove_version, ) ); @@ -213,7 +217,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { &mut cost, op.worst_case_cost( &key, - false, + TreeType::NormalTree, worst_case_layer_element_estimates, false, grove_version @@ -225,7 +229,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { &mut cost, worst_case_merk_propagate(worst_case_layer_element_estimates).map_err(Error::MerkError) ); - Ok(([0u8; 32], None, None)).wrap_with_cost(cost) + Ok(([0u8; 32], None, AggregateData::NoAggregateData)).wrap_with_cost(cost) } fn update_base_merk_root_key( @@ -244,7 +248,7 @@ impl TreeCache for WorstCaseTreeCacheKnownPaths { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, &base_path, - false, + TreeType::NormalTree, grove_version, ) ); diff --git a/grovedb/src/batch/just_in_time_reference_update.rs b/grovedb/src/batch/just_in_time_reference_update.rs index 9fb1ec158..e0fd5f94c 100644 --- a/grovedb/src/batch/just_in_time_reference_update.rs +++ b/grovedb/src/batch/just_in_time_reference_update.rs @@ -9,10 +9,10 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ + merk::TreeType, tree::{kv::KV, value_hash, TreeNode}, CryptoHash, Merk, }; -use grovedb_merk::merk::TreeType; use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; @@ -103,7 +103,11 @@ where in_tree_type.inner_node_type(), ) } else { - KV::node_value_byte_cost_size(key.len() as u32, serialized.len() as u32, in_tree_type.inner_node_type()) + KV::node_value_byte_cost_size( + key.len() as u32, + serialized.len() as u32, + in_tree_type.inner_node_type(), + ) }; let mut i = 0; diff --git a/grovedb/src/batch/mod.rs b/grovedb/src/batch/mod.rs index 87bd59f62..97b0ed721 100644 --- a/grovedb/src/batch/mod.rs +++ b/grovedb/src/batch/mod.rs @@ -48,12 +48,12 @@ use grovedb_costs::{ CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ + merk::TreeType, tree::{ kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, - value_hash, NULL_HASH, + value_hash, AggregateData, NULL_HASH, }, CryptoHash, Error as MerkError, Merk, MerkType, Op, RootHashKeyAndAggregateData, - TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; use grovedb_path::SubtreePath; use grovedb_storage::{ @@ -66,8 +66,6 @@ use grovedb_version::{ use grovedb_visualize::{Drawer, Visualize}; use integer_encoding::VarInt; use itertools::Itertools; -use grovedb_merk::merk::{NodeType, TreeType}; -use grovedb_merk::tree::AggregateData; use key_info::{KeyInfo, KeyInfo::KnownKey}; pub use options::BatchApplyOptions; @@ -76,7 +74,10 @@ pub use crate::batch::batch_structure::{OpsByLevelPath, OpsByPath}; use crate::batch::estimated_costs::EstimatedCostsType; use crate::{ batch::{batch_structure::BatchStructure, mode::BatchRunMode}, - element::{MaxReferenceHop, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{ + MaxReferenceHop, BIG_SUM_TREE_COST_SIZE, COUNT_TREE_COST_SIZE, SUM_ITEM_COST_SIZE, + SUM_TREE_COST_SIZE, TREE_COST_SIZE, + }, operations::{get::MAX_REFERENCE_HOPS, proof::util::hex_to_ascii}, reference_path::{ path_from_reference_path_type, path_from_reference_qualified_path_type, ReferencePathType, @@ -854,10 +855,10 @@ where /// /// # Returns /// - /// * `Ok((Element, Vec, TreeType))` - Returns the deserialized `Element` - /// and the serialized counterpart if the retrieval and deserialization - /// are successful, wrapped in the associated cost. Also returns if the - /// merk of the element is a sum tree as a TreeType. + /// * `Ok((Element, Vec, TreeType))` - Returns the deserialized + /// `Element` and the serialized counterpart if the retrieval and + /// deserialization are successful, wrapped in the associated cost. Also + /// returns if the merk of the element is a sum tree as a TreeType. /// * `Err(Error)` - Returns an error if any issue occurs during the /// retrieval or deserialization of the referenced element. /// @@ -1018,7 +1019,10 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1136,12 +1140,13 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { - Err(Error::InvalidBatchOperation( - "references can not point to trees being updated", - )) - .wrap_with_cost(cost) - } + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + "references can not point to trees being updated", + )) + .wrap_with_cost(cost), } } GroveOp::InsertOnly { element } => match element { @@ -1165,7 +1170,10 @@ where grove_version, ) } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidBatchOperation( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidBatchOperation( "references can not point to trees being updated", )) .wrap_with_cost(cost), @@ -1191,12 +1199,10 @@ where grove_version, ) } - GroveOp::Delete | GroveOp::DeleteTree(_) => { - Err(Error::InvalidBatchOperation( - "references can not point to something currently being deleted", - )) - .wrap_with_cost(cost) - } + GroveOp::Delete | GroveOp::DeleteTree(_) => Err(Error::InvalidBatchOperation( + "references can not point to something currently being deleted", + )) + .wrap_with_cost(cost), } } else { self.process_reference( @@ -1339,7 +1345,10 @@ where ) ); } - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let merk_feature_type = cost_return_on_error!( &mut cost, element @@ -1489,7 +1498,7 @@ where key_info.get_key(), false, in_tree_type, /* we are in a sum tree, this might or might not be a - * sum item */ + * sum item */ &mut batch_operations, grove_version ) @@ -1533,14 +1542,26 @@ where aggregate_data, } => { let element = match aggregate_data { - AggregateData::NoAggregateData => Element::new_tree_with_flags(root_key, flags), - AggregateData::Sum(sum_value) => Element::new_sum_tree_with_flags_and_sum_value( - root_key, aggregate_data.as_i64(), flags, - ), - AggregateData::BigSum(sum_value) => Element::new_big_sum_tree_with_flags_and_sum_value( - root_key, aggregate_data.as_i128(), flags, - ), - AggregateData::Count(count_value) => Element::new_count_tree_with_flags_and_count_value(root_key, aggregate_data.as_u64(), flags) + AggregateData::NoAggregateData => { + Element::new_tree_with_flags(root_key, flags) + } + AggregateData::Sum(sum_value) => { + Element::new_sum_tree_with_flags_and_sum_value( + root_key, sum_value, flags, + ) + } + AggregateData::BigSum(sum_value) => { + Element::new_big_sum_tree_with_flags_and_sum_value( + root_key, sum_value, flags, + ) + } + AggregateData::Count(count_value) => { + Element::new_count_tree_with_flags_and_count_value( + root_key, + count_value, + flags, + ) + } }; let merk_feature_type = cost_return_on_error_no_add!(&cost, element.get_feature_type(in_tree_type)); @@ -1569,8 +1590,13 @@ where &[], Some(batch_apply_options.as_merk_options()), &|key, value| { - Element::specialized_costs_for_key_value(key, value, in_tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Element::specialized_costs_for_key_value( + key, + value, + in_tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &|old_value, new_value| { @@ -1621,7 +1647,10 @@ where // we need to give back the value defined cost in the case that the // new element is a tree match new_element { - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let tree_type = new_element.tree_type().unwrap(); let tree_cost_size = match tree_type { TreeType::NormalTree => TREE_COST_SIZE, @@ -1820,7 +1849,8 @@ impl GroveDb { hash: root_hash, root_key: calculated_root_key, flags: flags.clone(), - aggregate_data: AggregateData::NoAggregateData, + aggregate_data: + AggregateData::NoAggregateData, } .into(); } else if let Element::SumTree(.., flags) = @@ -1848,8 +1878,7 @@ impl GroveDb { )) .wrap_with_cost(cost); } - GroveOp::Delete - | GroveOp::DeleteTree(_) => { + GroveOp::Delete | GroveOp::DeleteTree(_) => { if calculated_root_key.is_some() { return Err(Error::InvalidBatchOperation( "modification of tree when it will be \ @@ -2140,7 +2169,12 @@ impl GroveDb { if let Some((parent_path, parent_key)) = path.derive_parent() { if new_merk { // TODO: can this be a sum tree - Ok(Merk::open_empty(storage, MerkType::LayeredMerk, TreeType::NormalTree)).wrap_with_cost(cost) + Ok(Merk::open_empty( + storage, + MerkType::LayeredMerk, + TreeType::NormalTree, + )) + .wrap_with_cost(cost) } else { let parent_storage = self .db @@ -2183,7 +2217,12 @@ impl GroveDb { } } } else if new_merk { - Ok(Merk::open_empty(storage, MerkType::BaseMerk, TreeType::NormalTree)).wrap_with_cost(cost) + Ok(Merk::open_empty( + storage, + MerkType::BaseMerk, + TreeType::NormalTree, + )) + .wrap_with_cost(cost) } else { Merk::open_base( storage, @@ -2223,7 +2262,8 @@ impl GroveDb { } else { MerkType::LayeredMerk }; - Ok(Merk::open_empty(storage, merk_type, TreeType::NormalTree)).wrap_with_cost(local_cost) + Ok(Merk::open_empty(storage, merk_type, TreeType::NormalTree)) + .wrap_with_cost(local_cost) } else if let Some((base_path, last)) = path.derive_parent() { let parent_storage = self .db diff --git a/grovedb/src/batch/single_deletion_cost_tests.rs b/grovedb/src/batch/single_deletion_cost_tests.rs index fceccce6e..ee5c6902b 100644 --- a/grovedb/src/batch/single_deletion_cost_tests.rs +++ b/grovedb/src/batch/single_deletion_cost_tests.rs @@ -7,9 +7,10 @@ mod tests { Identifier, StorageRemovalPerEpochByIdentifier, StorageRemovedBytes::SectionedStorageRemoval, }; + use grovedb_merk::merk::TreeType; use grovedb_version::version::GroveVersion; use intmap::IntMap; - use grovedb_merk::merk::TreeType; + use crate::{ batch::QualifiedGroveDbOp, tests::{common::EMPTY_PATH, make_empty_grovedb}, diff --git a/grovedb/src/element/constructor.rs b/grovedb/src/element/constructor.rs index 3cc381a62..556df1dac 100644 --- a/grovedb/src/element/constructor.rs +++ b/grovedb/src/element/constructor.rs @@ -1,13 +1,13 @@ //! Constructor //! Functions for setting an element's type +use crate::element::{BigSumValue, CountValue}; #[cfg(feature = "full")] use crate::{ element::{MaxReferenceHop, SumValue}, reference_path::ReferencePathType, Element, ElementFlags, }; -use crate::element::{BigSumValue, CountValue}; impl Element { #[cfg(feature = "full")] diff --git a/grovedb/src/element/delete.rs b/grovedb/src/element/delete.rs index af2992f1f..2ab01e02f 100644 --- a/grovedb/src/element/delete.rs +++ b/grovedb/src/element/delete.rs @@ -5,9 +5,9 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use grovedb_costs::{storage_cost::removal::StorageRemovedBytes, CostResult, CostsExt}; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_merk::{BatchEntry, Error as MerkError, Merk, MerkOptions, Op}; -use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::StorageContext; #[cfg(feature = "full")] @@ -43,14 +43,20 @@ impl Element { | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary + // todo not sure we get it again, we need to see if this is necessary + let tree_type = merk.tree_type; merk.apply_with_specialized_costs::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -94,14 +100,20 @@ impl Element { | (TreeType::CountTree, false) => Op::DeleteMaybeSpecialized, }; let batch = [(key, op)]; - let tree_type = merk.tree_type; //todo not sure we get it again, we need to see if this is necessary + // todo not sure we get it again, we need to see if this is necessary + let tree_type = merk.tree_type; merk.apply_with_costs_just_in_time_value_update::<_, Vec>( &batch, &[], merk_options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), &|_, _| Ok(None), diff --git a/grovedb/src/element/get.rs b/grovedb/src/element/get.rs index 876971a79..f61a20d61 100644 --- a/grovedb/src/element/get.rs +++ b/grovedb/src/element/get.rs @@ -5,19 +5,19 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::tree::kv::KV; #[cfg(feature = "full")] use grovedb_merk::Merk; #[cfg(feature = "full")] use grovedb_merk::{ed::Decode, tree::TreeNodeInner}; +use grovedb_merk::{merk::NodeType, tree::kv::KV}; #[cfg(feature = "full")] use grovedb_storage::StorageContext; use grovedb_version::{ check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::NodeType; -use crate::element::{CostSize, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}; + +use crate::element::{CostSize, SUM_ITEM_COST_SIZE}; #[cfg(feature = "full")] use crate::{Element, Error, Hash}; @@ -121,14 +121,18 @@ impl Element { match grove_version .grovedb_versions .element - .get_optional_from_storage { + .get_optional_from_storage + { 0 => Self::get_optional_from_storage_v0(storage, key, grove_version), 1 => Self::get_optional_from_storage_v1(storage, key, grove_version), - version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { - method: "get_optional_from_storage".to_string(), - known_versions: vec![0, 1], - received: version, - })).wrap_with_cost(OperationCost::default()) + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "get_optional_from_storage".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), } } @@ -187,10 +191,16 @@ impl Element { flags_len + flags_len.required_space() as u32 }); let value_len = cost_size + flags_len; - cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, NodeType::NormalNode) as u64 + cost.storage_loaded_bytes = KV::node_value_byte_cost_size( + key_ref.len() as u32, + value_len, + NodeType::NormalNode, + ) as u64 } - Some(Element::Tree(_, flags)) | Some(Element::SumTree(_, _, flags )) | Some(Element::BigSumTree(_, _, flags )) | Some(Element::CountTree(_, _, flags )) => { + Some(Element::Tree(_, flags)) + | Some(Element::SumTree(_, _, flags)) + | Some(Element::BigSumTree(_, _, flags)) + | Some(Element::CountTree(_, _, flags)) => { let tree_cost_size = element.as_ref().unwrap().tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; @@ -209,7 +219,6 @@ impl Element { Ok(element).wrap_with_cost(cost) } - #[cfg(feature = "full")] /// Get an element directly from storage under a key /// Merk does not need to be loaded @@ -236,15 +245,17 @@ impl Element { .transpose() ); - let Some((value, tree_feature_type)) = maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned_with_feature()) else { - return Ok(None).wrap_with_cost(cost) + let Some((value, tree_feature_type)) = + maybe_tree_inner.map(|tree_inner| tree_inner.value_as_owned_with_feature()) + else { + return Ok(None).wrap_with_cost(cost); }; let node_type = tree_feature_type.node_type(); let element = cost_return_on_error_no_add!( &cost, Self::deserialize(value.as_slice(), grove_version).map_err(|_| { - Error::CorruptedData(String::from("unable to deserialize element")) - }) + Error::CorruptedData(String::from("unable to deserialize element")) + }) ); match &element { Element::Item(..) | Element::Reference(..) => { @@ -264,9 +275,13 @@ impl Element { }); let value_len = cost_size + flags_len; cost.storage_loaded_bytes = - KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, node_type) as u64 // this is changed to sum node in v1 + KV::node_value_byte_cost_size(key_ref.len() as u32, value_len, node_type) as u64 + // this is changed to sum node in v1 } - Element::Tree(_, flags) | Element::SumTree(_, _, flags ) | Element::BigSumTree(_, _, flags ) | Element::CountTree(_, _, flags ) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let tree_cost_size = element.tree_type().unwrap().cost_size(); let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; diff --git a/grovedb/src/element/helpers.rs b/grovedb/src/element/helpers.rs index 4b0ecc80d..f22a2c7e4 100644 --- a/grovedb/src/element/helpers.rs +++ b/grovedb/src/element/helpers.rs @@ -6,6 +6,10 @@ use grovedb_merk::tree::kv::{ ValueDefinedCostType, ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, }; +use grovedb_merk::{ + merk::{NodeType, TreeType}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(feature = "full")] use grovedb_merk::{ tree::{kv::KV, TreeNode}, @@ -16,8 +20,8 @@ use grovedb_merk::{ use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; #[cfg(feature = "full")] use integer_encoding::VarInt; -use grovedb_merk::merk::{NodeType, TreeType}; -use grovedb_merk::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; + +use crate::element::BIG_SUM_TREE_COST_SIZE; #[cfg(feature = "full")] use crate::reference_path::path_from_reference_path_type; #[cfg(any(feature = "full", feature = "verify"))] @@ -29,7 +33,6 @@ use crate::{ }; #[cfg(any(feature = "full", feature = "verify"))] use crate::{Element, Error}; -use crate::element::BIG_SUM_TREE_COST_SIZE; impl Element { #[cfg(any(feature = "full", feature = "verify"))] @@ -47,7 +50,9 @@ impl Element { /// everything else pub fn big_sum_value_or_default(&self) -> i128 { match self { - Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => *sum_value as i128, + Element::SumItem(sum_value, _) | Element::SumTree(_, sum_value, _) => { + *sum_value as i128 + } Element::BigSumTree(_, sum_value, _) => *sum_value, _ => 0, } @@ -123,25 +128,27 @@ impl Element { } #[cfg(any(feature = "full", feature = "verify"))] - /// Check if the element is a tree and return the root_tree info and tree type + /// Check if the element is a tree and return the root_tree info and tree + /// type pub fn root_key_and_tree_type_owned(self) -> Option<(Option>, TreeType)> { match self { Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), - Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), - Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), - Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), _ => None, } } #[cfg(any(feature = "full", feature = "verify"))] - /// Check if the element is a tree and return the root_tree info and the tree type + /// Check if the element is a tree and return the root_tree info and the + /// tree type pub fn root_key_and_tree_type(&self) -> Option<(&Option>, TreeType)> { match self { Element::Tree(root_key, _) => Some((root_key, TreeType::NormalTree)), - Element::SumTree(root_key, _, _) => Some((root_key, TreeType::SumTree)), - Element::BigSumTree(root_key, _, _) => Some((root_key, TreeType::BigSumTree)), - Element::CountTree(root_key, _, _) => Some((root_key, TreeType::CountTree)), + Element::SumTree(root_key, ..) => Some((root_key, TreeType::SumTree)), + Element::BigSumTree(root_key, ..) => Some((root_key, TreeType::BigSumTree)), + Element::CountTree(root_key, ..) => Some((root_key, TreeType::CountTree)), _ => None, } } @@ -162,10 +169,10 @@ impl Element { /// Check if the element is a tree and return the tree type pub fn tree_type(&self) -> Option { match self { - Element::Tree(_, _) => Some(TreeType::NormalTree), - Element::SumTree(_, _, _) => Some(TreeType::SumTree), - Element::BigSumTree(_, _, _) => Some(TreeType::BigSumTree), - Element::CountTree(_, _, _) => Some(TreeType::CountTree), + Element::Tree(..) => Some(TreeType::NormalTree), + Element::SumTree(..) => Some(TreeType::SumTree), + Element::BigSumTree(..) => Some(TreeType::BigSumTree), + Element::CountTree(..) => Some(TreeType::CountTree), _ => None, } } @@ -185,7 +192,13 @@ impl Element { #[cfg(any(feature = "full", feature = "verify"))] /// Check if the element is a tree pub fn is_any_tree(&self) -> bool { - matches!(self, Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..)) + matches!( + self, + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) + ) } #[cfg(any(feature = "full", feature = "verify"))] @@ -350,9 +363,7 @@ impl Element { let value_len = TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::SumTree(_, _sum_value, flags) => { @@ -363,9 +374,7 @@ impl Element { let value_len = SUM_TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::BigSumTree(_, _sum_value, flags) => { @@ -376,9 +385,7 @@ impl Element { let value_len = BIG_SUM_TREE_COST_SIZE + flags_len; let key_len = key.len() as u32; KV::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_len, - node_type, + key_len, value_len, node_type, ) } Element::SumItem(.., flags) => { diff --git a/grovedb/src/element/insert.rs b/grovedb/src/element/insert.rs index 2dd56da5f..f32a1f358 100644 --- a/grovedb/src/element/insert.rs +++ b/grovedb/src/element/insert.rs @@ -62,8 +62,13 @@ impl Element { options, &|key, value| { // it is possible that a normal item was being replaced with a - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -320,8 +325,13 @@ impl Element { &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, @@ -407,8 +417,13 @@ impl Element { &[], options, &|key, value| { - Self::specialized_costs_for_key_value(key, value, tree_type.inner_node_type(), grove_version) - .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) + Self::specialized_costs_for_key_value( + key, + value, + tree_type.inner_node_type(), + grove_version, + ) + .map_err(|e| MerkError::ClientCorruptionError(e.to_string())) }, Some(&Element::value_defined_cost_for_serialized_value), grove_version, diff --git a/grovedb/src/element/mod.rs b/grovedb/src/element/mod.rs index b09844588..2784bb98f 100644 --- a/grovedb/src/element/mod.rs +++ b/grovedb/src/element/mod.rs @@ -27,7 +27,9 @@ use bincode::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use grovedb_merk::estimated_costs::SUM_VALUE_EXTRA_COST; #[cfg(feature = "full")] -use grovedb_merk::estimated_costs::{LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, BIG_SUM_LAYER_COST_SIZE, BIG_SUM_VALUE_EXTRA_COST}; +use grovedb_merk::estimated_costs::{ + BIG_SUM_LAYER_COST_SIZE, LAYER_COST_SIZE, SUM_LAYER_COST_SIZE, +}; #[cfg(feature = "full")] use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] @@ -97,7 +99,6 @@ impl CostSize for TreeType { } } - #[cfg(any(feature = "full", feature = "verify"))] /// Variants of GroveDB stored entities /// diff --git a/grovedb/src/element/query.rs b/grovedb/src/element/query.rs index 5a94797c4..0df089719 100644 --- a/grovedb/src/element/query.rs +++ b/grovedb/src/element/query.rs @@ -9,6 +9,8 @@ use grovedb_costs::{ OperationCost, }; #[cfg(feature = "full")] +use grovedb_merk::merk::TreeType; +#[cfg(feature = "full")] use grovedb_merk::proofs::query::query_item::QueryItem; #[cfg(feature = "full")] use grovedb_merk::proofs::query::SubqueryBranch; @@ -22,8 +24,6 @@ use grovedb_storage::{rocksdb_storage::RocksDbStorage, RawIterator, StorageConte use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; -#[cfg(feature = "full")] -use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use crate::operations::proof::util::hex_to_ascii; diff --git a/grovedb/src/estimated_costs/average_case_costs.rs b/grovedb/src/estimated_costs/average_case_costs.rs index 32248516e..ed07857ee 100644 --- a/grovedb/src/estimated_costs/average_case_costs.rs +++ b/grovedb/src/estimated_costs/average_case_costs.rs @@ -12,6 +12,7 @@ use grovedb_merk::{ add_average_case_merk_replace_layered, EstimatedLayerInformation, }, }, + merk::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -20,13 +21,12 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, - element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{CostSize, SUM_ITEM_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; -use crate::element::CostSize; impl GroveDb { /// Add average case for getting a merk tree @@ -78,14 +78,30 @@ impl GroveDb { .grovedb_versions .operations .average_case - .average_case_merk_replace_tree { - 0 => Self::average_case_merk_replace_tree_v0(key, estimated_layer_information, replacing_tree_type, propagate), - 1 => Self::average_case_merk_replace_tree_v1(key, estimated_layer_information, replacing_tree_type, propagate), - version => Err(Error::VersionError(GroveVersionError::UnknownVersionMismatch { - method: "average_case_merk_replace_tree".to_string(), - known_versions: vec![0, 1], - received: version, - })).wrap_with_cost(OperationCost::default()) + .average_case_merk_replace_tree + { + 0 => Self::average_case_merk_replace_tree_v0( + key, + estimated_layer_information, + replacing_tree_type, + propagate, + grove_version, + ), + 1 => Self::average_case_merk_replace_tree_v1( + key, + estimated_layer_information, + replacing_tree_type, + propagate, + grove_version, + ), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "average_case_merk_replace_tree".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )) + .wrap_with_cost(OperationCost::default()), } } @@ -95,9 +111,10 @@ impl GroveDb { estimated_layer_information: &EstimatedLayerInformation, _replacing_tree_type: TreeType, propagate: bool, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { - // In v0 we used the estimated layer information tree type (which is the parent) in order - // to figure out the cost + // In v0 we used the estimated layer information tree type (which is the parent) + // in order to figure out the cost let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; let flags_size = cost_return_on_error_no_add!( @@ -109,7 +126,7 @@ impl GroveDb { ) .map(|f| f + f.required_space() as u32) .unwrap_or_default(); - let tree_cost_size = estimated_layer_information.tree_type.cost_size(); //this was wrong + let tree_cost_size = estimated_layer_information.tree_type.cost_size(); // this was wrong let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_replace_layered( &mut cost, @@ -118,7 +135,7 @@ impl GroveDb { estimated_layer_information.tree_type.inner_node_type(), ); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -132,6 +149,7 @@ impl GroveDb { estimated_layer_information: &EstimatedLayerInformation, replacing_tree_type: TreeType, propagate: bool, + grove_version: &GroveVersion, ) -> CostResult<(), Error> { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; @@ -142,8 +160,8 @@ impl GroveDb { .layered_flags_size() .map_err(Error::MerkError) ) - .map(|f| f + f.required_space() as u32) - .unwrap_or_default(); + .map(|f| f + f.required_space() as u32) + .unwrap_or_default(); let tree_cost_size = replacing_tree_type.cost_size(); let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_replace_layered( @@ -153,12 +171,12 @@ impl GroveDb { estimated_layer_information.tree_type.inner_node_type(), ); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) } - .wrap_with_cost(cost) + .wrap_with_cost(cost) } /// Add average case for insertion into merk @@ -189,7 +207,8 @@ impl GroveDb { let value_len = tree_cost_size + flags_len; add_cost_case_merk_insert_layered(&mut cost, key_len, value_len, in_parent_tree_type); if let Some(input) = propagate_if_input { - add_average_case_merk_propagate(&mut cost, input).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, input, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -228,7 +247,7 @@ impl GroveDb { let layer_extra_size = tree_cost_size + flags_size; add_average_case_merk_delete_layered(&mut cost, key_len, layer_extra_size); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -274,7 +293,8 @@ impl GroveDb { ) } if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -303,19 +323,17 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 }); let tree_cost_size = value.tree_type().unwrap().cost_size(); let value_len = tree_cost_size + flags_len; - add_cost_case_merk_replace_layered( - &mut cost, - key_len, - value_len, - in_tree_type, - ) + add_cost_case_merk_replace_layered(&mut cost, key_len, value_len, in_tree_type) } Element::Item(_, flags) | Element::SumItem(_, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { @@ -329,12 +347,7 @@ impl GroveDb { cost_return_on_error_no_add!(&cost, value.serialized_size(grove_version)) as u32 }; let value_len = sum_item_cost_size + flags_len; - add_cost_case_merk_replace_same_size( - &mut cost, - key_len, - value_len, - in_tree_type, - ) + add_cost_case_merk_replace_same_size(&mut cost, key_len, value_len, in_tree_type) } _ => add_cost_case_merk_replace_same_size( &mut cost, @@ -344,7 +357,8 @@ impl GroveDb { ), }; if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -398,7 +412,8 @@ impl GroveDb { } }; if let Some(level) = propagate_for_level { - add_average_case_merk_propagate(&mut cost, level).map_err(Error::MerkError) + add_average_case_merk_propagate(&mut cost, level, grove_version) + .map_err(Error::MerkError) } else { Ok(()) } @@ -432,7 +447,7 @@ impl GroveDb { ); add_average_case_merk_delete(&mut cost, key_len, value_size); if propagate { - add_average_case_merk_propagate(&mut cost, estimated_layer_information) + add_average_case_merk_propagate(&mut cost, estimated_layer_information, grove_version) .map_err(Error::MerkError) } else { Ok(()) @@ -600,7 +615,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ - estimated_costs::average_case_costs::add_average_case_get_merk_node, + estimated_costs::average_case_costs::add_average_case_get_merk_node, merk::TreeType, test_utils::make_batch_seq, tree::kv::ValueDefinedCostType, Merk, }; use grovedb_storage::{ @@ -608,7 +623,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, @@ -672,8 +687,13 @@ mod test { // (this will be the max_element_size) let mut cost = OperationCost::default(); let key = KnownKey(8_u64.to_be_bytes().to_vec()); - add_average_case_get_merk_node(&mut cost, key.max_length() as u32, 60, TreeType::NormalTree.inner_node_type()) - .expect("expected to add cost"); + add_average_case_get_merk_node( + &mut cost, + key.max_length() as u32, + 60, + TreeType::NormalTree.inner_node_type(), + ) + .expect("expected to add cost"); assert_eq!(cost, node_result.cost); } diff --git a/grovedb/src/estimated_costs/worst_case_costs.rs b/grovedb/src/estimated_costs/worst_case_costs.rs index d745669dc..54cabcc42 100644 --- a/grovedb/src/estimated_costs/worst_case_costs.rs +++ b/grovedb/src/estimated_costs/worst_case_costs.rs @@ -14,6 +14,7 @@ use grovedb_merk::{ MERK_BIGGEST_VALUE_SIZE, }, }, + merk::TreeType, tree::TreeNode, HASH_LENGTH, }; @@ -22,13 +23,12 @@ use grovedb_version::{ check_grovedb_v0, check_grovedb_v0_with_cost, error::GroveVersionError, version::GroveVersion, }; use integer_encoding::VarInt; -use grovedb_merk::merk::TreeType; + use crate::{ batch::{key_info::KeyInfo, KeyInfoPath}, - element::{SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, + element::{CostSize, SUM_ITEM_COST_SIZE, SUM_TREE_COST_SIZE, TREE_COST_SIZE}, Element, ElementFlags, Error, GroveDb, }; -use crate::element::CostSize; pub const WORST_CASE_FLAGS_LEN: u32 = 16386; // 2 bytes to represent this number for varint @@ -56,7 +56,7 @@ impl GroveDb { cost.storage_loaded_bytes += TreeNode::worst_case_encoded_tree_size( key.max_length() as u32, HASH_LENGTH as u32, - tree_type.inner_node_type(), //todo This is probably wrong + tree_type.inner_node_type(), // todo This is probably wrong ) as u64; } } @@ -189,7 +189,10 @@ impl GroveDb { let mut cost = OperationCost::default(); let key_len = key.max_length() as u32; match value { - Element::Tree(_, flags) | Element::SumTree(_, _, flags) | Element::BigSumTree(_, _, flags) | Element::CountTree(_, _, flags) => { + Element::Tree(_, flags) + | Element::SumTree(_, _, flags) + | Element::BigSumTree(_, _, flags) + | Element::CountTree(_, _, flags) => { let flags_len = flags.as_ref().map_or(0, |flags| { let flags_len = flags.len() as u32; flags_len + flags_len.required_space() as u32 @@ -494,6 +497,7 @@ mod test { use grovedb_costs::OperationCost; use grovedb_merk::{ estimated_costs::worst_case_costs::add_worst_case_get_merk_node, + merk::{NodeType, TreeType}, test_utils::{empty_path_merk, empty_path_merk_read_only, make_batch_seq}, tree::kv::ValueDefinedCostType, }; @@ -504,7 +508,7 @@ mod test { }; use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use grovedb_merk::merk::{NodeType, TreeType}; + use crate::{ batch::{key_info::KeyInfo::KnownKey, KeyInfoPath}, tests::{common::EMPTY_PATH, TEST_LEAF}, diff --git a/grovedb/src/lib.rs b/grovedb/src/lib.rs index f89bd9c2a..84cfeac53 100644 --- a/grovedb/src/lib.rs +++ b/grovedb/src/lib.rs @@ -186,6 +186,7 @@ use grovedb_merk::{ tree::{combine_hash, value_hash}, BatchEntry, CryptoHash, KVIterator, Merk, }; +use grovedb_merk::{merk::TreeType, tree::AggregateData}; #[cfg(feature = "full")] use grovedb_path::SubtreePath; #[cfg(feature = "full")] @@ -209,8 +210,7 @@ pub use query::{PathQuery, SizedQuery}; use reference_path::path_from_reference_path_type; #[cfg(feature = "grovedbg")] use tokio::net::ToSocketAddrs; -use grovedb_merk::merk::TreeType; -use grovedb_merk::tree::AggregateData; + #[cfg(feature = "full")] use crate::element::helpers::raw_decode; #[cfg(any(feature = "full", feature = "verify"))] @@ -422,7 +422,10 @@ impl GroveDb { }) .unwrap()?; let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + if let Element::Tree(root_key, _) + | Element::SumTree(root_key, ..) + | Element::BigSumTree(root_key, ..) = element + { let tree_type = tree_type.expect("expected tree type"); Ok(( Merk::open_layered_with_root_key( @@ -496,7 +499,10 @@ impl GroveDb { ) ); let tree_type = element.tree_type(); - if let Element::Tree(root_key, _) | Element::SumTree(root_key, ..) | Element::BigSumTree(root_key, ..) = element { + if let Element::Tree(root_key, _) + | Element::SumTree(root_key, ..) + | Element::BigSumTree(root_key, ..) = element + { let tree_type = tree_type.expect("expected tree type"); Merk::open_layered_with_root_key( storage, @@ -655,7 +661,9 @@ impl GroveDb { ); let (root_hash, root_key, aggregate_data) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -709,7 +717,9 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -760,7 +770,9 @@ impl GroveDb { ); let (root_hash, root_key, sum) = cost_return_on_error!( &mut cost, - child_tree.root_hash_key_and_aggregate_data().map_err(Error::MerkError) + child_tree + .root_hash_key_and_aggregate_data() + .map_err(Error::MerkError) ); cost_return_on_error!( &mut cost, @@ -1129,7 +1141,10 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, @@ -1273,7 +1288,10 @@ impl GroveDb { while let Some((key, element_value)) = element_iterator.next_kv().unwrap() { let element = raw_decode(&element_value, grove_version)?; match element { - Element::SumTree(..) | Element::Tree(..) | Element::BigSumTree(..) | Element::CountTree(..) => { + Element::SumTree(..) + | Element::Tree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => { let (kv_value, element_value_hash) = merk .get_value_and_value_hash( &key, diff --git a/grovedb/src/operations/delete/average_case.rs b/grovedb/src/operations/delete/average_case.rs index 986f2b90a..1b018dbfa 100644 --- a/grovedb/src/operations/delete/average_case.rs +++ b/grovedb/src/operations/delete/average_case.rs @@ -8,6 +8,7 @@ use grovedb_merk::{ average_case_costs::EstimatedLayerInformation, worst_case_costs::add_average_case_cost_for_is_empty_tree_except, }, + merk::TreeType, HASH_LENGTH_U32, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; @@ -65,7 +66,7 @@ impl GroveDb { except_keys_count, key_len, estimated_element_size, - is_sum_tree, + tree_type, ) = cost_return_on_error_no_add!( &cost, if height == path_len - 1 { @@ -84,7 +85,7 @@ impl GroveDb { 0, key.max_length() as u32, estimated_value_len, - layer_info.is_sum_tree, + layer_info.tree_type, )) } else { Err(Error::InvalidParameter( @@ -109,7 +110,7 @@ impl GroveDb { 1, last_key.max_length() as u32, estimated_value_len, - layer_info.is_sum_tree, + layer_info.tree_type, )) } else { Err(Error::InvalidParameter("intermediate layer info missing")) @@ -121,7 +122,7 @@ impl GroveDb { Self::average_case_delete_operation_for_delete::( &KeyInfoPath::from_vec(path_at_level.to_vec()), key_at_level, - is_sum_tree, + tree_type, validate, check_if_tree, except_keys_count, @@ -139,7 +140,7 @@ impl GroveDb { pub fn average_case_delete_operation_for_delete<'db, S: Storage<'db>>( path: &KeyInfoPath, key: &KeyInfo, - parent_tree_is_sum_tree: bool, + in_parent_tree_type: TreeType, validate: bool, check_if_tree: bool, except_keys_count: u16, @@ -163,7 +164,7 @@ impl GroveDb { &mut cost, path, false, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); @@ -176,7 +177,7 @@ impl GroveDb { path, key, estimated_key_element_size.1, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); diff --git a/grovedb/src/operations/delete/mod.rs b/grovedb/src/operations/delete/mod.rs index ec8ef98fb..01d2252f0 100644 --- a/grovedb/src/operations/delete/mod.rs +++ b/grovedb/src/operations/delete/mod.rs @@ -18,10 +18,9 @@ use grovedb_costs::{ storage_cost::removal::{StorageRemovedBytes, StorageRemovedBytes::BasicStorageRemoval}, CostResult, CostsExt, OperationCost, }; -use grovedb_merk::{proofs::Query, KVIterator}; +use grovedb_merk::{merk::TreeType, proofs::Query, KVIterator}; #[cfg(feature = "full")] use grovedb_merk::{Error as MerkError, Merk, MerkOptions}; -use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_storage::{ @@ -592,7 +591,7 @@ impl GroveDb { // If there is any current batch operation that is inserting something in this // tree then it is not empty either is_empty &= !current_batch_operations.iter().any(|op| match op.op { - GroveOp::Delete | GroveOp::DeleteTree(_) => false, + GroveOp::Delete | GroveOp::DeleteTree(_) => false, // todo: fix for to_path (it clones) _ => op.path.to_path() == subtree_merk_path_vec, }); diff --git a/grovedb/src/operations/delete/worst_case.rs b/grovedb/src/operations/delete/worst_case.rs index 8533cde55..09f9da76c 100644 --- a/grovedb/src/operations/delete/worst_case.rs +++ b/grovedb/src/operations/delete/worst_case.rs @@ -4,7 +4,8 @@ use grovedb_costs::{ cost_return_on_error, cost_return_on_error_no_add, CostResult, CostsExt, OperationCost, }; use grovedb_merk::{ - estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, tree::kv::KV, + estimated_costs::worst_case_costs::add_worst_case_cost_for_is_empty_tree_except, + merk::TreeType, tree::kv::KV, }; use grovedb_storage::{worst_case_costs::WorstKeyLength, Storage}; use grovedb_version::{ @@ -26,7 +27,7 @@ impl GroveDb { key: &KeyInfo, stop_path_height: Option, validate: bool, - intermediate_tree_info: IntMap<(bool, u32)>, + intermediate_tree_info: IntMap<(TreeType, u32)>, max_element_size: u32, grove_version: &GroveVersion, ) -> CostResult, Error> { @@ -59,13 +60,12 @@ impl GroveDb { check_if_tree, except_keys_count, max_element_size, - is_sum_tree, + tree_type, ) = cost_return_on_error_no_add!( &cost, if height == path_len { - if let Some((is_in_sum_tree, _)) = intermediate_tree_info.get(height as u64) - { - Ok((used_path, key, true, 0, max_element_size, *is_in_sum_tree)) + if let Some((tree_type, _)) = intermediate_tree_info.get(height as u64) { + Ok((used_path, key, true, 0, max_element_size, *tree_type)) } else { Err(Error::InvalidParameter( "intermediate flag size missing for height at path length", @@ -74,25 +74,19 @@ impl GroveDb { } else { let (last_key, smaller_path) = used_path.split_last().unwrap(); used_path = smaller_path; - if let Some((is_in_sum_tree, flags_size_at_level)) = + if let Some((tree_type, flags_size_at_level)) = intermediate_tree_info.get(height as u64) { // the worst case is that we are only in sum trees + // Todo the worst case is actually now big sum trees let value_len = SUM_TREE_COST_SIZE + flags_size_at_level; let max_tree_size = KV::layered_node_byte_cost_size_for_key_and_value_lengths( last_key.max_length() as u32, value_len, - *is_in_sum_tree, + tree_type.inner_node_type(), ); - Ok(( - used_path, - last_key, - false, - 1, - max_tree_size, - *is_in_sum_tree, - )) + Ok((used_path, last_key, false, 1, max_tree_size, *tree_type)) } else { Err(Error::InvalidParameter("intermediate flag size missing")) } @@ -103,7 +97,7 @@ impl GroveDb { Self::worst_case_delete_operation_for_delete::( &KeyInfoPath::from_vec(path_at_level.to_vec()), key_at_level, - is_sum_tree, + tree_type, validate, check_if_tree, except_keys_count, @@ -121,7 +115,7 @@ impl GroveDb { pub fn worst_case_delete_operation_for_delete<'db, S: Storage<'db>>( path: &KeyInfoPath, key: &KeyInfo, - parent_tree_is_sum_tree: bool, + in_parent_tree_type: TreeType, validate: bool, check_if_tree: bool, except_keys_count: u16, @@ -144,7 +138,7 @@ impl GroveDb { GroveDb::add_worst_case_get_merk_at_path::( &mut cost, path, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); @@ -157,7 +151,7 @@ impl GroveDb { path, key, max_element_size, - parent_tree_is_sum_tree, + in_parent_tree_type, grove_version, ) ); diff --git a/grovedb/src/operations/get/average_case.rs b/grovedb/src/operations/get/average_case.rs index aca4426df..e05e63b1d 100644 --- a/grovedb/src/operations/get/average_case.rs +++ b/grovedb/src/operations/get/average_case.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -21,7 +22,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -38,7 +39,7 @@ impl GroveDb { path, key, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -50,8 +51,8 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -68,8 +69,8 @@ impl GroveDb { path, key, estimated_flags_size, - is_sum_tree, - in_parent_tree_using_sums, + tree_type, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -81,7 +82,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -98,7 +99,7 @@ impl GroveDb { path, key, estimated_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -108,7 +109,7 @@ impl GroveDb { pub fn average_case_for_get( path: &KeyInfoPath, key: &KeyInfo, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, estimated_element_size: u32, estimated_references_sizes: Vec, grove_version: &GroveVersion, @@ -126,7 +127,7 @@ impl GroveDb { &mut cost, path, key, - in_parent_tree_using_sums, + in_parent_tree_type, estimated_element_size, estimated_references_sizes, grove_version, @@ -139,8 +140,8 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, estimated_flags_size: u32, - is_sum_tree: bool, - in_parent_tree_using_sums: bool, + tree_type: TreeType, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -157,8 +158,8 @@ impl GroveDb { path, key, estimated_flags_size, - is_sum_tree, - in_parent_tree_using_sums, + tree_type, + in_parent_tree_type, grove_version, )?; Ok(cost) diff --git a/grovedb/src/operations/get/query.rs b/grovedb/src/operations/get/query.rs index 778942e05..14ed383b6 100644 --- a/grovedb/src/operations/get/query.rs +++ b/grovedb/src/operations/get/query.rs @@ -14,7 +14,8 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use crate::element::SumValue; use crate::{ - element::QueryOptions, operations::proof::ProveOptions, + element::{BigSumValue, CountValue, QueryOptions}, + operations::proof::ProveOptions, query_result_type::PathKeyOptionalElementTrio, }; #[cfg(feature = "full")] @@ -23,7 +24,6 @@ use crate::{ reference_path::ReferencePathType, Element, Error, GroveDb, PathQuery, TransactionArg, }; -use crate::element::{BigSumValue, CountValue}; #[cfg(feature = "full")] #[derive(Debug, Eq, PartialEq, Clone)] @@ -227,7 +227,11 @@ where { )), } } - Element::Item(..) | Element::SumItem(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Ok(element), + Element::Item(..) + | Element::SumItem(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Ok(element), Element::Tree(..) => Err(Error::InvalidQuery("path_queries can not refer to trees")), } } @@ -346,7 +350,10 @@ where { } Element::Item(item, _) => Ok(item), Element::SumItem(item, _) => Ok(item.encode_var_vec()), - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) => Err(Error::InvalidQuery( + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) => Err(Error::InvalidQuery( "path_queries can only refer to items and references", )), } @@ -537,12 +544,14 @@ where { } } Element::SumItem(item, _) => Ok(item), - Element::Tree(..) | Element::SumTree(..) | Element::BigSumTree(..) | Element::CountTree(..) | Element::Item(..) => { - Err(Error::InvalidQuery( - "path_queries over sum items can only refer to sum items and \ - references", - )) - } + Element::Tree(..) + | Element::SumTree(..) + | Element::BigSumTree(..) + | Element::CountTree(..) + | Element::Item(..) => Err(Error::InvalidQuery( + "path_queries over sum items can only refer to sum items and \ + references", + )), } } _ => Err(Error::CorruptedCodeExecution( diff --git a/grovedb/src/operations/get/worst_case.rs b/grovedb/src/operations/get/worst_case.rs index 7554a9111..663e74bbf 100644 --- a/grovedb/src/operations/get/worst_case.rs +++ b/grovedb/src/operations/get/worst_case.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use grovedb_costs::OperationCost; +use grovedb_merk::merk::TreeType; #[cfg(feature = "full")] use grovedb_storage::rocksdb_storage::RocksDbStorage; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -20,7 +21,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -37,7 +38,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -48,7 +49,7 @@ impl GroveDb { path: &KeyInfoPath, key: &KeyInfo, max_element_size: u32, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -65,7 +66,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, grove_version, )?; Ok(cost) @@ -77,7 +78,7 @@ impl GroveDb { key: &KeyInfo, max_element_size: u32, max_references_sizes: Vec, - in_parent_tree_using_sums: bool, + in_parent_tree_type: TreeType, grove_version: &GroveVersion, ) -> Result { check_grovedb_v0!( @@ -94,7 +95,7 @@ impl GroveDb { path, key, max_element_size, - in_parent_tree_using_sums, + in_parent_tree_type, max_references_sizes, grove_version, )?; diff --git a/grovedb/src/operations/is_empty_tree.rs b/grovedb/src/operations/is_empty_tree.rs index 9548286e4..37a56d643 100644 --- a/grovedb/src/operations/is_empty_tree.rs +++ b/grovedb/src/operations/is_empty_tree.rs @@ -2,11 +2,12 @@ #[cfg(feature = "full")] use grovedb_costs::{cost_return_on_error, CostResult, CostsExt, OperationCost}; +use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; #[cfg(feature = "full")] use grovedb_version::error::GroveVersionError; use grovedb_version::{check_grovedb_v0_with_cost, version::GroveVersion}; -use grovedb_merk::merk::TreeType; + #[cfg(feature = "full")] use crate::{util::merk_optional_tx, Element, Error, GroveDb, TransactionArg}; diff --git a/grovedb/src/operations/proof/verify.rs b/grovedb/src/operations/proof/verify.rs index 7ff21910a..0130bac62 100644 --- a/grovedb/src/operations/proof/verify.rs +++ b/grovedb/src/operations/proof/verify.rs @@ -308,7 +308,10 @@ impl GroveDb { println!("lower layer had key {}", hex_to_ascii(key)); } match element { - Element::Tree(Some(_), _) | Element::SumTree(Some(_), ..) | Element::BigSumTree(Some(_), ..) | Element::CountTree(Some(_), ..) => { + Element::Tree(Some(_), _) + | Element::SumTree(Some(_), ..) + | Element::BigSumTree(Some(_), ..) + | Element::CountTree(Some(_), ..) => { path.push(key); let lower_hash = Self::verify_layer_proof( lower_layer, diff --git a/grovedb/src/replication.rs b/grovedb/src/replication.rs index 9bb559c56..5f167719c 100644 --- a/grovedb/src/replication.rs +++ b/grovedb/src/replication.rs @@ -2,8 +2,7 @@ mod state_sync_session; use std::pin::Pin; -use grovedb_merk::{tree::hash::CryptoHash, ChunkProducer}; -use grovedb_merk::merk::TreeType; +use grovedb_merk::{merk::TreeType, tree::hash::CryptoHash, ChunkProducer}; use grovedb_path::SubtreePath; use grovedb_version::{check_grovedb_v0, error::GroveVersionError, version::GroveVersion}; @@ -257,9 +256,10 @@ impl GroveDb { pub(crate) mod utils { use grovedb_merk::{ ed::Encode, + merk::TreeType, proofs::{Decoder, Op}, }; - use grovedb_merk::merk::TreeType; + use crate::{replication::ChunkIdentifier, Error}; /// Converts a path, represented as a slice of byte vectors (`&[Vec]`), diff --git a/grovedb/src/replication/state_sync_session.rs b/grovedb/src/replication/state_sync_session.rs index 3c48506dc..79defcfde 100644 --- a/grovedb/src/replication/state_sync_session.rs +++ b/grovedb/src/replication/state_sync_session.rs @@ -6,10 +6,10 @@ use std::{ }; use grovedb_merk::{ + merk::TreeType, tree::{kv::ValueDefinedCostType, value_hash}, CryptoHash, Restorer, }; -use grovedb_merk::merk::TreeType; use grovedb_path::SubtreePath; use grovedb_storage::{ rocksdb_storage::{PrefixedRocksDbImmediateStorageContext, RocksDbStorage}, diff --git a/grovedb/src/tests/sum_tree_tests.rs b/grovedb/src/tests/sum_tree_tests.rs index d72c26f76..1371e27e5 100644 --- a/grovedb/src/tests/sum_tree_tests.rs +++ b/grovedb/src/tests/sum_tree_tests.rs @@ -2,10 +2,9 @@ use grovedb_merk::{ proofs::Query, - tree::kv::ValueDefinedCostType, + tree::{kv::ValueDefinedCostType, AggregateData}, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; -use grovedb_merk::tree::AggregateData; use grovedb_storage::StorageBatch; use grovedb_version::version::GroveVersion; @@ -319,7 +318,10 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(SummedMerkNode(0)) )); - assert_eq!(merk.aggregate_data().expect("expected to get sum").as_i64(), 40); + assert_eq!( + merk.aggregate_data().expect("expected to get sum").as_i64(), + 40 + ); // Perform the same test on regular trees let db = make_test_grovedb(grove_version); @@ -384,7 +386,10 @@ fn test_homogenous_node_type_in_sum_trees_and_regular_trees() { .expect("node should exist"), Some(BasicMerkNode) )); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); } #[test] @@ -414,7 +419,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::NoAggregateData); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::NoAggregateData + ); // Add sum tree db.insert( @@ -453,7 +461,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(30)); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(30) + ); // Add more sum items db.insert( @@ -484,7 +495,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); // 30 - 10 + 50 = 70 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); // 30 - 10 + 50 = 70 // Add non sum items, result should remain the same db.insert( @@ -505,7 +519,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(70)); + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(70) + ); // Update existing sum items db.insert( @@ -536,7 +553,10 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(-60)); // 30 + 10 - 100 = -60 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(-60) + ); // 30 + 10 - 100 = -60 // We can not replace a normal item with a sum item, so let's delete it first db.delete( @@ -567,10 +587,13 @@ fn test_sum_tree_feature() { ) .unwrap() .expect("should open tree"); - assert_eq!(merk.aggregate_data().expect("expected to get sum"), AggregateData::Sum(9999940)); // 30 + - // 10 - - // 100 + - // 10000000 + assert_eq!( + merk.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(9999940) + ); // 30 + + // 10 - + // 100 + + // 10000000 // TODO: Test out overflows } @@ -866,7 +889,10 @@ fn test_sum_tree_with_batches() { .expect("node should exist"), Some(SummedMerkNode(10)) )); - assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(20)); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(20) + ); // Test propagation // Add a new sum tree with its own sum items, should affect sum of original @@ -941,5 +967,8 @@ fn test_sum_tree_with_batches() { ) .unwrap() .expect("should open tree"); - assert_eq!(sum_tree.aggregate_data().expect("expected to get sum"), AggregateData::Sum(41)); + assert_eq!( + sum_tree.aggregate_data().expect("expected to get sum"), + AggregateData::Sum(41) + ); } diff --git a/grovedb/src/util.rs b/grovedb/src/util.rs index 63d214e4f..20ec46d8a 100644 --- a/grovedb/src/util.rs +++ b/grovedb/src/util.rs @@ -54,7 +54,8 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -84,7 +85,8 @@ macro_rules! storage_context_with_parent_optional_tx { ) }) ); - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -140,7 +142,9 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) + = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), @@ -177,7 +181,9 @@ macro_rules! storage_context_with_parent_optional_tx_internal_error { }).unwrap_add_cost(&mut $cost); match result { Ok(element) => { - let Some(($root_key, $tree_type)) = element.root_key_and_tree_type_owned() else { + let Some(($root_key, $tree_type)) + = element.root_key_and_tree_type_owned() else + { return Err(Error::CorruptedData( "parent is not a tree" .to_owned(), diff --git a/merk/src/estimated_costs/average_case_costs.rs b/merk/src/estimated_costs/average_case_costs.rs index 89f7b94b9..f5a8fdd37 100644 --- a/merk/src/estimated_costs/average_case_costs.rs +++ b/merk/src/estimated_costs/average_case_costs.rs @@ -2,9 +2,14 @@ #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use grovedb_version::{ + error::GroveVersionError, + version::{FeatureVersion, GroveVersion}, +}; #[cfg(feature = "full")] use integer_encoding::VarInt; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] use crate::{ error::Error, @@ -12,7 +17,6 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, HASH_LENGTH_U32, }; -use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] /// Average key size @@ -395,9 +399,12 @@ pub fn add_average_case_merk_root_hash(cost: &mut OperationCost) { #[cfg(feature = "full")] /// Average case cost of propagating a merk -pub fn average_case_merk_propagate(input: &EstimatedLayerInformation) -> CostResult<(), Error> { +pub fn average_case_merk_propagate( + input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> CostResult<(), Error> { let mut cost = OperationCost::default(); - add_average_case_merk_propagate(&mut cost, input).wrap_with_cost(cost) + add_average_case_merk_propagate(&mut cost, input, grove_version).wrap_with_cost(cost) } #[cfg(feature = "full")] @@ -405,6 +412,297 @@ pub fn average_case_merk_propagate(input: &EstimatedLayerInformation) -> CostRes pub fn add_average_case_merk_propagate( cost: &mut OperationCost, input: &EstimatedLayerInformation, + grove_version: &GroveVersion, +) -> Result<(), Error> { + match grove_version + .merk_versions + .average_case_costs + .add_average_case_merk_propagate + { + 0 => add_average_case_merk_propagate_v0(cost, input), + 1 => add_average_case_merk_propagate_v1(cost, input), + version => Err(Error::VersionError( + GroveVersionError::UnknownVersionMismatch { + method: "add_average_case_merk_propagate".to_string(), + known_versions: vec![0, 1], + received: version, + }, + )), + } +} +#[cfg(feature = "full")] +/// Add average case cost for propagating a merk +fn add_average_case_merk_propagate_v1( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, +) -> Result<(), Error> { + let mut nodes_updated = 0; + // Propagation requires to recompute and write hashes up to the root + let EstimatedLayerInformation { + tree_type, + estimated_layer_count, + estimated_layer_sizes, + } = input; + let levels = estimated_layer_count.estimate_levels(); + nodes_updated += levels; + + if levels > 1 { + // we can get about 1 rotation, if there are more than 2 levels + nodes_updated += 1; + } + cost.seek_count += nodes_updated as u32; + + cost.hash_node_calls += nodes_updated * 2; + + cost.storage_cost.replaced_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + // it is normal to have LAYER_COST_SIZE here, as we add estimated sum tree + // additions right after + let value_len = LAYER_COST_SIZE + + average_flags_size + .map_or(0, |flags_len| flags_len + flags_len.required_space() as u32); + // in order to simplify calculations we get the estimated size and remove the + // cost for the basic merk + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + nodes_updated + * (KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = match subtrees_size { + None => 0, + Some((average_key_size, estimated_sum_trees, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let cost = KV::layered_value_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ) + sum_tree_addition; + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates"))? + } + }; + let item_node_updates_cost = match items_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + let reference_node_updates_cost = match references_size { + None => 0, + Some((average_key_size, average_value_size, average_flags_size, weight)) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::value_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates"))? + } + }; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_replaced_bytes = total_updates_cost / weighted_nodes_updated; + if total_replaced_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_replaced_bytes as u32 + } + } + }; + cost.storage_loaded_bytes += match estimated_layer_sizes { + EstimatedLayerSizes::AllSubtrees( + average_key_size, + estimated_sum_trees, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + nodes_updated + * KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::AllItems(average_key_size, average_item_size, average_flags_size) + | EstimatedLayerSizes::AllReference( + average_key_size, + average_item_size, + average_flags_size, + ) => { + let flags_len = average_flags_size.unwrap_or(0); + let average_value_len = average_item_size + flags_len; + nodes_updated + * KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + average_value_len, + tree_type.inner_node_type(), + ) + } + EstimatedLayerSizes::Mix { + subtrees_size, + items_size, + references_size, + } => { + let total_weight = subtrees_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + items_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default() + + references_size + .as_ref() + .map(|(_, _, _, weight)| *weight as u32) + .unwrap_or_default(); + if total_weight == 0 { + 0 + } else { + let weighted_nodes_updated = (nodes_updated as u64) + .checked_mul(total_weight as u64) + .ok_or(Error::Overflow("overflow for weights average cost"))?; + let tree_node_updates_cost = subtrees_size + .as_ref() + .map( + |(average_key_size, estimated_sum_trees, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = LAYER_COST_SIZE + flags_len; + let sum_tree_addition = estimated_sum_trees.estimated_size()?; + let cost = KV::layered_node_byte_cost_size_for_key_and_value_lengths( + *average_key_size as u32, + value_len + sum_tree_addition, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed tree nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let item_node_updates_cost = items_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + tree_type.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + let reference_node_updates_cost = references_size + .as_ref() + .map( + |(average_key_size, average_value_size, average_flags_size, weight)| { + let flags_len = average_flags_size.unwrap_or(0); + let value_len = average_value_size + flags_len; + let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( + *average_key_size as u32, + value_len, + TreeType::NormalTree.inner_node_type(), + ); + (*weight as u64) + .checked_mul(cost as u64) + .ok_or(Error::Overflow("overflow for mixed item nodes updates")) + }, + ) + .unwrap_or(Ok(0))?; + + let total_updates_cost = tree_node_updates_cost + .checked_add(item_node_updates_cost) + .and_then(|c| c.checked_add(reference_node_updates_cost)) + .ok_or(Error::Overflow("overflow for mixed item adding parts"))?; + let total_loaded_bytes = total_updates_cost / weighted_nodes_updated; + if total_loaded_bytes > u32::MAX as u64 { + return Err(Error::Overflow( + "overflow for total replaced bytes more than u32 max", + )); + } + total_loaded_bytes as u32 + } + } + } as u64; + Ok(()) +} + +#[cfg(feature = "full")] +/// Add average case cost for propagating a merk +fn add_average_case_merk_propagate_v0( + cost: &mut OperationCost, + input: &EstimatedLayerInformation, ) -> Result<(), Error> { let mut nodes_updated = 0; // Propagation requires to recompute and write hashes up to the root @@ -642,7 +940,7 @@ pub fn add_average_case_merk_propagate( let cost = KV::node_byte_cost_size_for_key_and_raw_value_lengths( *average_key_size as u32, value_len, - tree_type.inner_node_type(), //todo this is an error that we will need to fix however most likely references were never in sum trees + tree_type.inner_node_type(), // this was changed in v1 ); (*weight as u64) .checked_mul(cost as u64) diff --git a/merk/src/estimated_costs/mod.rs b/merk/src/estimated_costs/mod.rs index 6d0f47a67..5d90f0f63 100644 --- a/merk/src/estimated_costs/mod.rs +++ b/merk/src/estimated_costs/mod.rs @@ -5,9 +5,9 @@ use grovedb_costs::OperationCost; #[cfg(feature = "full")] use integer_encoding::VarInt; +use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] use crate::{tree::kv::KV, HASH_BLOCK_SIZE_U32, HASH_LENGTH_U32}; -use crate::merk::{NodeType, TreeType}; #[cfg(feature = "full")] pub mod average_case_costs; diff --git a/merk/src/estimated_costs/worst_case_costs.rs b/merk/src/estimated_costs/worst_case_costs.rs index 0f206700a..b02753b82 100644 --- a/merk/src/estimated_costs/worst_case_costs.rs +++ b/merk/src/estimated_costs/worst_case_costs.rs @@ -33,6 +33,7 @@ use std::cmp::Ordering; #[cfg(feature = "full")] use grovedb_costs::{CostResult, CostsExt, OperationCost}; +use crate::merk::NodeType; #[cfg(feature = "full")] use crate::{ error::Error, @@ -40,7 +41,6 @@ use crate::{ tree::{kv::KV, Link, TreeNode}, HASH_BLOCK_SIZE, HASH_BLOCK_SIZE_U32, HASH_LENGTH, }; -use crate::merk::NodeType; #[cfg(feature = "full")] #[derive(Clone, PartialEq, Eq, Debug)] diff --git a/merk/src/merk/apply.rs b/merk/src/merk/apply.rs index 795124926..e524bac6e 100644 --- a/merk/src/merk/apply.rs +++ b/merk/src/merk/apply.rs @@ -11,13 +11,13 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::NodeType, tree::{ kv::{ValueDefinedCostType, KV}, AuxMerkBatch, Walker, }, Error, Merk, MerkBatch, MerkOptions, }; -use crate::merk::NodeType; impl<'db, S> Merk where @@ -65,7 +65,7 @@ where KB: AsRef<[u8]>, KA: AsRef<[u8]>, { - let node_type : NodeType = self.tree_type.inner_node_type(); + let node_type: NodeType = self.tree_type.inner_node_type(); self.apply_with_costs_just_in_time_value_update( batch, aux, diff --git a/merk/src/merk/mod.rs b/merk/src/merk/mod.rs index 406da48b1..eadbe6966 100644 --- a/merk/src/merk/mod.rs +++ b/merk/src/merk/mod.rs @@ -59,7 +59,6 @@ use grovedb_version::version::GroveVersion; use source::MerkSource; use crate::{ - TreeFeatureType, error::Error, merk::{defaults::ROOT_KEY_KEY, options::MerkOptions}, proofs::{ @@ -71,13 +70,14 @@ use crate::{ Query, }, tree::{ - kv::ValueDefinedCostType, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, NULL_HASH, + kv::ValueDefinedCostType, AggregateData, AuxMerkBatch, CryptoHash, Op, RefWalker, TreeNode, + NULL_HASH, }, Error::{CostsError, EdError, StorageError}, Link, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, + TreeFeatureType, }; -use crate::tree::AggregateData; /// Key update types pub struct KeyUpdates { @@ -287,6 +287,7 @@ impl TreeType { TreeType::CountTree => false, } } + pub fn inner_node_type(&self) -> NodeType { match self { TreeType::NormalTree => NodeType::NormalNode, @@ -298,11 +299,11 @@ impl TreeType { pub fn empty_tree_feature_type(&self) -> TreeFeatureType { match self { - TreeType::NormalTree => TreeFeatureType::BasicMerkNode, - TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), - TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), - TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), - } + TreeType::NormalTree => TreeFeatureType::BasicMerkNode, + TreeType::SumTree => TreeFeatureType::SummedMerkNode(0), + TreeType::BigSumTree => TreeFeatureType::BigSummedMerkNode(0), + TreeType::CountTree => TreeFeatureType::CountedMerkNode(0), + } } } @@ -407,9 +408,12 @@ where } /// Returns the root hash and non-prefixed key of the tree. - pub fn root_hash_key_and_aggregate_data(&self) -> CostResult { + pub fn root_hash_key_and_aggregate_data( + &self, + ) -> CostResult { self.use_tree(|tree| match tree { - None => Ok((NULL_HASH, None, AggregateData::NoAggregateData)).wrap_with_cost(Default::default()), + None => Ok((NULL_HASH, None, AggregateData::NoAggregateData)) + .wrap_with_cost(Default::default()), Some(tree) => { let aggregate_data = cost_return_on_error_default!(tree.aggregate_data()); tree.hash() @@ -756,9 +760,12 @@ where grove_version: &GroveVersion, ) { let (hash, key, aggregate_data) = match link { - Link::Reference { hash, key, aggregate_data, .. } => { - (hash.to_owned(), key.to_owned(), aggregate_data.to_owned()) - } + Link::Reference { + hash, + key, + aggregate_data, + .. + } => (hash.to_owned(), key.to_owned(), aggregate_data.to_owned()), Link::Modified { tree, .. } => ( tree.hash().unwrap(), tree.key().to_vec(), @@ -769,7 +776,11 @@ where child_heights: _, aggregate_data, tree, - } => (hash.to_owned(), tree.key().to_vec(), aggregate_data.to_owned()), + } => ( + hash.to_owned(), + tree.key().to_vec(), + aggregate_data.to_owned(), + ), _ => todo!(), }; @@ -852,7 +863,7 @@ mod test { use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use super::{Merk, RefWalker}; + use super::{Merk, RefWalker, TreeType}; use crate::{ merk::source::MerkSource, test_utils::*, tree::kv::ValueDefinedCostType, Op, TreeFeatureType::BasicMerkNode, @@ -1083,7 +1094,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1109,7 +1120,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1165,7 +1176,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1184,7 +1195,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1205,7 +1216,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1245,7 +1256,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1266,7 +1277,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1281,7 +1292,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1305,7 +1316,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), Some(&batch)) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) @@ -1369,7 +1380,7 @@ mod test { storage .get_storage_context(SubtreePath::empty(), None) .unwrap(), - false, + TreeType::NormalTree, None::<&fn(&[u8], &GroveVersion) -> Option>, grove_version, ) diff --git a/merk/src/merk/open.rs b/merk/src/merk/open.rs index d47febcd4..fa172bf24 100644 --- a/merk/src/merk/open.rs +++ b/merk/src/merk/open.rs @@ -5,11 +5,11 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::TreeType, tree::kv::ValueDefinedCostType, Error, Merk, MerkType, MerkType::{BaseMerk, LayeredMerk, StandaloneMerk}, }; -use crate::merk::TreeType; impl<'db, S> Merk where @@ -102,8 +102,9 @@ mod test { use grovedb_version::version::GroveVersion; use tempfile::TempDir; - use crate::{tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode}; - use crate::merk::TreeType; + use crate::{ + merk::TreeType, tree::kv::ValueDefinedCostType, Merk, Op, TreeFeatureType::BasicMerkNode, + }; #[test] fn test_reopen_root_hash() { diff --git a/merk/src/merk/restore.rs b/merk/src/merk/restore.rs index cb694b418..5fb91b8be 100644 --- a/merk/src/merk/restore.rs +++ b/merk/src/merk/restore.rs @@ -36,7 +36,7 @@ use grovedb_version::version::GroveVersion; use crate::{ merk, - merk::MerkSource, + merk::{MerkSource, TreeType}, proofs::{ chunk::{ chunk::{LEFT, RIGHT}, @@ -52,7 +52,6 @@ use crate::{ Error::{CostsError, StorageError}, Link, Merk, }; -use crate::merk::TreeType; /// Restorer handles verification of chunks and replication of Merk trees. /// Chunks can be processed randomly as long as their parent has been processed @@ -318,7 +317,12 @@ impl<'db, S: StorageContext<'db>> Restorer { let updated_key = chunk_tree.key(); let updated_sum = chunk_tree.aggregate_data(); - if let Some(Link::Reference { key, aggregate_data, .. }) = parent.link_mut(*is_left) { + if let Some(Link::Reference { + key, + aggregate_data, + .. + }) = parent.link_mut(*is_left) + { *key = updated_key.to_vec(); *aggregate_data = updated_sum; } @@ -557,7 +561,7 @@ mod tests { use super::*; use crate::{ - merk::chunks::ChunkProducer, + merk::{chunks::ChunkProducer, TreeType}, proofs::chunk::{ chunk::tests::traverse_get_node_hash, error::ChunkError::InvalidChunkProof, }, @@ -565,7 +569,6 @@ mod tests { Error::ChunkRestoringError, Merk, PanicSource, }; - use crate::merk::TreeType; #[test] fn test_chunk_verification_non_avl_tree() { diff --git a/merk/src/merk/source.rs b/merk/src/merk/source.rs index 620373a86..14c707958 100644 --- a/merk/src/merk/source.rs +++ b/merk/src/merk/source.rs @@ -3,10 +3,10 @@ use grovedb_storage::StorageContext; use grovedb_version::version::GroveVersion; use crate::{ + merk::TreeType, tree::{kv::ValueDefinedCostType, Fetch, TreeNode}, Error, Link, Merk, }; -use crate::merk::TreeType; impl<'db, S> Merk where diff --git a/merk/src/proofs/tree.rs b/merk/src/proofs/tree.rs index 094fe48fa..6a1a93e53 100644 --- a/merk/src/proofs/tree.rs +++ b/merk/src/proofs/tree.rs @@ -21,8 +21,10 @@ use crate::{ Link, TreeFeatureType::SummedMerkNode, }; -use crate::tree::AggregateData; -use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; +use crate::{ + tree::AggregateData, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(any(feature = "full", feature = "verify"))] /// Contains a tree's child node and its hash. The hash can always be assumed to @@ -39,7 +41,9 @@ impl Child { #[cfg(feature = "full")] pub fn as_link(&self) -> Link { let (key, aggregate_data) = match &self.tree.node { - Node::KV(key, _) | Node::KVValueHash(key, ..) => (key.as_slice(), AggregateData::NoAggregateData), + Node::KV(key, _) | Node::KVValueHash(key, ..) => { + (key.as_slice(), AggregateData::NoAggregateData) + } Node::KVValueHashFeatureType(key, _, _, feature_type) => { (key.as_slice(), (*feature_type).into()) } @@ -634,7 +638,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), key: vec![1] } @@ -644,7 +648,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), key: vec![3] } @@ -683,7 +687,7 @@ mod test { left_link, Link::Reference { hash: tree.left.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: Some(3), + aggregate_data: AggregateData::Sum(3), child_heights: (0, 0), key: vec![1] } @@ -693,7 +697,7 @@ mod test { right_link, Link::Reference { hash: tree.right.as_ref().map(|node| node.hash).unwrap(), - aggregate_data: Some(1), + aggregate_data: AggregateData::Sum(1), child_heights: (0, 0), key: vec![3] } diff --git a/merk/src/test_utils/mod.rs b/merk/src/test_utils/mod.rs index 24ac94fad..f53ce730c 100644 --- a/merk/src/test_utils/mod.rs +++ b/merk/src/test_utils/mod.rs @@ -40,6 +40,7 @@ use rand::prelude::*; pub use temp_merk::TempMerk; use crate::{ + merk::TreeType, tree::{ kv::{ValueDefinedCostType, KV}, BatchEntry, MerkBatch, NoopCommit, Op, PanicSource, TreeNode, Walker, @@ -47,7 +48,6 @@ use crate::{ Merk, TreeFeatureType::{BasicMerkNode, SummedMerkNode}, }; -use crate::merk::TreeType; /// Assert tree invariants pub fn assert_tree_invariants(tree: &TreeNode) { diff --git a/merk/src/test_utils/temp_merk.rs b/merk/src/test_utils/temp_merk.rs index 3133ce6bb..93cff86f5 100644 --- a/merk/src/test_utils/temp_merk.rs +++ b/merk/src/test_utils/temp_merk.rs @@ -40,10 +40,9 @@ use grovedb_storage::{ }; use grovedb_version::version::GroveVersion; -use crate::tree::kv::ValueDefinedCostType; #[cfg(feature = "full")] use crate::Merk; -use crate::merk::TreeType; +use crate::{merk::TreeType, tree::kv::ValueDefinedCostType}; #[cfg(feature = "full")] /// Wraps a Merk instance and deletes it from disk it once it goes out of scope. diff --git a/merk/src/tree/encoding.rs b/merk/src/tree/encoding.rs index 31ef980e9..2f5b88c61 100644 --- a/merk/src/tree/encoding.rs +++ b/merk/src/tree/encoding.rs @@ -146,9 +146,11 @@ impl TreeNode { #[cfg(feature = "full")] #[cfg(test)] mod tests { - use crate::tree::AggregateData; use super::{super::Link, *}; - use crate::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; + use crate::{ + tree::AggregateData, + TreeFeatureType::{BasicMerkNode, SummedMerkNode}, + }; #[test] fn encode_leaf_tree() { diff --git a/merk/src/tree/kv.rs b/merk/src/tree/kv.rs index ccad4dd2a..5047fe594 100644 --- a/merk/src/tree/kv.rs +++ b/merk/src/tree/kv.rs @@ -12,7 +12,10 @@ use integer_encoding::VarInt; #[cfg(feature = "full")] use super::hash::{CryptoHash, HASH_LENGTH, NULL_HASH}; -use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; +use crate::{ + merk::NodeType, + tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}, +}; #[cfg(feature = "full")] use crate::{ tree::{ @@ -21,7 +24,6 @@ use crate::{ }, Link, HASH_LENGTH_U32, HASH_LENGTH_U32_X2, }; -use crate::merk::NodeType; // TODO: maybe use something similar to Vec but without capacity field, // (should save 16 bytes per entry). also, maybe a shorter length // field to save even more. also might be possible to combine key @@ -418,11 +420,7 @@ impl KV { let key_len = self.key.len() as u32; let node_type = self.feature_type.node_type(); - Self::layered_value_byte_cost_size_for_key_and_value_lengths( - key_len, - value_cost, - node_type, - ) + Self::layered_value_byte_cost_size_for_key_and_value_lengths(key_len, value_cost, node_type) } /// This function is used to calculate the cost of groveDB sum item nodes diff --git a/merk/src/tree/link.rs b/merk/src/tree/link.rs index 2537b15bd..4b8c9f650 100644 --- a/merk/src/tree/link.rs +++ b/merk/src/tree/link.rs @@ -2,6 +2,7 @@ #[cfg(feature = "full")] use std::io::{Read, Write}; + use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::{Decode, Encode, Result, Terminated}; @@ -10,11 +11,11 @@ use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; #[cfg(feature = "full")] use super::{hash::CryptoHash, TreeNode}; -#[cfg(feature = "full")] -use crate::HASH_LENGTH_U32; use crate::merk::NodeType; #[cfg(feature = "full")] use crate::tree::tree_feature_type::AggregateData; +#[cfg(feature = "full")] +use crate::HASH_LENGTH_U32; // TODO: optimize memory footprint #[cfg(feature = "full")] @@ -271,7 +272,11 @@ impl Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, aggregate_data, .. } => match aggregate_data { + Link::Reference { + key, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => key.len() + 36, // 1 + HASH_LENGTH + 2 + 1, AggregateData::Count(_) | AggregateData::Sum(_) => { // 1 for key len @@ -297,7 +302,16 @@ impl Link { } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + Link::Uncommitted { + tree, + aggregate_data, + .. + } + | Link::Loaded { + tree, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1, AggregateData::Count(_) | AggregateData::Sum(_) => { tree.key().len() + 44 // 1 + 32 + 2 + 1 + 8 @@ -359,7 +373,7 @@ impl Encode for Link { out.write_i128::(*big_sum_value)?; } AggregateData::Count(count_value) => { - out.write_all(&[2])?; + out.write_all(&[3])?; out.write_varint(*count_value)?; } } @@ -372,7 +386,11 @@ impl Encode for Link { debug_assert!(self.key().len() < 256, "Key length must be less than 256"); Ok(match self { - Link::Reference { key, aggregate_data, .. } => match aggregate_data { + Link::Reference { + key, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => key.len() + 36, // 1 + 32 + 2 + 1 AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); @@ -411,7 +429,16 @@ impl Encode for Link { } }, Link::Modified { .. } => panic!("No encoding for Link::Modified"), - Link::Uncommitted { tree, aggregate_data, .. } | Link::Loaded { tree, aggregate_data, .. } => match aggregate_data { + Link::Uncommitted { + tree, + aggregate_data, + .. + } + | Link::Loaded { + tree, + aggregate_data, + .. + } => match aggregate_data { AggregateData::NoAggregateData => tree.key().len() + 36, // 1 + 32 + 2 + 1 AggregateData::Sum(sum_value) => { let encoded_sum_value = sum_value.encode_var_vec(); @@ -708,7 +735,7 @@ mod test { bytes, vec![ 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 3, 50, ] ); } @@ -721,7 +748,7 @@ mod test { child_heights: (123, 124), hash: [55; 32], }; - assert_eq!(link.encoding_length().unwrap(), 40); + assert_eq!(link.encoding_length().unwrap(), 55); let mut bytes = vec![]; link.encode_into(&mut bytes).unwrap(); @@ -731,7 +758,8 @@ mod test { bytes, vec![ 3, 1, 2, 3, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, - 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 1, 100, + 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 55, 123, 124, 2, 0, 0, 0, 0, 0, + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 50 ] ); } diff --git a/merk/src/tree/mod.rs b/merk/src/tree/mod.rs index ed9d470e2..2d24825eb 100644 --- a/merk/src/tree/mod.rs +++ b/merk/src/tree/mod.rs @@ -60,12 +60,13 @@ pub use link::Link; #[cfg(feature = "full")] pub use ops::{AuxMerkBatch, BatchEntry, MerkBatch, Op, PanicSource}; #[cfg(any(feature = "full", feature = "verify"))] -pub use tree_feature_type::TreeFeatureType; -#[cfg(any(feature = "full", feature = "verify"))] pub use tree_feature_type::AggregateData; +#[cfg(any(feature = "full", feature = "verify"))] +pub use tree_feature_type::TreeFeatureType; #[cfg(feature = "full")] pub use walk::{Fetch, RefWalker, Walker}; +use crate::merk::NodeType; #[cfg(feature = "full")] use crate::tree::hash::HASH_LENGTH_X2; #[cfg(feature = "full")] @@ -74,7 +75,6 @@ use crate::tree::kv::ValueDefinedCostType; use crate::tree::kv::ValueDefinedCostType::{LayeredValueDefinedCost, SpecializedValueDefinedCost}; #[cfg(feature = "full")] use crate::{error::Error, Error::Overflow}; -use crate::merk::NodeType; // TODO: remove need for `TreeInner`, and just use `Box` receiver for // relevant methods @@ -99,7 +99,6 @@ impl TreeNodeInner { (self.kv.value, self.kv.feature_type) } - /// Get the value as slice of the key value struct pub fn value_as_slice(&self) -> &[u8] { self.kv.value.as_slice() @@ -461,7 +460,7 @@ impl TreeNode { AggregateData::Sum(s) => s.encode_var_vec().len() as u32, AggregateData::BigSum(s) => 16 as u32, AggregateData::Count(c) => c.encode_var_vec().len() as u32, - } + }, ) }) } @@ -507,7 +506,9 @@ impl TreeNode { Some(link) => match link.aggregateData() { AggregateData::NoAggregateData => Ok(0), AggregateData::Sum(s) => Ok(s), - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as i64".to_string())), + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + "for aggregate data as i64".to_string(), + )), AggregateData::Count(c) => { if c > i64::MAX as u64 { Err(Overflow("count overflow when below sum tree")) @@ -515,7 +516,7 @@ impl TreeNode { Ok(c as i64) } } - } + }, _ => Ok(0), } } @@ -533,10 +534,12 @@ impl TreeNode { } else { Ok(s as u64) } - }, - AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree("for aggregate data as u64".to_string())), - AggregateData::Count(c) => Ok(c) - } + } + AggregateData::BigSum(s) => Err(Error::BigSumTreeUnderNormalSumTree( + "for aggregate data as u64".to_string(), + )), + AggregateData::Count(c) => Ok(c), + }, _ => Ok(0), } } @@ -551,7 +554,7 @@ impl TreeNode { AggregateData::Sum(s) => s as i128, AggregateData::BigSum(s) => s, AggregateData::Count(c) => c as i128, - } + }, _ => 0, } } @@ -580,7 +583,7 @@ impl TreeNode { .and_then(|a| a.checked_add(right)) .ok_or(Overflow("sum is overflowing")) .map(AggregateData::Sum) - }, + } TreeFeatureType::BigSummedMerkNode(value) => value .checked_add(self.child_aggregate_data_as_i128(true)) .and_then(|a| a.checked_add(self.child_aggregate_data_as_i128(false))) @@ -594,7 +597,7 @@ impl TreeNode { .and_then(|a| a.checked_add(right)) .ok_or(Overflow("count is overflowing")) .map(AggregateData::Count) - }, + } } } @@ -1118,7 +1121,7 @@ pub const fn side_to_str(left: bool) -> &'static str { #[cfg(test)] mod test { - use super::{commit::NoopCommit, hash::NULL_HASH, TreeNode}; + use super::{commit::NoopCommit, hash::NULL_HASH, AggregateData, TreeNode}; use crate::tree::{ tree_feature_type::TreeFeatureType::SummedMerkNode, TreeFeatureType::BasicMerkNode, }; @@ -1327,6 +1330,10 @@ mod test { .unwrap() .expect("commit failed"); - assert_eq!(Some(8), tree.sum().expect("expected to get sum from tree")); + assert_eq!( + AggregateData::Sum(8), + tree.aggregate_data() + .expect("expected to get sum from tree") + ); } } diff --git a/merk/src/tree/ops.rs b/merk/src/tree/ops.rs index 3e10b2c81..690e136fe 100644 --- a/merk/src/tree/ops.rs +++ b/merk/src/tree/ops.rs @@ -1100,7 +1100,7 @@ mod test { None, Some(Link::Loaded { hash: [123; 32], - sum: None, + aggregate_data: AggregateData::NoAggregateData, child_heights: (0, 0), tree: TreeNode::new(b"foo2".to_vec(), b"bar2".to_vec(), None, BasicMerkNode) .unwrap(), diff --git a/merk/src/tree/tree_feature_type.rs b/merk/src/tree/tree_feature_type.rs index 7dd6472e2..1ecf967e8 100644 --- a/merk/src/tree/tree_feature_type.rs +++ b/merk/src/tree/tree_feature_type.rs @@ -2,6 +2,7 @@ #[cfg(any(feature = "full", feature = "verify"))] use std::io::{Read, Write}; + use byteorder::{BigEndian, ReadBytesExt, WriteBytesExt}; #[cfg(feature = "full")] use ed::Terminated; @@ -9,10 +10,13 @@ use ed::Terminated; use ed::{Decode, Encode}; #[cfg(any(feature = "full", feature = "verify"))] use integer_encoding::{VarInt, VarIntReader, VarIntWriter}; -use crate::merk::NodeType; + #[cfg(any(feature = "full", feature = "verify"))] use crate::tree::tree_feature_type::TreeFeatureType::{BasicMerkNode, SummedMerkNode}; -use crate::TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}; +use crate::{ + merk::{NodeType, TreeType}, + TreeFeatureType::{BigSummedMerkNode, CountedMerkNode}, +}; #[cfg(any(feature = "full", feature = "verify"))] #[derive(Copy, Clone, PartialEq, Eq, Debug)] @@ -49,28 +53,37 @@ pub enum AggregateData { } impl AggregateData { + pub fn parent_tree_type(&self) -> TreeType { + match self { + AggregateData::NoAggregateData => TreeType::NormalTree, + AggregateData::Sum(_) => TreeType::SumTree, + AggregateData::BigSum(_) => TreeType::BigSumTree, + AggregateData::Count(_) => TreeType::CountTree, + } + } + pub fn as_i64(&self) -> i64 { - match self { - AggregateData::NoAggregateData => 0, - AggregateData::Sum(s) => *s, - AggregateData::BigSum(i) => { - let max = i64::MAX as i128; - if *i > max { - i64::MAX - } else { - *i as i64 + match self { + AggregateData::NoAggregateData => 0, + AggregateData::Sum(s) => *s, + AggregateData::BigSum(i) => { + let max = i64::MAX as i128; + if *i > max { + i64::MAX + } else { + *i as i64 + } } - } - AggregateData::Count(c) => { - let max = i64::MAX as u64; - if *c > max { - i64::MAX - } else { - *c as i64 + AggregateData::Count(c) => { + let max = i64::MAX as u64; + if *c > max { + i64::MAX + } else { + *c as i64 + } } } } -} pub fn as_u64(&self) -> u64 { match self { @@ -81,7 +94,7 @@ impl AggregateData { } else { *s as u64 } - }, + } AggregateData::BigSum(i) => { let max = u64::MAX as i128; if *i > max { @@ -92,24 +105,16 @@ impl AggregateData { *i as u64 } } - AggregateData::Count(c) => { - *c - } + AggregateData::Count(c) => *c, } } pub fn as_i128(&self) -> i128 { match self { AggregateData::NoAggregateData => 0, - AggregateData::Sum(s) => { - *s as i128 - }, - AggregateData::BigSum(i) => { - *i - } - AggregateData::Count(c) => { - *c as i128 - } + AggregateData::Sum(s) => *s as i128, + AggregateData::BigSum(i) => *i, + AggregateData::Count(c) => *c as i128, } } } @@ -189,9 +194,7 @@ impl Encode for TreeFeatureType { // encoded_sum.len() for the length of the encoded vector Ok(1 + encoded_sum.len()) } - BigSummedMerkNode(_) => { - Ok(17) - } + BigSummedMerkNode(_) => Ok(17), CountedMerkNode(count) => { let encoded_sum = count.encode_var_vec(); // 1 for the enum type diff --git a/merk/src/tree/walk/mod.rs b/merk/src/tree/walk/mod.rs index 2274f804f..f0a0aeff4 100644 --- a/merk/src/tree/walk/mod.rs +++ b/merk/src/tree/walk/mod.rs @@ -403,7 +403,7 @@ mod test { use grovedb_version::version::GroveVersion; use super::{super::NoopCommit, *}; - use crate::tree::{TreeFeatureType::BasicMerkNode, TreeNode}; + use crate::tree::{AggregateData, TreeFeatureType::BasicMerkNode, TreeNode}; #[derive(Clone)] struct MockSource {} @@ -491,7 +491,7 @@ mod test { hash: Default::default(), key: b"foo".to_vec(), child_heights: (0, 0), - aggregate_data: None, + aggregate_data: AggregateData::NoAggregateData, }), None, BasicMerkNode, diff --git a/node-grove/src/converter.rs b/node-grove/src/converter.rs index a822faadc..2aca7b0d9 100644 --- a/node-grove/src/converter.rs +++ b/node-grove/src/converter.rs @@ -38,6 +38,8 @@ fn element_to_string(element: Element) -> String { Element::Reference(..) => "reference".to_string(), Element::Tree(..) => "tree".to_string(), Element::SumTree(..) => "sum_tree".to_string(), + Element::BigSumTree(..) => "big_sum_tree".to_string(), + Element::CountTree(..) => "big_sum_tree".to_string(), } } @@ -92,6 +94,8 @@ pub fn element_to_js_object<'a, C: Context<'a>>( Element::Reference(..) => nested_vecs_to_js(vec![], cx)?, Element::Tree(..) => nested_vecs_to_js(vec![], cx)?, Element::SumTree(..) => nested_vecs_to_js(vec![], cx)?, + Element::BigSumTree(..) => nested_vecs_to_js(vec![], cx)?, + Element::CountTree(..) => nested_vecs_to_js(vec![], cx)?, }; js_object.set(cx, "value", js_value)?;