feat: big sum trees and count trees

costs/src/lib.rs

@@ -72,9 +72,29 @@ pub type ChildrenSizesWithValue = Option<(
     Option<(ChildKeyLength, ChildSumLength)>,
 )>;
 
+/// The tree cost type
+pub enum TreeCostType {
+    /// This is for sum trees and count trees
+    TreeFeatureUsesVarIntCostAs8Bytes,
+    /// This is for count sum trees
+    TreeFeatureUsesTwoVarIntsCostAs16Bytes,
+    /// This is for big sum trees
+    TreeFeatureUses16Bytes,
+}
+
+impl TreeCostType {
+    fn cost_size(&self) -> u32 {
+        match self {
+            TreeCostType::TreeFeatureUsesVarIntCostAs8Bytes => 8,
+            TreeCostType::TreeFeatureUsesTwoVarIntsCostAs16Bytes => 16,
+            TreeCostType::TreeFeatureUses16Bytes => 16,
+        }
+    }
+}
+
 /// Children sizes starting with if we are in a sum tree
 pub type ChildrenSizesWithIsSumTree = Option<(
-    Option<FeatureSumLength>,
+    Option<(TreeCostType, FeatureSumLength)>,
     Option<(ChildKeyLength, ChildSumLength)>,
     Option<(ChildKeyLength, ChildSumLength)>,
 )>;
@@ -199,20 +219,21 @@ impl OperationCost {
                     paid_value_len -= right_child_sum_len;
                 }
 
-                if let Some(sum_tree_len) = in_sum_tree {
+                let sum_tree_node_size = if let Some((tree_cost_type, sum_tree_len)) = in_sum_tree {
+                    let cost_size = tree_cost_type.cost_size();
                     paid_value_len -= sum_tree_len;
-                    paid_value_len += 8;
-                }
+                    paid_value_len += cost_size;
+                    cost_size
+                } else {
+                    0
+                };
 
                 // This is the moment we need to add the required space (after removing
                 // children) but before adding the parent to child hook
                 paid_value_len += paid_value_len.required_space() as u32;
 
                 // Now we are the parent to child hook
 
-                // we need to add the sum tree node size
-                let sum_tree_node_size = if in_sum_tree.is_some() { 8 } else { 0 };
-
                 // We need to add the cost of a parent
                 // key_len has a hash length already in it from the key prefix
                 // So we need to remove it and then add a hash length

grovedb-version/src/lib.rs

@@ -34,6 +34,23 @@ macro_rules! check_grovedb_v0 {
     }};
 }
 
+#[macro_export]
+macro_rules! check_grovedb_v0_or_v1 {
+    ($method:expr, $version:expr) => {{
+        const EXPECTED_VERSION_V0: u16 = 0;
+        const EXPECTED_VERSION_V1: u16 = 1;
+        if $version != EXPECTED_VERSION_V0 && $version != EXPECTED_VERSION_V1 {
+            return Err(GroveVersionError::UnknownVersionMismatch {
+                method: $method.to_string(),
+                known_versions: vec![EXPECTED_VERSION_V0, EXPECTED_VERSION_V1],
+                received: $version,
+            }
+            .into());
+        }
+        $version
+    }};
+}
+
 #[macro_export]
 macro_rules! check_merk_v0_with_cost {
     ($method:expr, $version:expr) => {{

grovedb-version/src/version/merk_versions.rs

@@ -1,2 +1,12 @@
+use versioned_feature_core::FeatureVersion;
+
 #[derive(Clone, Debug, Default)]
-pub struct MerkVersions {}
+pub struct MerkVersions {
+    pub average_case_costs: MerkAverageCaseCostsVersions,
+}
+
+#[derive(Clone, Debug, Default)]
+pub struct MerkAverageCaseCostsVersions {
+    pub add_average_case_merk_propagate: FeatureVersion,
+    pub sum_tree_estimated_size: FeatureVersion,
+}

grovedb-version/src/version/mod.rs

@@ -1,11 +1,12 @@
 pub mod grovedb_versions;
 pub mod merk_versions;
 pub mod v1;
+pub mod v2;
 
 pub use versioned_feature_core::*;
 
 use crate::version::{
-    grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1,
+    grovedb_versions::GroveDBVersions, merk_versions::MerkVersions, v1::GROVE_V1, v2::GROVE_V2,
 };
 
 #[derive(Clone, Debug, Default)]
@@ -16,11 +17,17 @@ pub struct GroveVersion {
 }
 
 impl GroveVersion {
+    pub fn first<'a>() -> &'a Self {
+        GROVE_VERSIONS
+            .first()
+            .expect("expected to have a platform version")
+    }
+
     pub fn latest<'a>() -> &'a Self {
         GROVE_VERSIONS
             .last()
             .expect("expected to have a platform version")
     }
 }
 
-pub const GROVE_VERSIONS: &[GroveVersion] = &[GROVE_V1];
+pub const GROVE_VERSIONS: &[GroveVersion] = &[GROVE_V1, GROVE_V2];

grovedb-version/src/version/v1.rs

@@ -8,7 +8,7 @@ use crate::version::{
         GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions,
         GroveDBReplicationVersions, GroveDBVersions,
     },
-    merk_versions::MerkVersions,
+    merk_versions::{MerkAverageCaseCostsVersions, MerkVersions},
     GroveVersion,
 };
 
@@ -184,5 +184,10 @@ pub const GROVE_V1: GroveVersion = GroveVersion {
             apply_chunk: 0,
         },
     },
-    merk_versions: MerkVersions {},
+    merk_versions: MerkVersions {
+        average_case_costs: MerkAverageCaseCostsVersions {
+            add_average_case_merk_propagate: 0,
+            sum_tree_estimated_size: 0,
+        },
+    },
 };

grovedb-version/src/version/v2.rs

@@ -0,0 +1,193 @@
+use crate::version::{
+    grovedb_versions::{
+        GroveDBApplyBatchVersions, GroveDBElementMethodVersions,
+        GroveDBOperationsAverageCaseVersions, GroveDBOperationsDeleteUpTreeVersions,
+        GroveDBOperationsDeleteVersions, GroveDBOperationsGetVersions,
+        GroveDBOperationsInsertVersions, GroveDBOperationsProofVersions,
+        GroveDBOperationsQueryVersions, GroveDBOperationsVersions,
+        GroveDBOperationsWorstCaseVersions, GroveDBPathQueryMethodVersions,
+        GroveDBReplicationVersions, GroveDBVersions,
+    },
+    merk_versions::{MerkAverageCaseCostsVersions, MerkVersions},
+    GroveVersion,
+};
+
+pub const GROVE_V2: GroveVersion = GroveVersion {
+    protocol_version: 1,
+    grovedb_versions: GroveDBVersions {
+        apply_batch: GroveDBApplyBatchVersions {
+            apply_batch_structure: 0,
+            apply_body: 0,
+            continue_partial_apply_body: 0,
+            apply_operations_without_batching: 0,
+            apply_batch: 0,
+            apply_partial_batch: 0,
+            open_batch_transactional_merk_at_path: 0,
+            open_batch_merk_at_path: 0,
+            apply_batch_with_element_flags_update: 0,
+            apply_partial_batch_with_element_flags_update: 0,
+            estimated_case_operations_for_batch: 0,
+        },
+        element: GroveDBElementMethodVersions {
+            delete: 0,
+            delete_with_sectioned_removal_bytes: 0,
+            delete_into_batch_operations: 0,
+            element_at_key_already_exists: 0,
+            get: 0,
+            get_optional: 0,
+            get_from_storage: 0,
+            get_optional_from_storage: 1,
+            get_with_absolute_refs: 0,
+            get_value_hash: 0,
+            get_specialized_cost: 0,
+            value_defined_cost: 0,
+            value_defined_cost_for_serialized_value: 0,
+            specialized_costs_for_key_value: 0,
+            required_item_space: 0,
+            insert: 0,
+            insert_into_batch_operations: 0,
+            insert_if_not_exists: 0,
+            insert_if_not_exists_into_batch_operations: 0,
+            insert_if_changed_value: 0,
+            insert_if_changed_value_into_batch_operations: 0,
+            insert_reference: 0,
+            insert_reference_into_batch_operations: 0,
+            insert_subtree: 0,
+            insert_subtree_into_batch_operations: 0,
+            get_query: 0,
+            get_query_values: 0,
+            get_query_apply_function: 0,
+            get_path_query: 0,
+            get_sized_query: 0,
+            path_query_push: 0,
+            query_item: 0,
+            basic_push: 0,
+            serialize: 0,
+            serialized_size: 0,
+            deserialize: 0,
+        },
+        operations: GroveDBOperationsVersions {
+            get: GroveDBOperationsGetVersions {
+                get: 0,
+                get_caching_optional: 0,
+                follow_reference: 0,
+                get_raw: 0,
+                get_raw_caching_optional: 0,
+                get_raw_optional: 0,
+                get_raw_optional_caching_optional: 0,
+                has_raw: 0,
+                check_subtree_exists_invalid_path: 0,
+                average_case_for_has_raw: 0,
+                average_case_for_has_raw_tree: 0,
+                average_case_for_get_raw: 0,
+                average_case_for_get: 0,
+                average_case_for_get_tree: 0,
+                worst_case_for_has_raw: 0,
+                worst_case_for_get_raw: 0,
+                worst_case_for_get: 0,
+                is_empty_tree: 0,
+            },
+            insert: GroveDBOperationsInsertVersions {
+                insert: 0,
+                insert_on_transaction: 0,
+                insert_without_transaction: 0,
+                add_element_on_transaction: 0,
+                add_element_without_transaction: 0,
+                insert_if_not_exists: 0,
+                insert_if_not_exists_return_existing_element: 0,
+                insert_if_changed_value: 0,
+            },
+            delete: GroveDBOperationsDeleteVersions {
+                delete: 0,
+                clear_subtree: 0,
+                delete_with_sectional_storage_function: 0,
+                delete_if_empty_tree: 0,
+                delete_if_empty_tree_with_sectional_storage_function: 0,
+                delete_operation_for_delete_internal: 0,
+                delete_internal_on_transaction: 0,
+                delete_internal_without_transaction: 0,
+                average_case_delete_operation_for_delete: 0,
+                worst_case_delete_operation_for_delete: 0,
+            },
+            delete_up_tree: GroveDBOperationsDeleteUpTreeVersions {
+                delete_up_tree_while_empty: 0,
+                delete_up_tree_while_empty_with_sectional_storage: 0,
+                delete_operations_for_delete_up_tree_while_empty: 0,
+                add_delete_operations_for_delete_up_tree_while_empty: 0,
+                average_case_delete_operations_for_delete_up_tree_while_empty: 0,
+                worst_case_delete_operations_for_delete_up_tree_while_empty: 0,
+            },
+            query: GroveDBOperationsQueryVersions {
+                query_encoded_many: 0,
+                query_many_raw: 0,
+                get_proved_path_query: 0,
+                query: 0,
+                query_item_value: 0,
+                query_item_value_or_sum: 0,
+                query_sums: 0,
+                query_raw: 0,
+                query_keys_optional: 0,
+                query_raw_keys_optional: 0,
+                follow_element: 0,
+            },
+            proof: GroveDBOperationsProofVersions {
+                prove_query: 0,
+                prove_query_many: 0,
+                verify_query_with_options: 0,
+                verify_query_raw: 0,
+                verify_layer_proof: 0,
+                verify_query: 0,
+                verify_subset_query: 0,
+                verify_query_with_absence_proof: 0,
+                verify_subset_query_with_absence_proof: 0,
+                verify_query_with_chained_path_queries: 0,
+            },
+            average_case: GroveDBOperationsAverageCaseVersions {
+                add_average_case_get_merk_at_path: 0,
+                average_case_merk_replace_tree: 1, // changed
+                average_case_merk_insert_tree: 0,
+                average_case_merk_delete_tree: 0,
+                average_case_merk_insert_element: 0,
+                average_case_merk_replace_element: 0,
+                average_case_merk_patch_element: 0,
+                average_case_merk_delete_element: 0,
+                add_average_case_has_raw_cost: 0,
+                add_average_case_has_raw_tree_cost: 0,
+                add_average_case_get_raw_cost: 0,
+                add_average_case_get_raw_tree_cost: 0,
+                add_average_case_get_cost: 0,
+            },
+            worst_case: GroveDBOperationsWorstCaseVersions {
+                add_worst_case_get_merk_at_path: 0,
+                worst_case_merk_replace_tree: 0,
+                worst_case_merk_insert_tree: 0,
+                worst_case_merk_delete_tree: 0,
+                worst_case_merk_insert_element: 0,
+                worst_case_merk_replace_element: 0,
+                worst_case_merk_patch_element: 0,
+                worst_case_merk_delete_element: 0,
+                add_worst_case_has_raw_cost: 0,
+                add_worst_case_get_raw_tree_cost: 0,
+                add_worst_case_get_raw_cost: 0,
+                add_worst_case_get_cost: 0,
+            },
+        },
+        path_query_methods: GroveDBPathQueryMethodVersions {
+            terminal_keys: 0,
+            merge: 0,
+            query_items_at_path: 0,
+        },
+        replication: GroveDBReplicationVersions {
+            get_subtrees_metadata: 0,
+            fetch_chunk: 0,
+            start_snapshot_syncing: 0,
+            apply_chunk: 0,
+        },
+    },
+    merk_versions: MerkVersions {
+        average_case_costs: MerkAverageCaseCostsVersions {
+            add_average_case_merk_propagate: 1, // changed
+            sum_tree_estimated_size: 1,         // changed
+        },
+    },
+};

grovedb/Cargo.toml

@@ -45,13 +45,14 @@ criterion = "0.5.1"
 hex = "0.4.3"
 pretty_assertions = "1.4.0"
 rand = "0.8.5"
+assert_matches = "1.5.0"
 
 [[bench]]
 name = "insertion_benchmark"
 harness = false
 
 [features]
-default = ["full"]
+default = ["full", "estimated_costs"]
 proof_debug = ["grovedb-merk/proof_debug"]
 serde = ["dep:serde", "grovedb-merk/serde", "indexmap/serde"]
 full = [

grovedb/src/batch/batch_structure.rs

@@ -17,7 +17,7 @@ use nohash_hasher::IntMap;
 #[cfg(feature = "full")]
 use crate::{
     batch::{key_info::KeyInfo, GroveOp, KeyInfoPath, QualifiedGroveDbOp, TreeCache},
-    Element, ElementFlags, Error,
+    ElementFlags, Error,
 };
 
 #[cfg(feature = "full")]
@@ -124,17 +124,14 @@ where
                 | GroveOp::InsertOrReplace { element }
                 | GroveOp::Replace { element }
                 | GroveOp::Patch { element, .. } => {
-                    if let Element::Tree(..) = element {
-                        cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, false));
-                    } else if let Element::SumTree(..) = element {
-                        cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, true));
+                    if let Some(tree_type) = element.tree_type() {
+                        cost_return_on_error!(&mut cost, merk_tree_cache.insert(&op, tree_type));
                     }
                     Ok(())
                 }
-                GroveOp::RefreshReference { .. }
-                | GroveOp::Delete
-                | GroveOp::DeleteTree
-                | GroveOp::DeleteSumTree => Ok(()),
+                GroveOp::RefreshReference { .. } | GroveOp::Delete | GroveOp::DeleteTree(_) => {
+                    Ok(())
+                }
                 GroveOp::ReplaceTreeRootKey { .. } | GroveOp::InsertTreeWithRootHash { .. } => {
                     Err(Error::InvalidBatchOperation(
                         "replace and insert tree hash are internal operations only",