From adb6512de2275b10e1190aab729e6a6ac45fb7d0 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Fri, 7 Jun 2024 06:51:55 -0700 Subject: [PATCH 01/10] WIP --- Cargo.lock | 52 ++++++------ Cargo.toml | 1 + synthesizer/Cargo.toml | 5 ++ synthesizer/src/vm/finalize.rs | 16 ++++ synthesizer/src/vm/helpers/history.rs | 111 ++++++++++++++++++++++++++ synthesizer/src/vm/helpers/mod.rs | 3 + 6 files changed, 162 insertions(+), 26 deletions(-) create mode 100644 synthesizer/src/vm/helpers/history.rs diff --git a/Cargo.lock b/Cargo.lock index 3697583dfc..f2f8454b64 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1124,7 +1124,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap 2.1.0", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -1282,9 +1282,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.1.0" +version = "2.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d530e1a18b1cb4c484e6e34556a0d948706958449fca0cab753d649f2bce3d1f" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" dependencies = [ "equivalent", "hashbrown 0.14.3", @@ -2413,11 +2413,11 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.111" +version = "1.0.117" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "176e46fa42316f18edd598015a5166857fc835ec732f5215eac6b7bdbf0a84f4" +checksum = "455182ea6142b14f93f4bc5320a2b31c1f266b66a4a5c858b013302a5d8cbfc3" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -2441,7 +2441,7 @@ version = "0.9.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b1bf28c79a99f70ee1f1d83d10c875d2e70618417fda01ad1785e027579d9d38" dependencies = [ - "indexmap 2.1.0", + "indexmap 2.2.6", "itoa", "ryu", "serde", @@ -2539,7 +2539,7 @@ dependencies = [ "clap", "colored", "dotenvy", - "indexmap 2.1.0", + "indexmap 2.2.6", "num-format", "once_cell", "parking_lot", @@ -2580,7 +2580,7 @@ dependencies = [ "fxhash", "hashbrown 0.14.3", "hex", - "indexmap 2.1.0", + "indexmap 2.2.6", "itertools 0.11.0", "lazy_static", "num-traits", @@ -2670,7 +2670,7 @@ name = "snarkvm-circuit-environment" version = "0.16.19" dependencies = [ "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "itertools 0.11.0", "nom", "num-traits", @@ -2861,7 +2861,7 @@ version = "0.16.19" dependencies = [ "aleo-std", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "rayon", "snarkvm-console-algorithms", "snarkvm-console-network", @@ -2873,7 +2873,7 @@ name = "snarkvm-console-network" version = "0.16.19" dependencies = [ "anyhow", - "indexmap 2.1.0", + "indexmap 2.2.6", "itertools 0.11.0", "lazy_static", "once_cell", @@ -2914,7 +2914,7 @@ dependencies = [ "bincode", "enum_index", "enum_index_derive", - "indexmap 2.1.0", + "indexmap 2.2.6", "num-derive", "num-traits", "once_cell", @@ -3065,7 +3065,7 @@ dependencies = [ "anyhow", "bincode", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "parking_lot", "rand", "rayon", @@ -3102,7 +3102,7 @@ name = "snarkvm-ledger-block" version = "0.16.19" dependencies = [ "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "once_cell", "rayon", "serde_json", @@ -3127,7 +3127,7 @@ version = "0.16.19" dependencies = [ "anyhow", "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "parking_lot", "proptest", "rand", @@ -3160,7 +3160,7 @@ name = "snarkvm-ledger-narwhal-batch-certificate" version = "0.16.19" dependencies = [ "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "rayon", "serde_json", "snarkvm-console", @@ -3174,7 +3174,7 @@ name = "snarkvm-ledger-narwhal-batch-header" version = "0.16.19" dependencies = [ "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "rayon", "serde_json", "snarkvm-console", @@ -3198,7 +3198,7 @@ name = "snarkvm-ledger-narwhal-subdag" version = "0.16.19" dependencies = [ "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "rayon", "serde_json", "snarkvm-console", @@ -3240,7 +3240,7 @@ dependencies = [ "anyhow", "bincode", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "lru", "once_cell", "parking_lot", @@ -3259,7 +3259,7 @@ version = "0.16.19" dependencies = [ "anyhow", "colored", - "indexmap 2.1.0", + "indexmap 2.2.6", "rand", "rand_chacha", "rayon", @@ -3286,7 +3286,7 @@ dependencies = [ "aleo-std-storage", "anyhow", "bincode", - "indexmap 2.1.0", + "indexmap 2.2.6", "once_cell", "parking_lot", "rayon", @@ -3343,7 +3343,7 @@ dependencies = [ "curl", "encoding", "hex", - "indexmap 2.1.0", + "indexmap 2.2.6", "itertools 0.11.0", "js-sys", "lazy_static", @@ -3371,7 +3371,7 @@ dependencies = [ "aleo-std", "anyhow", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "itertools 0.11.0", "lru", "once_cell", @@ -3407,7 +3407,7 @@ dependencies = [ "bincode", "colored", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "once_cell", "parking_lot", "rand", @@ -3432,7 +3432,7 @@ version = "0.16.19" dependencies = [ "bincode", "criterion", - "indexmap 2.1.0", + "indexmap 2.2.6", "paste", "rand", "rand_chacha", diff --git a/Cargo.toml b/Cargo.toml index d391deaf14..2140fafde8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -131,6 +131,7 @@ cli = [ aleo-cli = [ "snarkvm-synthesizer/aleo-cli" ] async = [ "snarkvm-ledger/async", "snarkvm-synthesizer/async" ] cuda = [ "snarkvm-algorithms/cuda" ] +history = [ "snarkvm-synthesizer/history" ] parameters_no_std_out = [ "snarkvm-parameters/no_std_out" ] noconfig = [ ] rocks = [ "snarkvm-ledger/rocks", "snarkvm-synthesizer/rocks" ] diff --git a/synthesizer/Cargo.toml b/synthesizer/Cargo.toml index 13167e27de..a7ebb9bf4d 100644 --- a/synthesizer/Cargo.toml +++ b/synthesizer/Cargo.toml @@ -31,6 +31,7 @@ snark = [ "synthesizer-snark" ] aleo-cli = [ ] async = [ "ledger-query/async", "synthesizer-process/async" ] cuda = [ "algorithms/cuda" ] +history = [ "serde_json" ] rocks = [ "ledger-store/rocks" ] serial = [ "console/serial", @@ -162,6 +163,10 @@ version = "0.8" version = "1" optional = true +[dependencies.serde_json] +version = "1.0.117" +optional = true + [dependencies.tracing] version = "0.1" diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 78e3ab8a66..47ff85b24e 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1260,6 +1260,22 @@ impl> VM { // Insert the next committee into storage. store.committee_store().insert(state.block_height(), next_committee)?; + + #[cfg(feature = "history")] + { + // Load a `History` object. + let history = History::new(N::ID, store.storage_mode().clone()); + + // Write the committee mapping as JSON. + history.store_entry(HistoryVariant::Committee, state.block_height(), &next_committee_map)?; + // Write the delegated mapping as JSON. + history.store_entry(HistoryVariant::Delegated, state.block_height(), &next_delegated_map)?; + // Write the bonded mapping as JSON. + history.store_entry(HistoryVariant::Bonded, state.block_height(), &next_bonded_map)?; + + // TODO: Write the unbonding mapping. + } + // Store the finalize operations for updating the committee and bonded mapping. finalize_operations.extend(&[ // Replace the committee mapping in storage. diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs new file mode 100644 index 0000000000..9b6790c3eb --- /dev/null +++ b/synthesizer/src/vm/helpers/history.rs @@ -0,0 +1,111 @@ +// Copyright (C) 2019-2023 Aleo Systems Inc. +// This file is part of the snarkVM library. + +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at: +// http://www.apache.org/licenses/LICENSE-2.0 + +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use console::prelude::Serialize; + +use aleo_std::{aleo_ledger_dir, StorageMode}; + +use anyhow::Result; +use serde_json; +use std::path::PathBuf; + +/// Returns the path where a history directory may be stored. +pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBuf { + const HISTORY_DIRECTORY_NAME: &str = "history"; + + // Create the name of the history directory. + let directory_name = match &storage_mode { + StorageMode::Development(id) => format!(".{HISTORY_DIRECTORY_NAME}-{}-{}", network, id), + StorageMode::Production | StorageMode::Custom(_) => format!("{HISTORY_DIRECTORY_NAME}-{}", network), + }; + + // Obtain the path to the ledger. + let mut path = aleo_ledger_dir(network, storage_mode); + // Go to the folder right above the ledger. + path.pop(); + // Append the history directory's name. + path.push(directory_name); + + path +} + +#[derive(Copy, Clone)] +pub enum HistoryVariant { + /// A `committee` mapping. + Committee, + /// A `bonded` mapping. + Bonded, + /// A `delegated` mapping. + Delegated, + /// An `unbonding` mapping. + Unbonding, +} + +impl HistoryVariant { + /// Returns the name of the variant. + pub fn name(&self) -> &'static str { + match self { + Self::Committee => "committee", + Self::Bonded => "bonded", + Self::Delegated => "delegated", + Self::Unbonding => "unbonding", + } + } +} + +pub struct History { + /// The path to the history directory. + path: PathBuf, +} + +impl History { + /// Initializes a new instance of the history. + pub fn new(network: u16, storage_mode: StorageMode) -> Self { + Self { path: history_directory_path(network, storage_mode) } + } + + /// Stores an entry into the history. + pub fn store_entry(&self, variant: HistoryVariant, height: u32, data: &T) -> Result<()> + where + T: Serialize + ?Sized, + { + // Get the path to the variant directory. + let variant_path = self.variant_path(variant); + // Create the variant directory if it does not exist. + if !variant_path.exists() { + std::fs::create_dir_all(&variant_path)?; + } + // Write the entry to the variant directory. + let entry_path = variant_path.join(format!("{}-{height}.json", variant.name())); + std::fs::write(entry_path, serde_json::to_string_pretty(data)?)?; + + Ok(()) + } + + /// Loads an entry from the history. + pub fn load_entry(&self, variant: HistoryVariant, height: u32) -> Result { + // Get the path to the variant directory. + let variant_path = self.variant_path(variant); + // Read the entry from the variant directory. + let entry_path = variant_path.join(format!("{}-{height}.json", variant.name())); + let result = std::fs::read_to_string(entry_path)?; + + Ok(result) + } + + /// Get the path to a variant directory. + fn variant_path(&self, variant: HistoryVariant) -> PathBuf { + self.path.join(variant.name()) + } +} diff --git a/synthesizer/src/vm/helpers/mod.rs b/synthesizer/src/vm/helpers/mod.rs index 1fd8ec7321..679f265e70 100644 --- a/synthesizer/src/vm/helpers/mod.rs +++ b/synthesizer/src/vm/helpers/mod.rs @@ -15,6 +15,9 @@ pub(crate) mod committee; pub use committee::*; +mod history; +pub use history::*; + mod macros; mod rewards; From b7242182fa6ee9a70da60bc7f58ab4f7394f3ee6 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Fri, 7 Jun 2024 14:52:01 -0700 Subject: [PATCH 02/10] WIP --- synthesizer/src/vm/finalize.rs | 9 ++++--- synthesizer/src/vm/helpers/history.rs | 36 +++++++++++---------------- 2 files changed, 20 insertions(+), 25 deletions(-) diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 47ff85b24e..7ba81bd57b 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1266,14 +1266,15 @@ impl> VM { // Load a `History` object. let history = History::new(N::ID, store.storage_mode().clone()); - // Write the committee mapping as JSON. - history.store_entry(HistoryVariant::Committee, state.block_height(), &next_committee_map)?; // Write the delegated mapping as JSON. - history.store_entry(HistoryVariant::Delegated, state.block_height(), &next_delegated_map)?; + history.store_entry(state.block_height(), HistoryVariant::Delegated, &next_delegated_map)?; // Write the bonded mapping as JSON. - history.store_entry(HistoryVariant::Bonded, state.block_height(), &next_bonded_map)?; + history.store_entry(state.block_height(), HistoryVariant::Bonded, &next_bonded_map)?; // TODO: Write the unbonding mapping. + let unbonding_mapping = Identifier::from_str("unbonding")?; + let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; + history.store_entry(state.block_height(), HistoryVariant::Unbonding, &unbonding_map)?; } // Store the finalize operations for updating the committee and bonded mapping. diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index 9b6790c3eb..dae6150967 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -42,8 +42,6 @@ pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBu #[derive(Copy, Clone)] pub enum HistoryVariant { - /// A `committee` mapping. - Committee, /// A `bonded` mapping. Bonded, /// A `delegated` mapping. @@ -56,7 +54,6 @@ impl HistoryVariant { /// Returns the name of the variant. pub fn name(&self) -> &'static str { match self { - Self::Committee => "committee", Self::Bonded => "bonded", Self::Delegated => "delegated", Self::Unbonding => "unbonding", @@ -76,36 +73,33 @@ impl History { } /// Stores an entry into the history. - pub fn store_entry(&self, variant: HistoryVariant, height: u32, data: &T) -> Result<()> + pub fn store_entry(&self, height: u32, variant: HistoryVariant, data: &T) -> Result<()> where T: Serialize + ?Sized, { - // Get the path to the variant directory. - let variant_path = self.variant_path(variant); - // Create the variant directory if it does not exist. - if !variant_path.exists() { - std::fs::create_dir_all(&variant_path)?; + // Get the path to the block directory. + let block_path = self.path.join(format!("block-{height}")); + // Create the block directory if it does not exist. + if !block_path.exists() { + std::fs::create_dir_all(&block_path)?; } - // Write the entry to the variant directory. - let entry_path = variant_path.join(format!("{}-{height}.json", variant.name())); + + // Write the entry to the block directory. + let entry_path = block_path.join(format!("block-{height}-{}.json", variant.name())); std::fs::write(entry_path, serde_json::to_string_pretty(data)?)?; Ok(()) } /// Loads an entry from the history. - pub fn load_entry(&self, variant: HistoryVariant, height: u32) -> Result { - // Get the path to the variant directory. - let variant_path = self.variant_path(variant); - // Read the entry from the variant directory. - let entry_path = variant_path.join(format!("{}-{height}.json", variant.name())); + pub fn load_entry(&self, height: u32, variant: HistoryVariant) -> Result { + // Get the path to the block directory. + let block_path = self.path.join(format!("block-{height}")); + // Get the path to the entry. + let entry_path = block_path.join(format!("block-{height}-{}.json", variant.name())); + // Load the entry. let result = std::fs::read_to_string(entry_path)?; Ok(result) } - - /// Get the path to a variant directory. - fn variant_path(&self, variant: HistoryVariant) -> PathBuf { - self.path.join(variant.name()) - } } From e3bc4f5bc715ee0bc7b209c1fd1d9be41f5c00e1 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Sun, 9 Jun 2024 09:56:04 -0700 Subject: [PATCH 03/10] Fix and cleanup --- Cargo.lock | 1 + synthesizer/Cargo.toml | 9 +++-- synthesizer/src/vm/finalize.rs | 6 ++-- synthesizer/src/vm/helpers/history.rs | 48 ++++++++++++++------------- 4 files changed, 36 insertions(+), 28 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f2f8454b64..741d737caf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3378,6 +3378,7 @@ dependencies = [ "parking_lot", "rand", "rayon", + "serde", "serde_json", "serde_yaml", "snarkvm-algorithms", diff --git a/synthesizer/Cargo.toml b/synthesizer/Cargo.toml index a7ebb9bf4d..92d5291804 100644 --- a/synthesizer/Cargo.toml +++ b/synthesizer/Cargo.toml @@ -31,7 +31,7 @@ snark = [ "synthesizer-snark" ] aleo-cli = [ ] async = [ "ledger-query/async", "synthesizer-process/async" ] cuda = [ "algorithms/cuda" ] -history = [ "serde_json" ] +history = [ "serde", "serde_json" ] rocks = [ "ledger-store/rocks" ] serial = [ "console/serial", @@ -163,8 +163,13 @@ version = "0.8" version = "1" optional = true +[dependencies.serde] +version = "1.0" +optional = true + [dependencies.serde_json] -version = "1.0.117" +version = "1.0" +features = [ "preserve_order" ] optional = true [dependencies.tracing] diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 7ba81bd57b..f0410eed8d 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1267,14 +1267,14 @@ impl> VM { let history = History::new(N::ID, store.storage_mode().clone()); // Write the delegated mapping as JSON. - history.store_entry(state.block_height(), HistoryVariant::Delegated, &next_delegated_map)?; + history.store_entry(state.block_height(), MappingName::Delegated, &next_delegated_map)?; // Write the bonded mapping as JSON. - history.store_entry(state.block_height(), HistoryVariant::Bonded, &next_bonded_map)?; + history.store_entry(state.block_height(), MappingName::Bonded, &next_bonded_map)?; // TODO: Write the unbonding mapping. let unbonding_mapping = Identifier::from_str("unbonding")?; let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; - history.store_entry(state.block_height(), HistoryVariant::Unbonding, &unbonding_map)?; + history.store_entry(state.block_height(), MappingName::Unbonding, &unbonding_map)?; } // Store the finalize operations for updating the committee and bonded mapping. diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index dae6150967..bece0c171b 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -12,22 +12,25 @@ // See the License for the specific language governing permissions and // limitations under the License. -use console::prelude::Serialize; +use console::prelude::{Deserialize, Serialize}; use aleo_std::{aleo_ledger_dir, StorageMode}; use anyhow::Result; use serde_json; -use std::path::PathBuf; +use std::{ + fmt::{Display, Formatter}, + path::PathBuf, +}; -/// Returns the path where a history directory may be stored. +/// Returns the path where a `history` directory may be stored. pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBuf { const HISTORY_DIRECTORY_NAME: &str = "history"; // Create the name of the history directory. let directory_name = match &storage_mode { - StorageMode::Development(id) => format!(".{HISTORY_DIRECTORY_NAME}-{}-{}", network, id), - StorageMode::Production | StorageMode::Custom(_) => format!("{HISTORY_DIRECTORY_NAME}-{}", network), + StorageMode::Development(id) => format!(".{HISTORY_DIRECTORY_NAME}-{network}-{id}"), + StorageMode::Production | StorageMode::Custom(_) => format!("{HISTORY_DIRECTORY_NAME}-{network}"), }; // Obtain the path to the ledger. @@ -40,23 +43,22 @@ pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBu path } -#[derive(Copy, Clone)] -pub enum HistoryVariant { - /// A `bonded` mapping. +#[derive(Copy, Clone, Serialize, Deserialize)] +pub enum MappingName { + /// The `bonded` mapping. Bonded, - /// A `delegated` mapping. + /// The `delegated` mapping. Delegated, - /// An `unbonding` mapping. + /// The `unbonding` mapping. Unbonding, } -impl HistoryVariant { - /// Returns the name of the variant. - pub fn name(&self) -> &'static str { +impl Display for MappingName { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { match self { - Self::Bonded => "bonded", - Self::Delegated => "delegated", - Self::Unbonding => "unbonding", + Self::Bonded => write!(f, "bonded"), + Self::Delegated => write!(f, "delegated"), + Self::Unbonding => write!(f, "unbonding"), } } } @@ -67,13 +69,13 @@ pub struct History { } impl History { - /// Initializes a new instance of the history. + /// Initializes a new instance of `History`. pub fn new(network: u16, storage_mode: StorageMode) -> Self { Self { path: history_directory_path(network, storage_mode) } } - /// Stores an entry into the history. - pub fn store_entry(&self, height: u32, variant: HistoryVariant, data: &T) -> Result<()> + /// Stores a mapping from a given block in the history directory as JSON. + pub fn store_entry(&self, height: u32, mapping: MappingName, data: &T) -> Result<()> where T: Serialize + ?Sized, { @@ -85,18 +87,18 @@ impl History { } // Write the entry to the block directory. - let entry_path = block_path.join(format!("block-{height}-{}.json", variant.name())); + let entry_path = block_path.join(format!("block-{height}-{mapping}.json")); std::fs::write(entry_path, serde_json::to_string_pretty(data)?)?; Ok(()) } - /// Loads an entry from the history. - pub fn load_entry(&self, height: u32, variant: HistoryVariant) -> Result { + /// Loads the JSON string for a mapping from a given block from the history directory. + pub fn load_entry(&self, height: u32, mapping: MappingName) -> Result { // Get the path to the block directory. let block_path = self.path.join(format!("block-{height}")); // Get the path to the entry. - let entry_path = block_path.join(format!("block-{height}-{}.json", variant.name())); + let entry_path = block_path.join(format!("block-{height}-{mapping}.json")); // Load the entry. let result = std::fs::read_to_string(entry_path)?; From 9d0771f82989bfdade6249eee32e48d3b0cfc7b5 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Sun, 9 Jun 2024 19:17:44 -0700 Subject: [PATCH 04/10] Cleanup --- synthesizer/src/vm/finalize.rs | 7 +++---- synthesizer/src/vm/helpers/history.rs | 5 +++-- synthesizer/src/vm/helpers/mod.rs | 2 ++ 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index f0410eed8d..b359e5bfe1 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1267,14 +1267,13 @@ impl> VM { let history = History::new(N::ID, store.storage_mode().clone()); // Write the delegated mapping as JSON. - history.store_entry(state.block_height(), MappingName::Delegated, &next_delegated_map)?; + history.store_mapping(state.block_height(), MappingName::Delegated, &next_delegated_map)?; // Write the bonded mapping as JSON. - history.store_entry(state.block_height(), MappingName::Bonded, &next_bonded_map)?; + history.store_mapping(state.block_height(), MappingName::Bonded, &next_bonded_map)?; - // TODO: Write the unbonding mapping. let unbonding_mapping = Identifier::from_str("unbonding")?; let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; - history.store_entry(state.block_height(), MappingName::Unbonding, &unbonding_map)?; + history.store_mapping(state.block_height(), MappingName::Unbonding, &unbonding_map)?; } // Store the finalize operations for updating the committee and bonded mapping. diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index bece0c171b..84a719b0c1 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -44,6 +44,7 @@ pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBu } #[derive(Copy, Clone, Serialize, Deserialize)] +#[serde(rename_all = "lowercase")] pub enum MappingName { /// The `bonded` mapping. Bonded, @@ -75,7 +76,7 @@ impl History { } /// Stores a mapping from a given block in the history directory as JSON. - pub fn store_entry(&self, height: u32, mapping: MappingName, data: &T) -> Result<()> + pub fn store_mapping(&self, height: u32, mapping: MappingName, data: &T) -> Result<()> where T: Serialize + ?Sized, { @@ -94,7 +95,7 @@ impl History { } /// Loads the JSON string for a mapping from a given block from the history directory. - pub fn load_entry(&self, height: u32, mapping: MappingName) -> Result { + pub fn load_mapping(&self, height: u32, mapping: MappingName) -> Result { // Get the path to the block directory. let block_path = self.path.join(format!("block-{height}")); // Get the path to the entry. diff --git a/synthesizer/src/vm/helpers/mod.rs b/synthesizer/src/vm/helpers/mod.rs index 679f265e70..7b9b7c238c 100644 --- a/synthesizer/src/vm/helpers/mod.rs +++ b/synthesizer/src/vm/helpers/mod.rs @@ -15,7 +15,9 @@ pub(crate) mod committee; pub use committee::*; +#[cfg(feature = "history")] mod history; +#[cfg(feature = "history")] pub use history::*; mod macros; From f2b14e3499bf60104bf112f110db982f591dfca8 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:03:26 -0700 Subject: [PATCH 05/10] Address feedback --- synthesizer/src/vm/finalize.rs | 33 ++++++++-------- synthesizer/src/vm/helpers/history.rs | 54 ++++++++++++++++++++------- 2 files changed, 58 insertions(+), 29 deletions(-) diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index bd46a34e07..0dfbba4e12 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -495,7 +495,7 @@ impl> VM { let post_ratifications = reward_ratifications.iter().chain(post_ratifications); // Process the post-ratifications. - match Self::atomic_post_ratify(&self.puzzle, store, state, post_ratifications, solutions) { + match Self::atomic_post_ratify::(&self.puzzle, store, state, post_ratifications, solutions) { // Store the finalize operations from the post-ratify. Ok(operations) => ratified_finalize_operations.extend(operations), // Note: This will abort the entire atomic batch. @@ -739,7 +739,7 @@ impl> VM { /* Perform the ratifications after finalize. */ - match Self::atomic_post_ratify(&self.puzzle, store, state, post_ratifications, solutions) { + match Self::atomic_post_ratify::(&self.puzzle, store, state, post_ratifications, solutions) { // Store the finalize operations from the post-ratify. Ok(operations) => ratified_finalize_operations.extend(operations), // Note: This will abort the entire atomic batch. @@ -1191,7 +1191,7 @@ impl> VM { /// Performs the post-ratifications after finalizing transactions. #[inline] - fn atomic_post_ratify<'a>( + fn atomic_post_ratify<'a, const IS_FINALIZE: bool>( puzzle: &Puzzle, store: &FinalizeStore, state: FinalizeGlobalState, @@ -1259,19 +1259,22 @@ impl> VM { // Insert the next committee into storage. store.committee_store().insert(state.block_height(), next_committee)?; - #[cfg(feature = "history")] + #[cfg(all(feature = "history", feature = "rocks"))] { - // Load a `History` object. - let history = History::new(N::ID, store.storage_mode().clone()); - - // Write the delegated mapping as JSON. - history.store_mapping(state.block_height(), MappingName::Delegated, &next_delegated_map)?; - // Write the bonded mapping as JSON. - history.store_mapping(state.block_height(), MappingName::Bonded, &next_bonded_map)?; - - let unbonding_mapping = Identifier::from_str("unbonding")?; - let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; - history.store_mapping(state.block_height(), MappingName::Unbonding, &unbonding_map)?; + // When finalizing in `FinalizeMode::RealRun`, store the delegated and bonded mappings in history. + if IS_FINALIZE { + // Load a `History` object. + let history = History::new(N::ID, store.storage_mode()); + + // Write the delegated mapping as JSON. + history.store_mapping(state.block_height(), MappingName::Delegated, &next_delegated_map)?; + // Write the bonded mapping as JSON. + history.store_mapping(state.block_height(), MappingName::Bonded, &next_bonded_map)?; + + let unbonding_mapping = Identifier::from_str("unbonding")?; + let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; + history.store_mapping(state.block_height(), MappingName::Unbonding, &unbonding_map)?; + } } // Store the finalize operations for updating the committee and bonded mapping. diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index 84a719b0c1..3791c4a76c 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -24,7 +24,7 @@ use std::{ }; /// Returns the path where a `history` directory may be stored. -pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBuf { +pub fn history_directory_path(network: u16, storage_mode: &StorageMode) -> PathBuf { const HISTORY_DIRECTORY_NAME: &str = "history"; // Create the name of the history directory. @@ -34,7 +34,7 @@ pub fn history_directory_path(network: u16, storage_mode: StorageMode) -> PathBu }; // Obtain the path to the ledger. - let mut path = aleo_ledger_dir(network, storage_mode); + let mut path = aleo_ledger_dir(network, storage_mode.clone()); // Go to the folder right above the ledger. path.pop(); // Append the history directory's name. @@ -71,7 +71,7 @@ pub struct History { impl History { /// Initializes a new instance of `History`. - pub fn new(network: u16, storage_mode: StorageMode) -> Self { + pub fn new(network: u16, storage_mode: &StorageMode) -> Self { Self { path: history_directory_path(network, storage_mode) } } @@ -81,15 +81,15 @@ impl History { T: Serialize + ?Sized, { // Get the path to the block directory. - let block_path = self.path.join(format!("block-{height}")); + let path = self.block_path(height); // Create the block directory if it does not exist. - if !block_path.exists() { - std::fs::create_dir_all(&block_path)?; + if !path.exists() { + std::fs::create_dir_all(&path)?; } // Write the entry to the block directory. - let entry_path = block_path.join(format!("block-{height}-{mapping}.json")); - std::fs::write(entry_path, serde_json::to_string_pretty(data)?)?; + let path = path.join(format!("block-{height}-{mapping}.json")); + std::fs::write(path, serde_json::to_string_pretty(data)?)?; Ok(()) } @@ -97,12 +97,38 @@ impl History { /// Loads the JSON string for a mapping from a given block from the history directory. pub fn load_mapping(&self, height: u32, mapping: MappingName) -> Result { // Get the path to the block directory. - let block_path = self.path.join(format!("block-{height}")); - // Get the path to the entry. - let entry_path = block_path.join(format!("block-{height}-{mapping}.json")); - // Load the entry. - let result = std::fs::read_to_string(entry_path)?; + let path = self.block_path(height); + // Get the path to the block file. + let path = path.join(format!("block-{height}-{mapping}.json")); - Ok(result) + // Read the file. + let data = std::fs::read_to_string(path)?; + + Ok(data) + } + + /// Removes data for a given block from the history directory. + pub fn remove_block(&self, height: u32) -> Result<()> { + let path = self.block_path(height); + // Remove the block directory if it exists. + if path.exists() { + std::fs::remove_dir_all(path)?; + } + + Ok(()) + } + + // A helper function to get the path to the block directory. + fn block_path(&self, height: u32) -> PathBuf { + // Get the path the directory group. + let group = Self::group(height); + let path = self.path.join(format!("group-{group}")); + // Get the path to the block directory. + path.join(format!("block-{height}")) + } + + // A helper function to calculate the group number for a given block height. + fn group(height: u32) -> u32 { + height % (2 << 16) } } From 38666d01a9b8f9511dc3098d506e8b0231e33cd3 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Thu, 13 Jun 2024 08:27:40 -0700 Subject: [PATCH 06/10] Fix grouping --- synthesizer/src/vm/helpers/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index 3791c4a76c..d4cc83d1f1 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -129,6 +129,6 @@ impl History { // A helper function to calculate the group number for a given block height. fn group(height: u32) -> u32 { - height % (2 << 16) + height.saturating_div(2u32 << 16) } } From a084a31aafa743d799b3749e43b9004a82e9247e Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Thu, 13 Jun 2024 09:44:57 -0700 Subject: [PATCH 07/10] Remove unused method --- synthesizer/src/vm/helpers/history.rs | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index d4cc83d1f1..7f57bf03b7 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -107,17 +107,6 @@ impl History { Ok(data) } - /// Removes data for a given block from the history directory. - pub fn remove_block(&self, height: u32) -> Result<()> { - let path = self.block_path(height); - // Remove the block directory if it exists. - if path.exists() { - std::fs::remove_dir_all(path)?; - } - - Ok(()) - } - // A helper function to get the path to the block directory. fn block_path(&self, height: u32) -> PathBuf { // Get the path the directory group. From c7ca7f47517b18685d2281938ae67c51a8f1752a Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Thu, 13 Jun 2024 11:32:38 -0700 Subject: [PATCH 08/10] Fix --- synthesizer/src/vm/helpers/history.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index 7f57bf03b7..1c172abf4f 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -118,6 +118,6 @@ impl History { // A helper function to calculate the group number for a given block height. fn group(height: u32) -> u32 { - height.saturating_div(2u32 << 16) + height.saturating_div(u16::MAX as u32) } } From 7e31e109b2d918337eac9d46224f4f22c3c72e44 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Thu, 13 Jun 2024 13:30:59 -0700 Subject: [PATCH 09/10] Clippy --- algorithms/src/r1cs/test_constraint_system.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/algorithms/src/r1cs/test_constraint_system.rs b/algorithms/src/r1cs/test_constraint_system.rs index 9f9b43ce73..d43b3d161c 100644 --- a/algorithms/src/r1cs/test_constraint_system.rs +++ b/algorithms/src/r1cs/test_constraint_system.rs @@ -306,7 +306,7 @@ impl TestConstraintSystem { ns_idx } Entry::Occupied(e) => { - let interned_segments = e.remove_entry().0; + let interned_segments = e.swap_remove_entry().0; panic!("tried to create object at existing path: {}", self.unintern_path(interned_segments)); } } From 8e8fd2bcf7cfc4c2e23e632c650b2491f6f13611 Mon Sep 17 00:00:00 2001 From: Pranav Gaddamadugu <23022326+d0cd@users.noreply.github.com> Date: Fri, 14 Jun 2024 08:55:38 -0700 Subject: [PATCH 10/10] Add metadata and withdraw mappings --- synthesizer/src/vm/finalize.rs | 12 ++++++++++++ synthesizer/src/vm/helpers/history.rs | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/synthesizer/src/vm/finalize.rs b/synthesizer/src/vm/finalize.rs index 0dfbba4e12..080a5414c9 100644 --- a/synthesizer/src/vm/finalize.rs +++ b/synthesizer/src/vm/finalize.rs @@ -1268,12 +1268,24 @@ impl> VM { // Write the delegated mapping as JSON. history.store_mapping(state.block_height(), MappingName::Delegated, &next_delegated_map)?; + // Write the bonded mapping as JSON. history.store_mapping(state.block_height(), MappingName::Bonded, &next_bonded_map)?; + // Write the metadata mapping as JSON. + let metadata_mapping = Identifier::from_str("metadata")?; + let metadata_map = store.get_mapping_speculative(program_id, metadata_mapping)?; + history.store_mapping(state.block_height(), MappingName::Metadata, &metadata_map)?; + + // Write the unbonding mapping as JSON. let unbonding_mapping = Identifier::from_str("unbonding")?; let unbonding_map = store.get_mapping_speculative(program_id, unbonding_mapping)?; history.store_mapping(state.block_height(), MappingName::Unbonding, &unbonding_map)?; + + // Write the withdraw mapping as JSON. + let withdraw_mapping = Identifier::from_str("withdraw")?; + let withdraw_map = store.get_mapping_speculative(program_id, withdraw_mapping)?; + history.store_mapping(state.block_height(), MappingName::Withdraw, &withdraw_map)?; } } diff --git a/synthesizer/src/vm/helpers/history.rs b/synthesizer/src/vm/helpers/history.rs index 1c172abf4f..efea1e1825 100644 --- a/synthesizer/src/vm/helpers/history.rs +++ b/synthesizer/src/vm/helpers/history.rs @@ -50,8 +50,12 @@ pub enum MappingName { Bonded, /// The `delegated` mapping. Delegated, + /// The `metadata` mapping. + Metadata, /// The `unbonding` mapping. Unbonding, + /// The `withdraw` mapping. + Withdraw, } impl Display for MappingName { @@ -59,7 +63,9 @@ impl Display for MappingName { match self { Self::Bonded => write!(f, "bonded"), Self::Delegated => write!(f, "delegated"), + Self::Metadata => write!(f, "metadata"), Self::Unbonding => write!(f, "unbonding"), + Self::Withdraw => write!(f, "withdraw"), } } }