incremental hash (#4197)

* incremental hash

* rebase

* reformat

* read/write to final state

* factor prng

* imrovements

* remove comment

* final_state_db corrrection

* load initial trail hash

* clippy

* fix tests

* debug tests

* Fix bootstrap test

* Fix execution tests

* remove comment

---------

Co-authored-by: Leo-Besancon <[email protected]>

massa-async-pool/src/message.rs

@@ -321,7 +321,7 @@ impl Default for AsyncMessage {
             destination: genesis_address,
             validity_start: slot_zero,
             validity_end: slot_zero,
-            hash: Hash::from_bytes(&[0; 32]),
+            hash: Hash::zero(),
             ..Default::default()
         }
     }
@@ -362,7 +362,7 @@ impl AsyncMessage {
             can_be_executed: can_be_executed.unwrap_or(trigger.is_none()),
             trigger,
             // placeholder hash to serialize the message, replaced below
-            hash: Hash::from_bytes(&[0; 32]),
+            hash: Hash::zero(),
         };
         async_message_ser
             .serialize(&message, &mut buffer)

massa-bootstrap/src/client.rs

@@ -429,6 +429,7 @@ pub fn get_state(
                             err
                         ))
                     })?;
+                final_state_guard.init_execution_trail_hash();
             }
 
             // create the initial cycle of PoS cycle_history

massa-bootstrap/src/tests/scenarios.rs

@@ -6,7 +6,8 @@ use super::tools::{
 use crate::listener::PollEvent;
 use crate::tests::tools::{
     assert_eq_bootstrap_graph, get_random_async_pool_changes, get_random_executed_de_changes,
-    get_random_executed_ops_changes, get_random_pos_changes,
+    get_random_executed_ops_changes, get_random_execution_trail_hash_change,
+    get_random_pos_changes,
 };
 use crate::BootstrapError;
 use crate::{
@@ -307,6 +308,7 @@ fn test_bootstrap_server() {
             async_pool_changes: get_random_async_pool_changes(10, thread_count),
             executed_ops_changes: get_random_executed_ops_changes(10),
             executed_denunciations_changes: get_random_executed_de_changes(10),
+            execution_trail_hash_change: get_random_execution_trail_hash_change(true),
         };
 
         let next = current_slot.get_next_slot(thread_count).unwrap();
@@ -452,6 +454,7 @@ fn test_bootstrap_server() {
                     async_pool_changes: get_random_async_pool_changes(10, thread_count),
                     executed_ops_changes: get_random_executed_ops_changes(10),
                     executed_denunciations_changes: get_random_executed_de_changes(10),
+                    execution_trail_hash_change: get_random_execution_trail_hash_change(true),
                 };
 
                 let mut batch = DBBatch::new();

massa-bootstrap/src/tests/tools.rs

@@ -18,7 +18,7 @@ use massa_executed_ops::{
 use massa_final_state::test_exports::create_final_state;
 use massa_final_state::{FinalState, FinalStateConfig};
 use massa_hash::Hash;
-use massa_ledger_exports::{LedgerChanges, LedgerEntry, SetUpdateOrDelete};
+use massa_ledger_exports::{LedgerChanges, LedgerEntry, SetOrKeep, SetUpdateOrDelete};
 use massa_ledger_worker::test_exports::create_final_ledger;
 use massa_models::block::BlockDeserializerArgs;
 use massa_models::bytecode::Bytecode;
@@ -287,6 +287,15 @@ pub fn get_random_executed_de_changes(r_limit: u64) -> ExecutedDenunciationsChan
     de_changes
 }
 
+/// generates a random execution trail hash change
+pub fn get_random_execution_trail_hash_change(always_set: bool) -> SetOrKeep<massa_hash::Hash> {
+    if always_set || rand::thread_rng().gen() {
+        SetOrKeep::Set(Hash::compute_from(&get_some_random_bytes()))
+    } else {
+        SetOrKeep::Keep
+    }
+}
+
 /// generates a random bootstrap state for the final state
 pub fn get_random_final_state_bootstrap(
     pos: PoSFinalState,
@@ -344,7 +353,7 @@ pub fn get_random_final_state_bootstrap(
     ))
     .unwrap();
 
-    create_final_state(
+    let mut final_state = create_final_state(
         config,
         Box::new(final_ledger),
         async_pool,
@@ -353,7 +362,10 @@ pub fn get_random_final_state_bootstrap(
         executed_denunciations,
         mip_store,
         db,
-    )
+    );
+
+    final_state.init_execution_trail_hash();
+    final_state
 }
 
 pub fn get_dummy_block_id(s: &str) -> BlockId {

massa-db-exports/src/constants.rs

@@ -1,6 +1,4 @@
 // Commons
-pub const LSMTREE_NODES_CF: &str = "lsmtree_nodes";
-pub const LSMTREE_VALUES_CF: &str = "lsmtree_values";
 pub const METADATA_CF: &str = "metadata";
 pub const STATE_CF: &str = "state";
 pub const VERSIONING_CF: &str = "versioning";
@@ -31,6 +29,7 @@ pub const EXECUTED_DENUNCIATIONS_PREFIX: &str = "executed_denunciations/";
 pub const LEDGER_PREFIX: &str = "ledger/";
 pub const MIP_STORE_PREFIX: &str = "versioning/";
 pub const MIP_STORE_STATS_PREFIX: &str = "versioning_stats/";
+pub const EXECUTION_TRAIL_HASH_PREFIX: &str = "execution_trail_hash/";
 
 // Async Pool
 pub const MESSAGE_DESER_ERROR: &str = "critical: message deserialization failed";

massa-execution-worker/src/active_history.rs

@@ -231,6 +231,17 @@ impl ActiveHistory {
         None
     }
 
+    /// Gets the execution trail hash
+    pub fn get_execution_trail_hash(&self) -> HistorySearchResult<massa_hash::Hash> {
+        for history_element in self.0.iter().rev() {
+            if let SetOrKeep::Set(hash) = history_element.state_changes.execution_trail_hash_change
+            {
+                return HistorySearchResult::Present(hash);
+            }
+        }
+        HistorySearchResult::NoInfo
+    }
+
     /// Gets the index of a slot in history
     pub fn get_slot_index(&self, slot: &Slot, thread_count: u8) -> SlotIndexPosition {
         let first_slot = match self.0.front() {

massa-execution-worker/src/context.rs

@@ -7,6 +7,7 @@
 //! More generally, the context acts only on its own state
 //! and does not write anything persistent to the consensus state.
 
+use crate::active_history::HistorySearchResult;
 use crate::speculative_async_pool::SpeculativeAsyncPool;
 use crate::speculative_executed_denunciations::SpeculativeExecutedDenunciations;
 use crate::speculative_executed_ops::SpeculativeExecutedOps;
@@ -21,7 +22,7 @@ use massa_execution_exports::{
 };
 use massa_final_state::{FinalState, StateChanges};
 use massa_hash::Hash;
-use massa_ledger_exports::LedgerChanges;
+use massa_ledger_exports::{LedgerChanges, SetOrKeep};
 use massa_models::address::ExecutionAddressCycleInfo;
 use massa_models::bytecode::Bytecode;
 use massa_models::denunciation::DenunciationIndex;
@@ -155,13 +156,16 @@ pub struct ExecutionContext {
     /// operation id that originally caused this execution (if any)
     pub origin_operation_id: Option<OperationId>,
 
-    // cache of compiled runtime modules
+    /// Execution trail hash
+    pub execution_trail_hash: Hash,
+
+    /// cache of compiled runtime modules
     pub module_cache: Arc<RwLock<ModuleCache>>,
 
-    // Vesting Manager
+    /// Vesting Manager
     pub vesting_manager: Arc<VestingManager>,
 
-    // Address factory
+    /// Address factory
     pub address_factory: AddressFactory,
 }
 
@@ -183,6 +187,7 @@ impl ExecutionContext {
         module_cache: Arc<RwLock<ModuleCache>>,
         vesting_manager: Arc<VestingManager>,
         mip_store: MipStore,
+        execution_trail_hash: massa_hash::Hash,
     ) -> Self {
         ExecutionContext {
             speculative_ledger: SpeculativeLedger::new(
@@ -219,13 +224,14 @@ impl ExecutionContext {
             stack: Default::default(),
             read_only: Default::default(),
             events: Default::default(),
-            unsafe_rng: Xoshiro256PlusPlus::from_seed([0u8; 32]),
+            unsafe_rng: init_prng(&execution_trail_hash),
             creator_address: Default::default(),
             origin_operation_id: Default::default(),
             module_cache,
             config,
             vesting_manager,
             address_factory: AddressFactory { mip_store },
+            execution_trail_hash,
         }
     }
 
@@ -306,35 +312,29 @@ impl ExecutionContext {
         vesting_manager: Arc<VestingManager>,
         mip_store: MipStore,
     ) -> Self {
-        // Deterministically seed the unsafe RNG to allow the bytecode to use it.
-        // Note that consecutive read-only calls for the same slot will get the same random seed.
-
-        // Add the current slot to the seed to ensure different draws at every slot
-        let mut seed: Vec<u8> = slot.to_bytes_key().to_vec();
-        // Add a marker to the seed indicating that we are in read-only mode
-        // to prevent random draw collisions with active executions
-        seed.push(0u8); // 0u8 = read-only
-        let seed = massa_hash::Hash::compute_from(&seed).into_bytes();
-        // We use Xoshiro256PlusPlus because it is very fast,
-        // has a period long enough to ensure no repetitions will ever happen,
-        // of decent quality (given the unsafe constraints)
-        // but not cryptographically secure (and that's ok because the internal state is exposed anyways)
-        let unsafe_rng = Xoshiro256PlusPlus::from_seed(seed);
+        // Get the execution hash trail
+        let prev_execution_trail_hash = active_history.read().get_execution_trail_hash();
+        let prev_execution_trail_hash = match prev_execution_trail_hash {
+            HistorySearchResult::Present(h) => h,
+            _ => final_state.read().get_execution_trail_hash(),
+        };
+        let execution_trail_hash =
+            generate_execution_trail_hash(&prev_execution_trail_hash, &slot, None, true);
 
         // return readonly context
         ExecutionContext {
             max_gas,
             slot,
             stack: call_stack,
             read_only: true,
-            unsafe_rng,
             ..ExecutionContext::new(
                 config,
                 final_state,
                 active_history,
                 module_cache,
                 vesting_manager,
                 mip_store,
+                execution_trail_hash,
             )
         }
     }
@@ -380,33 +380,31 @@ impl ExecutionContext {
         vesting_manager: Arc<VestingManager>,
         mip_store: MipStore,
     ) -> Self {
-        // Deterministically seed the unsafe RNG to allow the bytecode to use it.
-
-        // Add the current slot to the seed to ensure different draws at every slot
-        let mut seed: Vec<u8> = slot.to_bytes_key().to_vec();
-        // Add a marker to the seed indicating that we are in active mode
-        // to prevent random draw collisions with read-only executions
-        seed.push(1u8); // 1u8 = active
-
-        // For more deterministic entropy, seed with the block ID if any
-        if let Some(block_id) = &opt_block_id {
-            seed.extend(block_id.to_bytes()); // append block ID
-        }
-        let seed = massa_hash::Hash::compute_from(&seed).into_bytes();
-        let unsafe_rng = Xoshiro256PlusPlus::from_seed(seed);
+        // Get the execution hash trail
+        let prev_execution_trail_hash = active_history.read().get_execution_trail_hash();
+        let prev_execution_trail_hash = match prev_execution_trail_hash {
+            HistorySearchResult::Present(h) => h,
+            _ => final_state.read().get_execution_trail_hash(),
+        };
+        let execution_trail_hash = generate_execution_trail_hash(
+            &prev_execution_trail_hash,
+            &slot,
+            opt_block_id.as_ref(),
+            false,
+        );
 
         // return active slot execution context
         ExecutionContext {
             slot,
             opt_block_id,
-            unsafe_rng,
             ..ExecutionContext::new(
                 config,
                 final_state,
                 active_history,
                 module_cache,
                 vesting_manager,
                 mip_store,
+                execution_trail_hash,
             )
         }
     }
@@ -936,6 +934,7 @@ impl ExecutionContext {
             pos_changes: self.speculative_roll_state.take(),
             executed_ops_changes: self.speculative_executed_ops.take(),
             executed_denunciations_changes: self.speculative_executed_denunciations.take(),
+            execution_trail_hash_change: SetOrKeep::Set(self.execution_trail_hash),
         };
 
         std::mem::take(&mut self.opt_block_id);
@@ -1074,3 +1073,42 @@ impl ExecutionContext {
             .get_address_deferred_credits(address, min_slot)
     }
 }
+
+/// Generate the execution trail hash
+fn generate_execution_trail_hash(
+    previous_execution_trail_hash: &massa_hash::Hash,
+    slot: &Slot,
+    opt_block_id: Option<&BlockId>,
+    read_only: bool,
+) -> massa_hash::Hash {
+    match opt_block_id {
+        None => massa_hash::Hash::compute_from_tuple(&[
+            previous_execution_trail_hash.to_bytes(),
+            &slot.to_bytes_key(),
+            &[if read_only { 1u8 } else { 0u8 }, 0u8],
+        ]),
+        Some(block_id) => massa_hash::Hash::compute_from_tuple(&[
+            previous_execution_trail_hash.to_bytes(),
+            &slot.to_bytes_key(),
+            &[if read_only { 1u8 } else { 0u8 }, 1u8],
+            block_id.to_bytes(),
+        ]),
+    }
+}
+
+/// Initializes and seeds the PRNG with the given execution trail hash.
+fn init_prng(execution_trail_hash: &massa_hash::Hash) -> Xoshiro256PlusPlus {
+    // Deterministically seed the unsafe RNG to allow the bytecode to use it.
+    // Note that consecutive read-only calls for the same slot will get the same random seed.
+    let seed = massa_hash::Hash::compute_from_tuple(&[
+        "PRNG_SEED".as_bytes(),
+        execution_trail_hash.to_bytes(),
+    ])
+    .into_bytes();
+
+    // We use Xoshiro256PlusPlus because it is very fast,
+    // has a period long enough to ensure no repetitions will ever happen,
+    // of decent quality (given the unsafe constraints)
+    // but not cryptographically secure (and that's ok because the internal state is exposed anyways)
+    Xoshiro256PlusPlus::from_seed(seed)
+}

massa-execution-worker/src/execution.rs

@@ -118,12 +118,13 @@ impl ExecutionState {
     ) -> ExecutionState {
         // Get the slot at the output of which the final state is attached.
         // This should be among the latest final slots.
-        let last_final_slot = final_state
-            .read()
-            .db
-            .read()
-            .get_change_id()
-            .expect("Critical error: Final state has no slot attached");
+        let last_final_slot;
+        let execution_trail_hash;
+        {
+            let final_state_read = final_state.read();
+            last_final_slot = final_state_read.get_slot();
+            execution_trail_hash = final_state_read.get_execution_trail_hash();
+        }
 
         // Create default active history
         let active_history: Arc<RwLock<ActiveHistory>> = Default::default();
@@ -160,6 +161,7 @@ impl ExecutionState {
             module_cache.clone(),
             vesting_manager.clone(),
             mip_store.clone(),
+            execution_trail_hash,
         )));
 
         // Instantiate the interface providing ABI access to the VM, share the execution context with it

massa-execution-worker/src/interface_impl.rs

@@ -119,6 +119,7 @@ impl InterfaceImpl {
             module_cache,
             vesting_manager,
             mip_store,
+            massa_hash::Hash::zero(),
         );
         execution_context.stack = vec![ExecutionStackElement {
             address: sender_addr,

massa-execution-worker/src/tests/mock.rs

@@ -144,6 +144,7 @@ pub fn get_sample_state(
         )
         .unwrap()
     };
+    final_state.init_execution_trail_hash();
     let mut batch: BTreeMap<Vec<u8>, Option<Vec<u8>>> = DBBatch::new();
     final_state.pos_state.create_initial_cycle(&mut batch);
     final_state

massa-execution-worker/src/tests/tests_active_history.rs

@@ -55,6 +55,7 @@ mod tests {
                 },
                 executed_ops_changes: Default::default(),
                 executed_denunciations_changes: Default::default(),
+                execution_trail_hash_change: Default::default(),
             },
             events: Default::default(),
         };