From ed6cff261c9fd057a3313fd1b7c23d3cb97c0d5d Mon Sep 17 00:00:00 2001 From: Alon Lukatch Date: Mon, 23 Dec 2024 12:00:35 +0200 Subject: [PATCH 1/3] refactor(papyrus_storage): add an events table and a corresponding file handler --- crates/papyrus_storage/src/lib.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/crates/papyrus_storage/src/lib.rs b/crates/papyrus_storage/src/lib.rs index b8a4ebef0a..a12c94bca1 100644 --- a/crates/papyrus_storage/src/lib.rs +++ b/crates/papyrus_storage/src/lib.rs @@ -749,7 +749,7 @@ impl FileHandlers { ("deprecated_contract_class".to_string(), self.deprecated_contract_class.stats()), ("transaction_output".to_string(), self.transaction_output.stats()), ("transaction".to_string(), self.transaction.stats()), - ("events".to_string(), self.event.stats()), + ("event".to_string(), self.event.stats()), ]) } @@ -865,8 +865,8 @@ fn open_storage_files( )?; let event_offset = table.get(&db_transaction, &OffsetKind::Event)?.unwrap_or_default(); - let (events_writer, events_reader) = - open_file(mmap_file_config, db_config.path().join("events.dat"), event_offset)?; + let (event_writer, event_reader) = + open_file(mmap_file_config, db_config.path().join("event.dat"), event_offset)?; Ok(( FileHandlers { @@ -876,7 +876,7 @@ fn open_storage_files( deprecated_contract_class: deprecated_contract_class_writer, transaction_output: transaction_output_writer, transaction: transaction_writer, - event: events_writer, + event: event_writer, }, FileHandlers { thin_state_diff: thin_state_diff_reader, @@ -885,7 +885,7 @@ fn open_storage_files( deprecated_contract_class: deprecated_contract_class_reader, transaction_output: transaction_output_reader, transaction: transaction_reader, - event: events_reader, + event: event_reader, }, )) } @@ -905,7 +905,7 @@ pub enum OffsetKind { TransactionOutput, /// A transaction file. Transaction, - /// An events file. + /// An event file. Event, } From dc0aee90322252e17df357c80992b906bed6f9e8 Mon Sep 17 00:00:00 2001 From: Alon Lukatch Date: Wed, 25 Dec 2024 14:33:48 +0200 Subject: [PATCH 2/3] refactor(papyrus_storage): add append_events_function --- crates/papyrus_storage/src/body/events.rs | 66 +++++++++++++++++++++-- crates/papyrus_storage/src/body/mod.rs | 7 +++ crates/starknet_api/src/transaction.rs | 1 + 3 files changed, 71 insertions(+), 3 deletions(-) diff --git a/crates/papyrus_storage/src/body/events.rs b/crates/papyrus_storage/src/body/events.rs index c4c6e8a29a..a9a9ee4edd 100644 --- a/crates/papyrus_storage/src/body/events.rs +++ b/crates/papyrus_storage/src/body/events.rs @@ -57,15 +57,23 @@ use starknet_api::transaction::{ Event, EventContent, EventIndexInTransactionOutput, + TransactionOffsetInBlock, TransactionOutput, }; -use super::TransactionMetadataTable; +use super::{update_marker, TransactionMetadataTable}; use crate::body::{AddressToTransactionIndexTableKey, TransactionIndex}; use crate::db::serialization::{NoVersionValueWrapper, VersionZeroWrapper}; use crate::db::table_types::{CommonPrefix, DbCursor, DbCursorTrait, NoValue, SimpleTable, Table}; -use crate::db::{DbTransaction, RO}; -use crate::{FileHandlers, StorageResult, StorageTxn, TransactionMetadata}; +use crate::db::{DbTransaction, RO, RW}; +use crate::{ + FileHandlers, + OffsetKind, + StorageResult, + StorageScope, + StorageTxn, + TransactionMetadata, +}; /// An identifier of an event. #[derive(Debug, Copy, Clone, Eq, PartialEq, Deserialize, Serialize, PartialOrd, Ord)] @@ -372,3 +380,55 @@ type AddressToTransactionIndexTableCursor<'txn> = DbCursor< /// A cursor of the transaction outputs table. type TransactionMetadataTableCursor<'txn> = DbCursor<'txn, RO, TransactionIndex, VersionZeroWrapper, SimpleTable>; + +/// interface for updating the events in the storage. +pub trait EventStorageWriter +where + Self: Sized, +{ + /// Appends the events of an entire block to the storage. + fn append_events( + self, + block_number: BlockNumber, + block_events: Vec>, + ) -> StorageResult; +} + +impl EventStorageWriter for StorageTxn<'_, RW> { + fn append_events( + self, + block_number: BlockNumber, + block_events: Vec>, + ) -> StorageResult { + let markers_table = self.open_table(&self.tables.markers)?; + update_marker(&self.txn, &markers_table, block_number)?; + if self.scope != StorageScope::StateOnly { + let events_table = self.open_table(&self.tables.events)?; + let file_offset_table = self.open_table(&self.tables.file_offsets)?; + let address_to_transaction_index = + self.open_table(&self.tables.address_to_transaction_index)?; + + for (index, transaction_events) in block_events.iter().enumerate() { + let transaction_index = + TransactionIndex(block_number, TransactionOffsetInBlock(index)); + let event_offset = self.file_handlers.append_events(&transaction_events.clone()); + events_table.append(&self.txn, &transaction_index, &event_offset)?; + for even in transaction_events { + address_to_transaction_index.insert( + &self.txn, + &(even.from_address, transaction_index), + &NoValue, + )?; + } + if index == block_events.len() - 1 { + file_offset_table.upsert( + &self.txn, + &OffsetKind::Events, + &event_offset.next_offset(), + )?; + } + } + } + Ok(self) + } +} diff --git a/crates/papyrus_storage/src/body/mod.rs b/crates/papyrus_storage/src/body/mod.rs index 320193f214..0ddc1795a2 100644 --- a/crates/papyrus_storage/src/body/mod.rs +++ b/crates/papyrus_storage/src/body/mod.rs @@ -60,6 +60,7 @@ use tracing::debug; use crate::db::serialization::{NoVersionValueWrapper, VersionZeroWrapper}; use crate::db::table_types::{CommonPrefix, DbCursorTrait, NoValue, SimpleTable, Table}; use crate::db::{DbTransaction, TableHandle, TransactionKind, RW}; +use crate::mmap_file::LocationInFile; use crate::{ FileHandlers, MarkerKind, @@ -85,6 +86,12 @@ type AddressToTransactionIndexTable<'env> = TableHandle< NoVersionValueWrapper, CommonPrefix, >; +// TODO: remove the dead code attribute. +#[allow(dead_code)] +type EventsTableKey = TransactionIndex; +#[allow(dead_code)] +type EventsTable<'env> = + TableHandle<'env, EventsTableKey, VersionZeroWrapper, SimpleTable>; /// The index of a transaction in a block. #[derive(Copy, Clone, Debug, Eq, PartialEq, Deserialize, Serialize, PartialOrd, Ord)] diff --git a/crates/starknet_api/src/transaction.rs b/crates/starknet_api/src/transaction.rs index 36980ca362..3075305353 100644 --- a/crates/starknet_api/src/transaction.rs +++ b/crates/starknet_api/src/transaction.rs @@ -948,4 +948,5 @@ pub struct TransactionOffsetInBlock(pub usize); #[derive( Debug, Default, Copy, Clone, Eq, PartialEq, Hash, Deserialize, Serialize, PartialOrd, Ord, )] +// TODO: Rename to EventIndexInTransaction. pub struct EventIndexInTransactionOutput(pub usize); From 6429ca02832ec6b36f5a79f14b5147a8cea2d3ca Mon Sep 17 00:00:00 2001 From: Alon Lukatch Date: Wed, 25 Dec 2024 15:09:18 +0200 Subject: [PATCH 3/3] refactor(papyrus_storage): fix CR comments --- crates/papyrus_storage/src/body/events.rs | 68 +++++++++++++---------- crates/papyrus_storage/src/body/mod.rs | 2 +- 2 files changed, 40 insertions(+), 30 deletions(-) diff --git a/crates/papyrus_storage/src/body/events.rs b/crates/papyrus_storage/src/body/events.rs index a9a9ee4edd..e5de90c18e 100644 --- a/crates/papyrus_storage/src/body/events.rs +++ b/crates/papyrus_storage/src/body/events.rs @@ -48,7 +48,7 @@ #[path = "events_test.rs"] mod events_test; -use std::collections::VecDeque; +use std::collections::{HashSet, VecDeque}; use serde::{Deserialize, Serialize}; use starknet_api::block::BlockNumber; @@ -381,16 +381,17 @@ type AddressToTransactionIndexTableCursor<'txn> = DbCursor< type TransactionMetadataTableCursor<'txn> = DbCursor<'txn, RO, TransactionIndex, VersionZeroWrapper, SimpleTable>; -/// interface for updating the events in the storage. +/// Interface for updating the events in the storage. pub trait EventStorageWriter where Self: Sized, { /// Appends the events of an entire block to the storage. + // To enforce that no commit happen after a failure, we consume and return Self on success. fn append_events( self, block_number: BlockNumber, - block_events: Vec>, + block_events: &[&Vec], ) -> StorageResult; } @@ -398,35 +399,44 @@ impl EventStorageWriter for StorageTxn<'_, RW> { fn append_events( self, block_number: BlockNumber, - block_events: Vec>, + block_events: &[&Vec], ) -> StorageResult { let markers_table = self.open_table(&self.tables.markers)?; update_marker(&self.txn, &markers_table, block_number)?; - if self.scope != StorageScope::StateOnly { - let events_table = self.open_table(&self.tables.events)?; - let file_offset_table = self.open_table(&self.tables.file_offsets)?; - let address_to_transaction_index = - self.open_table(&self.tables.address_to_transaction_index)?; - - for (index, transaction_events) in block_events.iter().enumerate() { - let transaction_index = - TransactionIndex(block_number, TransactionOffsetInBlock(index)); - let event_offset = self.file_handlers.append_events(&transaction_events.clone()); - events_table.append(&self.txn, &transaction_index, &event_offset)?; - for even in transaction_events { - address_to_transaction_index.insert( - &self.txn, - &(even.from_address, transaction_index), - &NoValue, - )?; - } - if index == block_events.len() - 1 { - file_offset_table.upsert( - &self.txn, - &OffsetKind::Events, - &event_offset.next_offset(), - )?; - } + if self.scope == StorageScope::StateOnly { + return Ok(self); + } + + let events_table = self.open_table(&self.tables.events)?; + let file_offset_table = self.open_table(&self.tables.file_offsets)?; + let address_to_transaction_index = + self.open_table(&self.tables.address_to_transaction_index)?; + + for (index, &transaction_events) in block_events.iter().enumerate() { + let transaction_index = TransactionIndex(block_number, TransactionOffsetInBlock(index)); + let event_location = self.file_handlers.append_events(transaction_events); + events_table.append(&self.txn, &transaction_index, &event_location)?; + + let mut contract_address_set = HashSet::new(); + + for event in transaction_events { + contract_address_set.insert(event.from_address); + } + + for contract_address in contract_address_set { + address_to_transaction_index.insert( + &self.txn, + &(contract_address, transaction_index), + &NoValue, + )?; + } + + if index == block_events.len() - 1 { + file_offset_table.upsert( + &self.txn, + &OffsetKind::Event, + &event_location.next_offset(), + )?; } } Ok(self) diff --git a/crates/papyrus_storage/src/body/mod.rs b/crates/papyrus_storage/src/body/mod.rs index 0ddc1795a2..b496d53365 100644 --- a/crates/papyrus_storage/src/body/mod.rs +++ b/crates/papyrus_storage/src/body/mod.rs @@ -86,7 +86,7 @@ type AddressToTransactionIndexTable<'env> = TableHandle< NoVersionValueWrapper, CommonPrefix, >; -// TODO: remove the dead code attribute. +// TODO(alonl): remove the dead code attribute. #[allow(dead_code)] type EventsTableKey = TransactionIndex; #[allow(dead_code)]