From acfe58ae9f117b45f07d3472a8ec1a4cc5e1c442 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 07:35:04 +0200 Subject: [PATCH 01/22] mark: 0xaatif/no-trace-decoder From 7c8cf14136bf8cf0b9071b659c0c317d152cacf7 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 08:33:32 +0200 Subject: [PATCH 02/22] refactor: remove trace_decoder crate --- Cargo.lock | 83 ++------- Cargo.toml | 2 - trace_decoder/Cargo.toml | 81 --------- trace_decoder/src/lib.rs | 112 ------------- trace_decoder/src/observer.rs | 113 ------------- zero/Cargo.toml | 43 ++++- .../benches/case.json | 0 .../benches/intra-block-tries.rs | 10 +- zero/src/bin/rpc.rs | 4 +- zero/src/bin/trie_diff.rs | 4 +- .../core.rs => zero/src/intra_block_tries.rs | 158 +++++++++++++++++- zero/src/lib.rs | 31 ++++ zero/src/prover.rs | 8 +- zero/src/rpc/jerigon.rs | 2 +- zero/src/rpc/mod.rs | 2 +- zero/src/rpc/native/mod.rs | 2 +- zero/src/rpc/native/state.rs | 6 +- zero/src/rpc/native/txn.rs | 2 +- .../interface.rs => zero/src/trace_decoder.rs | 4 +- {trace_decoder => zero}/src/typed_mpt.rs | 0 .../src/wire_tries}/cases/.gitattributes | 0 .../src/wire_tries}/cases/README.md | 0 .../wire_tries}/cases/hermez_cdk_erigon.json | 0 .../src/wire_tries}/cases/zero_jerigon.json | 0 .../src/wire.rs => zero/src/wire_tries/mod.rs | 22 +++ .../src => zero/src/wire_tries}/type1.rs | 4 +- .../src => zero/src/wire_tries}/type2.rs | 4 +- .../tests/cases/.gitattributes | 0 .../tests/cases/b19807080_main.json | 0 .../tests/cases/b19807080_main_header.json | 0 .../tests/cases/b19840104_main.json | 0 .../tests/cases/b19840104_main_header.json | 0 .../tests/cases/b20240052_main.json | 0 .../tests/cases/b20240052_main_header.json | 0 .../tests/cases/b20240058_main.json | 0 .../tests/cases/b20240058_main_header.json | 0 .../tests/cases/b20472570_main.json | 0 .../tests/cases/b20472570_main_header.json | 0 .../tests/cases/b28_dev.json | 0 .../tests/cases/b28_dev_header.json | 0 .../tests/cases/b4_dev.json | 0 .../tests/cases/b4_dev_header.json | 0 {trace_decoder => zero}/tests/common/mod.rs | 2 +- .../tests/consistent-with-header.rs | 4 +- .../tests/simulate-execution.rs | 16 +- 45 files changed, 297 insertions(+), 422 deletions(-) delete mode 100644 trace_decoder/Cargo.toml delete mode 100644 trace_decoder/src/lib.rs delete mode 100644 trace_decoder/src/observer.rs rename trace_decoder/benches/block_input.json => zero/benches/case.json (100%) rename trace_decoder/benches/block_processing.rs => zero/benches/intra-block-tries.rs (84%) rename trace_decoder/src/core.rs => zero/src/intra_block_tries.rs (84%) rename trace_decoder/src/interface.rs => zero/src/trace_decoder.rs (97%) rename {trace_decoder => zero}/src/typed_mpt.rs (100%) rename {trace_decoder/src => zero/src/wire_tries}/cases/.gitattributes (100%) rename {trace_decoder/src => zero/src/wire_tries}/cases/README.md (100%) rename {trace_decoder/src => zero/src/wire_tries}/cases/hermez_cdk_erigon.json (100%) rename {trace_decoder/src => zero/src/wire_tries}/cases/zero_jerigon.json (100%) rename trace_decoder/src/wire.rs => zero/src/wire_tries/mod.rs (94%) rename {trace_decoder/src => zero/src/wire_tries}/type1.rs (99%) rename {trace_decoder/src => zero/src/wire_tries}/type2.rs (98%) rename {trace_decoder => zero}/tests/cases/.gitattributes (100%) rename {trace_decoder => zero}/tests/cases/b19807080_main.json (100%) rename {trace_decoder => zero}/tests/cases/b19807080_main_header.json (100%) rename {trace_decoder => zero}/tests/cases/b19840104_main.json (100%) rename {trace_decoder => zero}/tests/cases/b19840104_main_header.json (100%) rename {trace_decoder => zero}/tests/cases/b20240052_main.json (100%) rename {trace_decoder => zero}/tests/cases/b20240052_main_header.json (100%) rename {trace_decoder => zero}/tests/cases/b20240058_main.json (100%) rename {trace_decoder => zero}/tests/cases/b20240058_main_header.json (100%) rename {trace_decoder => zero}/tests/cases/b20472570_main.json (100%) rename {trace_decoder => zero}/tests/cases/b20472570_main_header.json (100%) rename {trace_decoder => zero}/tests/cases/b28_dev.json (100%) rename {trace_decoder => zero}/tests/cases/b28_dev_header.json (100%) rename {trace_decoder => zero}/tests/cases/b4_dev.json (100%) rename {trace_decoder => zero}/tests/cases/b4_dev_header.json (100%) rename {trace_decoder => zero}/tests/common/mod.rs (98%) rename {trace_decoder => zero}/tests/consistent-with-header.rs (96%) rename {trace_decoder => zero}/tests/simulate-execution.rs (76%) diff --git a/Cargo.lock b/Cargo.lock index 03223ee5f..09eef0a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -757,7 +757,7 @@ dependencies = [ "assert2-macros", "diff", "is-terminal", - "yansi 1.0.1", + "yansi", ] [[package]] @@ -3770,16 +3770,6 @@ dependencies = [ "zerocopy", ] -[[package]] -name = "pretty_assertions" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7cee1a6c8a5b9208b3cb1061f10c0cb689087b3d8ce85fb9d2dd7a29b6ba66" -dependencies = [ - "diff", - "yansi 0.5.1", -] - [[package]] name = "pretty_env_logger" version = "0.5.0" @@ -5068,54 +5058,6 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" -[[package]] -name = "trace_decoder" -version = "0.6.0" -dependencies = [ - "alloy", - "alloy-compat", - "anyhow", - "assert2", - "bitflags 2.6.0", - "bitvec", - "bytes", - "camino", - "ciborium", - "ciborium-io", - "clap", - "copyvec", - "criterion", - "either", - "enum-as-inner", - "ethereum-types", - "evm_arithmetization", - "glob", - "hex", - "hex-literal", - "itertools 0.13.0", - "keccak-hash 0.10.0", - "libtest-mimic", - "log", - "mpt_trie", - "nunny", - "plonky2", - "plonky2_maybe_rayon 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", - "pretty_assertions", - "pretty_env_logger", - "rlp", - "serde", - "serde_json", - "serde_path_to_error", - "smt_trie", - "stackstack", - "strum", - "thiserror", - "u4", - "winnow", - "zero", - "zk_evm_common", -] - [[package]] name = "tracing" version = "0.1.40" @@ -5749,12 +5691,6 @@ dependencies = [ "time", ] -[[package]] -name = "yansi" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09041cd90cf85f7f8b2df60c646f853b7f535ce68f85244eb6731cf89fa498ec" - [[package]] name = "yansi" version = "1.0.1" @@ -5768,23 +5704,35 @@ dependencies = [ "alloy", "alloy-compat", "anyhow", + "assert2", "async-stream", "axum", + "bitflags 2.6.0", + "bitvec", + "camino", "cargo_metadata", + "ciborium", "clap", "compat", + "copyvec", + "criterion", "directories", "dotenvy", + "either", + "ethereum-types", "evm_arithmetization", "futures", + "glob", "hashbrown", "hex", "itertools 0.13.0", "jemallocator", "keccak-hash 0.10.0", + "libtest-mimic", "lru", "mpt_trie", "num-traits", + "nunny", "once_cell", "paladin-core", "plonky2", @@ -5796,15 +5744,18 @@ dependencies = [ "serde", "serde_json", "serde_path_to_error", + "smt_trie", + "stackstack", "thiserror", "tokio", "toml", "tower 0.4.13", - "trace_decoder", "tracing", "tracing-subscriber", + "u4", "url", "vergen-git2", + "winnow", "zk_evm_common", ] diff --git a/Cargo.toml b/Cargo.toml index 9cbc60fd4..0418d1c4f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,7 +6,6 @@ members = [ "mpt_trie", "proc_macro", "smt_trie", - "trace_decoder", "zero", ] resolver = "2" @@ -104,7 +103,6 @@ winnow = "0.6.13" evm_arithmetization = { path = "evm_arithmetization", version = "0.4.0", default-features = false } mpt_trie = { path = "mpt_trie", version = "0.4.1" } smt_trie = { path = "smt_trie", version = "0.1.1" } -trace_decoder = { path = "trace_decoder", version = "0.6.0", default-features = false } zk_evm_common = { path = "common", version = "0.1.0" } zk_evm_proc_macro = { path = "proc_macro", version = "0.1.0" } zero = { path = "zero", default-features = false } diff --git a/trace_decoder/Cargo.toml b/trace_decoder/Cargo.toml deleted file mode 100644 index 8f6a63159..000000000 --- a/trace_decoder/Cargo.toml +++ /dev/null @@ -1,81 +0,0 @@ -[package] -name = "trace_decoder" -description = "Ethereum node witness -> Prover input" -authors = ["Polygon Zero"] -version = "0.6.0" -edition.workspace = true -license.workspace = true -repository.workspace = true -homepage.workspace = true -keywords.workspace = true - -[dependencies] -alloy = { workspace = true } -alloy-compat = "0.1.0" -anyhow = { workspace = true } -bitflags = { workspace = true } -bitvec = { workspace = true } -bytes = { workspace = true } -ciborium = { workspace = true } -ciborium-io = { workspace = true } -copyvec = "0.2.0" -either = { workspace = true } -enum-as-inner = { workspace = true } -ethereum-types = { workspace = true } -hex = { workspace = true } -hex-literal = { workspace = true } -itertools = { workspace = true } -keccak-hash = { workspace = true } -log = { workspace = true } -nunny = { workspace = true, features = ["serde"] } -plonky2 = { workspace = true } -rlp = { workspace = true } -serde = { workspace = true } -stackstack = "0.3.0" -strum = { version = "0.26.3", features = ["derive"] } -thiserror = { workspace = true } -u4 = { workspace = true } -winnow = { workspace = true } - -# Local dependencies -evm_arithmetization = { workspace = true } -mpt_trie = { workspace = true } -smt_trie = { workspace = true } -zk_evm_common = { workspace = true } - -[dev-dependencies] -alloy = { workspace = true } -alloy-compat = "0.1.0" -assert2 = "0.3.15" -camino = "1.1.9" -clap = { workspace = true } -criterion = { workspace = true } -glob = "0.3.1" -libtest-mimic = "0.7.3" -plonky2_maybe_rayon = { workspace = true } -pretty_assertions = "1.4.0" -zero = { workspace = true } -pretty_env_logger = { workspace = true } -serde_json = { workspace = true } -serde_path_to_error = { workspace = true } - -[features] -default = ["eth_mainnet"] -eth_mainnet = ["evm_arithmetization/eth_mainnet", "zero/eth_mainnet"] -cdk_erigon = ["evm_arithmetization/cdk_erigon", "zero/cdk_erigon"] -polygon_pos = ["evm_arithmetization/polygon_pos", "zero/polygon_pos"] - -[[bench]] -name = "block_processing" -harness = false -required-features = ["eth_mainnet"] - -[[test]] -name = "consistent-with-header" -harness = false -required-features = ["eth_mainnet"] - -[[test]] -name = "simulate-execution" -harness = false -required-features = ["eth_mainnet"] diff --git a/trace_decoder/src/lib.rs b/trace_decoder/src/lib.rs deleted file mode 100644 index 049472c40..000000000 --- a/trace_decoder/src/lib.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! An _Ethereum Node_ executes _transactions_ in _blocks_. -//! -//! Execution mutates two key data structures: -//! - [The state trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#state-trie). -//! - [The storage tries](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#storage-trie). -//! -//! Ethereum nodes expose information about the transactions over RPC, e.g: -//! - [The specific changes to the storage tries](TxnTrace::storage_written). -//! - [Changes to account balance in the state trie](TxnTrace::balance). -//! -//! The state execution correctness is then asserted by the zkEVM prover in -//! [`evm_arithmetization`], relying on `starky` and [`plonky2`]. -//! -//! **Prover perfomance is a high priority.** -//! -//! The aformentioned trie structures may have subtries _hashed out_. -//! That is, any node (and its children!) may be replaced by its hash, -//! while maintaining provability of its contents: -//! -//! ```text -//! A A -//! / \ / \ -//! B C -> H C -//! / \ \ \ -//! D E F F -//! ``` -//! (where `H` is the hash of the `D/B\E` subtrie). -//! -//! The principle concern of this module is to step through the transactions, -//! and reproduce the _intermediate tries_, -//! while hashing out all possible subtries to minimise prover load -//! (since prover performance is sensitive to the size of the trie). -//! The prover can therefore prove each batch of transactions independently. -//! -//! # Non-goals -//! - Performance - this will never be the bottleneck in any proving stack. -//! - Robustness - this library depends on other libraries that are not robust, -//! so may panic at any time. - -#![deny(rustdoc::broken_intra_doc_links)] -#![warn(missing_debug_implementations)] -#![warn(missing_docs)] - -/// Over RPC, ethereum nodes expose their tries as a series of binary -/// [`wire::Instruction`]s in a node-dependant format. -/// -/// These are parsed into the relevant trie depending on the node: -/// - [`type2`], which contains an [`smt_trie`]. -/// - [`type1`], which contains an [`mpt_trie`]. -/// -/// After getting the tries, -/// we can continue to do the main work of "executing" the transactions. -const _DEVELOPER_DOCS: () = (); - -mod interface; - -pub use interface::*; - -mod type1; -// TODO(0xaatif): https://github.com/0xPolygonZero/zk_evm/issues/275 -// add backend/prod support for type 2 -#[cfg(test)] -#[allow(dead_code)] -mod type2; -mod typed_mpt; -mod wire; - -pub use core::entrypoint; - -mod core; - -/// Implementation of the observer for the trace decoder. -pub mod observer; -/// Like `#[serde(with = "hex")`, but tolerates and emits leading `0x` prefixes -mod hex { - use serde::{de::Error as _, Deserialize as _, Deserializer, Serializer}; - - pub fn serialize(data: T, serializer: S) -> Result - where - T: hex::ToHex, - { - let s = data.encode_hex::(); - serializer.serialize_str(&format!("0x{}", s)) - } - - pub fn deserialize<'de, D: Deserializer<'de>, T>(deserializer: D) -> Result - where - T: hex::FromHex, - T::Error: std::fmt::Display, - { - let s = String::deserialize(deserializer)?; - match s.strip_prefix("0x") { - Some(rest) => T::from_hex(rest), - None => T::from_hex(&*s), - } - .map_err(D::Error::custom) - } -} - -#[cfg(test)] -#[derive(serde::Deserialize)] -struct Case { - #[serde(with = "hex")] - pub bytes: Vec, - #[serde(deserialize_with = "h256")] - pub expected_state_root: ethereum_types::H256, -} - -#[cfg(test)] -fn h256<'de, D: serde::Deserializer<'de>>(it: D) -> Result { - Ok(ethereum_types::H256(hex::deserialize(it)?)) -} diff --git a/trace_decoder/src/observer.rs b/trace_decoder/src/observer.rs deleted file mode 100644 index 320019e55..000000000 --- a/trace_decoder/src/observer.rs +++ /dev/null @@ -1,113 +0,0 @@ -use std::collections::BTreeMap; -use std::marker::PhantomData; - -use ethereum_types::{H256, U256}; - -use crate::core::IntraBlockTries; -use crate::typed_mpt::{ReceiptTrie, StorageTrie, TransactionTrie}; - -/// Observer API for the trace decoder. -/// Observer is used to collect various debugging and metadata info -/// from the trace decoder run. -pub trait Observer { - /// Collect tries after the transaction/batch execution. - /// - /// Passing the arguments one by one through reference, because - /// we don't want to clone argument tries in case they are not used in - /// observer. - fn collect_tries( - &mut self, - block: U256, - batch: usize, - state_trie: &StateTrieT, - storage: &BTreeMap, - transaction_trie: &TransactionTrie, - receipt_trie: &ReceiptTrie, - ); -} - -#[derive(Debug)] -/// Tries observer collected data element - contains -/// the data collected during the trace decoder processing of the batches in a -/// block, one element is retrieved after every batch. -pub struct TriesObserverElement { - /// Block where the tries are collected. - pub block: U256, - /// Tries were collected after trace decoder processes batch number `batch`. - pub batch: usize, - /// State, transaction, and receipt tries after the batch - /// execution (how the trace decoder sees them). - pub tries: IntraBlockTries, -} - -/// Observer for collection of post-execution tries from the -/// trace decoder run. -#[derive(Debug)] -pub struct TriesObserver { - /// Collected data in the observer pass - pub data: Vec>, -} - -impl TriesObserver { - /// Create new tries collecting observer. - pub fn new() -> Self { - TriesObserver:: { data: Vec::new() } - } -} - -impl Observer for TriesObserver { - fn collect_tries( - &mut self, - block: U256, - batch: usize, - state_trie: &StateTrieT, - storage: &BTreeMap, - transaction_trie: &TransactionTrie, - receipt_trie: &ReceiptTrie, - ) { - self.data.push(TriesObserverElement { - block, - batch, - tries: IntraBlockTries { - state: state_trie.clone(), - storage: storage.clone(), - transaction: transaction_trie.clone(), - receipt: receipt_trie.clone(), - }, - }); - } -} - -impl Default for TriesObserver { - fn default() -> Self { - Self::new() - } -} - -/// Dummy observer which does not collect any data. -#[derive(Default, Debug)] -pub struct DummyObserver { - phantom: PhantomData, -} - -impl DummyObserver { - /// Create a new dummy observer. - pub fn new() -> Self { - DummyObserver:: { - phantom: Default::default(), - } - } -} - -impl Observer for DummyObserver { - fn collect_tries( - &mut self, - _block: U256, - _batch: usize, - _state_trie: &StateTrieT, - _storage: &BTreeMap, - _transaction_trie: &TransactionTrie, - _receipt_trie: &ReceiptTrie, - ) { - } -} diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 5ccb57c96..9332d7480 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -15,10 +15,16 @@ alloy-compat = "0.1.0" anyhow = { workspace = true } async-stream = { workspace = true } axum = { workspace = true } +bitflags = { workspace = true } +bitvec = { workspace = true } cargo_metadata = { workspace = true } +ciborium = { workspace = true } clap = { workspace = true, features = ["derive", "string"] } +copyvec = "0.2.1" directories = "5.0.1" dotenvy = { workspace = true } +either = { workspace = true } +ethereum-types = { workspace = true } futures = { workspace = true } hashbrown = { workspace = true } hex = { workspace = true } @@ -26,31 +32,36 @@ itertools = { workspace = true } keccak-hash = { workspace = true } lru = { workspace = true } num-traits = { workspace = true } +nunny = { workspace = true } once_cell = { workspace = true } paladin-core = { workspace = true } plonky2 = { workspace = true } plonky2_maybe_rayon = { workspace = true } regex = "1.5.4" -rlp = {workspace = true} +rlp = { workspace = true } ruint = { workspace = true, features = ["num-traits", "primitive-types"] } serde = { workspace = true } serde_json = { workspace = true } serde_path_to_error = { workspace = true } +stackstack = "0.3.0" thiserror = { workspace = true } tokio = { workspace = true } toml = { workspace = true } tower = { workspace = true, features = ["retry"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } +u4 = { workspace = true } url = { workspace = true } +winnow = { workspace = true } # Local dependencies compat = { workspace = true } evm_arithmetization = { workspace = true } mpt_trie = { workspace = true } -trace_decoder = { workspace = true } +smt_trie = { workspace = true } zk_evm_common = { workspace = true } + [target.'cfg(not(target_env = "msvc"))'.dependencies] jemallocator = "0.5.4" @@ -63,9 +74,31 @@ vergen-git2 = { version = "1.0.0", features = ["build"] } [features] default = ["eth_mainnet"] -eth_mainnet = ["evm_arithmetization/eth_mainnet", "trace_decoder/eth_mainnet"] -cdk_erigon = ["evm_arithmetization/cdk_erigon", "trace_decoder/cdk_erigon"] -polygon_pos = ["evm_arithmetization/polygon_pos", "trace_decoder/polygon_pos"] +eth_mainnet = ["evm_arithmetization/eth_mainnet"] +cdk_erigon = ["evm_arithmetization/cdk_erigon"] +polygon_pos = ["evm_arithmetization/polygon_pos"] [lints] workspace = true + +[[bench]] +name = "intra-block-tries" +harness = false +required-features = ["eth_mainnet"] + +[[test]] +name = "consistent-with-header" +harness = false +required-features = ["eth_mainnet"] + +[[test]] +name = "simulate-execution" +harness = false +required-features = ["eth_mainnet"] + +[dev-dependencies] +assert2 = "0.3.15" +camino = "1.1.9" +criterion.workspace = true +glob = "0.3.1" +libtest-mimic = "0.7.3" diff --git a/trace_decoder/benches/block_input.json b/zero/benches/case.json similarity index 100% rename from trace_decoder/benches/block_input.json rename to zero/benches/case.json diff --git a/trace_decoder/benches/block_processing.rs b/zero/benches/intra-block-tries.rs similarity index 84% rename from trace_decoder/benches/block_processing.rs rename to zero/benches/intra-block-tries.rs index adefdae3f..154d39203 100644 --- a/trace_decoder/benches/block_processing.rs +++ b/zero/benches/intra-block-tries.rs @@ -6,8 +6,8 @@ //! for a total of 24,479,837 gas. use criterion::{criterion_group, criterion_main, BatchSize, Criterion}; -use trace_decoder::observer::DummyObserver; -use trace_decoder::{BlockTrace, OtherBlockData}; +use zero::intra_block_tries::DummyObserver; +use zero::trace_decoder::{BlockTrace, OtherBlockData}; #[derive(Clone, Debug, serde::Deserialize)] pub struct ProverInput { @@ -16,9 +16,7 @@ pub struct ProverInput { } fn criterion_benchmark(c: &mut Criterion) { - let prover_input = - serde_json::from_slice::(include_bytes!("block_input.json").as_slice()) - .unwrap(); + let prover_input = serde_json::from_str::(include_str!("case.json")).unwrap(); let batch_sizes = vec![1, 2, 4, 8]; @@ -34,7 +32,7 @@ fn criterion_benchmark(c: &mut Criterion) { block_trace, other_data, }| { - trace_decoder::entrypoint( + zero::intra_block_tries::entrypoint( block_trace, other_data, batch_size, diff --git a/zero/src/bin/rpc.rs b/zero/src/bin/rpc.rs index d49cdde5c..e4e481bb5 100644 --- a/zero/src/bin/rpc.rs +++ b/zero/src/bin/rpc.rs @@ -8,11 +8,11 @@ use alloy::transports::Transport; use anyhow::anyhow; use clap::{Args, Parser, Subcommand, ValueHint}; use futures::StreamExt; -use trace_decoder::observer::DummyObserver; use tracing_subscriber::{prelude::*, EnvFilter}; use url::Url; use zero::block_interval::BlockInterval; use zero::block_interval::BlockIntervalStream; +use zero::intra_block_tries::DummyObserver; use zero::prover::BlockProverInput; use zero::provider::CachedProvider; use zero::rpc; @@ -167,7 +167,7 @@ impl Cli { block_number ))?; - let generation_inputs = trace_decoder::entrypoint( + let generation_inputs = zero::intra_block_tries::entrypoint( block_prover_input.block_trace, block_prover_input.other_data, batch_size, diff --git a/zero/src/bin/trie_diff.rs b/zero/src/bin/trie_diff.rs index 312feceb1..9c04776eb 100644 --- a/zero/src/bin/trie_diff.rs +++ b/zero/src/bin/trie_diff.rs @@ -23,8 +23,8 @@ use futures::{future, TryStreamExt}; use paladin::directive::{Directive, IndexedStream}; use paladin::runtime::Runtime; use regex::Regex; -use trace_decoder::observer::TriesObserver; use tracing::{error, info}; +use zero::intra_block_tries::TriesObserver; use zero::ops::register; use zero::prover::{cli::CliProverConfig, BlockProverInput, ProverConfig}; @@ -84,7 +84,7 @@ async fn main() -> Result<()> { .b_meta .block_number .low_u64(); - let block_generation_inputs = trace_decoder::entrypoint( + let block_generation_inputs = zero::intra_block_tries::entrypoint( block_prover_input.block_trace.clone(), block_prover_input.other_data.clone(), prover_config.batch_size, diff --git a/trace_decoder/src/core.rs b/zero/src/intra_block_tries.rs similarity index 84% rename from trace_decoder/src/core.rs rename to zero/src/intra_block_tries.rs index d96778dec..3002f6f12 100644 --- a/trace_decoder/src/core.rs +++ b/zero/src/intra_block_tries.rs @@ -1,3 +1,42 @@ +//! An _Ethereum Node_ executes _transactions_ in _blocks_. +//! +//! Execution mutates two key data structures: +//! - [The state trie](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#state-trie). +//! - [The storage tries](https://ethereum.org/en/developers/docs/data-structures-and-encoding/patricia-merkle-trie/#storage-trie). +//! +//! Ethereum nodes expose information about the transactions over RPC, e.g: +//! - [The specific changes to the storage tries](TxnTrace::storage_written). +//! - [Changes to account balance in the state trie](TxnTrace::balance). +//! +//! The state execution correctness is then asserted by the zkEVM prover in +//! [`evm_arithmetization`], relying on `starky` and [`plonky2`]. +//! +//! **Prover perfomance is a high priority.** +//! +//! The aformentioned trie structures may have subtries _hashed out_. +//! That is, any node (and its children!) may be replaced by its hash, +//! while maintaining provability of its contents: +//! +//! ```text +//! A A +//! / \ / \ +//! B C -> H C +//! / \ \ \ +//! D E F F +//! ``` +//! (where `H` is the hash of the `D/B\E` subtrie). +//! +//! The principle concern of this module is to step through the transactions, +//! and reproduce the _intermediate tries_, +//! while hashing out all possible subtries to minimise prover load +//! (since prover performance is sensitive to the size of the trie). +//! The prover can therefore prove each batch of transactions independently. +//! +//! # Non-goals +//! - Performance - this will never be the bottleneck in any proving stack. +//! - Robustness - this library depends on other libraries that are not robust, +//! so may panic at any time. + use std::{ cmp, collections::{BTreeMap, BTreeSet, HashMap}, @@ -19,12 +58,13 @@ use mpt_trie::partial_trie::PartialTrie as _; use nunny::NonEmpty; use zk_evm_common::gwei_to_wei; -use crate::observer::Observer; use crate::{ + trace_decoder::{ + BlockLevelData, BlockTrace, BlockTraceTriePreImages, CombinedPreImages, ContractCodeUsage, + OtherBlockData, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, + TxnInfo, TxnMeta, TxnTrace, + }, typed_mpt::{ReceiptTrie, StateMpt, StateTrie, StorageTrie, TransactionTrie, TrieKey}, - BlockLevelData, BlockTrace, BlockTraceTriePreImages, CombinedPreImages, ContractCodeUsage, - OtherBlockData, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, - TxnInfo, TxnMeta, TxnTrace, }; /// TODO(0xaatif): document this after https://github.com/0xPolygonZero/zk_evm/issues/275 @@ -180,13 +220,13 @@ fn start( (state, storage, Hash2Code::new()) } BlockTraceTriePreImages::Combined(CombinedPreImages { compact }) => { - let instructions = crate::wire::parse(&compact) + let instructions = crate::wire_tries::parse(&compact) .context("couldn't parse instructions from binary format")?; - let crate::type1::Frontend { + let crate::wire_tries::type1::Frontend { state, storage, code, - } = crate::type1::frontend(instructions)?; + } = crate::wire_tries::type1::frontend(instructions)?; (state, storage, code.into_iter().map(Into::into).collect()) } }) @@ -807,3 +847,107 @@ impl FromIterator> for Hash2Code { this } } + +/// Collects various debugging and metadata info through [`middle`]. +pub trait Observer { + /// Collect tries after the transaction/batch execution. + /// + /// Passing the arguments one by one through reference, because + /// we don't want to clone argument tries in case they are not used in + /// observer. + fn collect_tries( + &mut self, + block: U256, + batch: usize, + state_trie: &StateTrieT, + storage: &BTreeMap, + transaction_trie: &TransactionTrie, + receipt_trie: &ReceiptTrie, + ); +} + +#[derive(Debug)] +/// Tries observer collected data element - contains +/// the data collected during the trace decoder processing of the batches in a +/// block, one element is retrieved after every batch. +pub struct TriesObserverElement { + /// Block where the tries are collected. + pub block: U256, + /// Tries were collected after trace decoder processes batch number `batch`. + pub batch: usize, + /// State, transaction, and receipt tries after the batch + /// execution (how the trace decoder sees them). + pub tries: IntraBlockTries, +} + +/// Observer for collection of post-execution tries from the +/// trace decoder run. +#[derive(Debug)] +pub struct TriesObserver { + /// Collected data in the observer pass + pub data: Vec>, +} + +impl TriesObserver { + /// Create new tries collecting observer. + pub fn new() -> Self { + TriesObserver:: { data: Vec::new() } + } +} + +impl Observer for TriesObserver { + fn collect_tries( + &mut self, + block: U256, + batch: usize, + state_trie: &StateTrieT, + storage: &BTreeMap, + transaction_trie: &TransactionTrie, + receipt_trie: &ReceiptTrie, + ) { + self.data.push(TriesObserverElement { + block, + batch, + tries: IntraBlockTries { + state: state_trie.clone(), + storage: storage.clone(), + transaction: transaction_trie.clone(), + receipt: receipt_trie.clone(), + }, + }); + } +} + +impl Default for TriesObserver { + fn default() -> Self { + Self::new() + } +} + +/// Dummy observer which does not collect any data. +#[derive(Default, Debug)] +pub struct DummyObserver { + phantom: std::marker::PhantomData, +} + +impl DummyObserver { + /// Create a new dummy observer. + pub fn new() -> Self { + DummyObserver:: { + phantom: Default::default(), + } + } +} + +impl Observer for DummyObserver { + fn collect_tries( + &mut self, + _block: U256, + _batch: usize, + _state_trie: &StateTrieT, + _storage: &BTreeMap, + _transaction_trie: &TransactionTrie, + _receipt_trie: &ReceiptTrie, + ) { + } +} diff --git a/zero/src/lib.rs b/zero/src/lib.rs index c2ca63f6a..02e599d8f 100644 --- a/zero/src/lib.rs +++ b/zero/src/lib.rs @@ -4,6 +4,7 @@ pub mod block_interval; pub mod debug_utils; pub mod env; pub mod fs; +pub mod intra_block_tries; pub mod ops; pub mod parsing; pub mod pre_checks; @@ -12,8 +13,11 @@ pub mod prover; pub mod prover_state; pub mod provider; pub mod rpc; +pub mod trace_decoder; pub mod tracing; pub mod trie_diff; +pub mod typed_mpt; +pub mod wire_tries; /// Size of the channel used to send block prover inputs to the per block /// proving task. If the proving task is slow and can not consume inputs fast @@ -29,3 +33,30 @@ pub fn version() -> String { let kernel_hash = &**prover_state::persistence::KERNEL_HASH; format!("{pkg_name} ({git_describe}) (kernel hash: {kernel_hash}) [built: {timestamp}]") } + +/// Like `#[serde(with = "::hex")]`, but tolerates and emits leading `0x` +/// prefixes +mod hex { + use serde::{de::Error as _, Deserialize as _, Deserializer, Serializer}; + + pub fn serialize(data: T, serializer: S) -> Result + where + T: hex::ToHex, + { + let s = data.encode_hex::(); + serializer.serialize_str(&format!("0x{}", s)) + } + + pub fn deserialize<'de, D: Deserializer<'de>, T>(deserializer: D) -> Result + where + T: hex::FromHex, + T::Error: std::fmt::Display, + { + let s = String::deserialize(deserializer)?; + match s.strip_prefix("0x") { + Some(rest) => T::from_hex(rest), + None => T::from_hex(&*s), + } + .map_err(D::Error::custom) + } +} diff --git a/zero/src/prover.rs b/zero/src/prover.rs index 665d4f828..d4656fe1b 100644 --- a/zero/src/prover.rs +++ b/zero/src/prover.rs @@ -20,13 +20,13 @@ use serde::{Deserialize, Serialize}; use tokio::io::AsyncWriteExt; use tokio::sync::mpsc::Receiver; use tokio::sync::{oneshot, Semaphore}; -use trace_decoder::observer::DummyObserver; -use trace_decoder::{BlockTrace, OtherBlockData}; use tracing::{error, info}; use crate::fs::generate_block_proof_file_name; +use crate::intra_block_tries::DummyObserver; use crate::ops; use crate::proof_types::GeneratedBlockProof; +use crate::trace_decoder::{BlockTrace, OtherBlockData}; // All proving tasks are executed concurrently, which can cause issues for large // block intervals, where distant future blocks may be proven first. @@ -83,7 +83,7 @@ impl BlockProverInput { let block_number = self.get_block_number(); - let block_generation_inputs = trace_decoder::entrypoint( + let block_generation_inputs = crate::intra_block_tries::entrypoint( self.block_trace, self.other_data, batch_size, @@ -176,7 +176,7 @@ impl BlockProverInput { let block_number = self.get_block_number(); info!("Testing witness generation for block {block_number}."); - let block_generation_inputs = trace_decoder::entrypoint( + let block_generation_inputs = crate::intra_block_tries::entrypoint( self.block_trace, self.other_data, batch_size, diff --git a/zero/src/rpc/jerigon.rs b/zero/src/rpc/jerigon.rs index df00bc605..7aefed896 100644 --- a/zero/src/rpc/jerigon.rs +++ b/zero/src/rpc/jerigon.rs @@ -2,11 +2,11 @@ use alloy::{providers::Provider, rpc::types::eth::BlockId, transports::Transport use anyhow::Context as _; use serde::Deserialize; use serde_json::json; -use trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; use super::fetch_other_block_data; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; +use crate::trace_decoder::{BlockTrace, BlockTraceTriePreImages, CombinedPreImages, TxnInfo}; /// Transaction traces retrieved from Erigon zeroTracer. #[derive(Debug, Deserialize)] diff --git a/zero/src/rpc/mod.rs b/zero/src/rpc/mod.rs index 007a4fdb2..07ed461e6 100644 --- a/zero/src/rpc/mod.rs +++ b/zero/src/rpc/mod.rs @@ -18,10 +18,10 @@ use evm_arithmetization::{ }; use futures::{StreamExt as _, TryStreamExt as _}; use serde_json::json; -use trace_decoder::{BlockLevelData, OtherBlockData}; use tracing::warn; use crate::prover::BlockProverInput; +use crate::trace_decoder::{BlockLevelData, OtherBlockData}; pub mod jerigon; pub mod native; diff --git a/zero/src/rpc/native/mod.rs b/zero/src/rpc/native/mod.rs index 5b4ed5dd9..2ec4963ae 100644 --- a/zero/src/rpc/native/mod.rs +++ b/zero/src/rpc/native/mod.rs @@ -8,10 +8,10 @@ use alloy::{ transports::Transport, }; use futures::try_join; -use trace_decoder::BlockTrace; use crate::prover::BlockProverInput; use crate::provider::CachedProvider; +use crate::trace_decoder::BlockTrace; mod state; mod txn; diff --git a/zero/src/rpc/native/state.rs b/zero/src/rpc/native/state.rs index 3c37e8cbc..a920a042a 100644 --- a/zero/src/rpc/native/state.rs +++ b/zero/src/rpc/native/state.rs @@ -12,13 +12,13 @@ use alloy_compat::Compat; use anyhow::Context as _; use futures::future::{try_join, try_join_all}; use mpt_trie::{builder::PartialTrieBuilder, partial_trie::HashedPartialTrie}; -use trace_decoder::{ + +use crate::provider::CachedProvider; +use crate::trace_decoder::{ BlockTraceTriePreImages, SeparateStorageTriesPreImage, SeparateTriePreImage, SeparateTriePreImages, TxnInfo, }; -use crate::provider::CachedProvider; - /// Processes the state witness for the given block. pub async fn process_state_witness( cached_provider: Arc>, diff --git a/zero/src/rpc/native/txn.rs b/zero/src/rpc/native/txn.rs index 79de06d79..79bdad06a 100644 --- a/zero/src/rpc/native/txn.rs +++ b/zero/src/rpc/native/txn.rs @@ -22,9 +22,9 @@ use alloy::{ use anyhow::Context as _; use compat::Compat; use futures::stream::{FuturesOrdered, TryStreamExt}; -use trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; use super::CodeDb; +use crate::trace_decoder::{ContractCodeUsage, TxnInfo, TxnMeta, TxnTrace}; /// Processes the transactions in the given block and updates the code db. pub(super) async fn process_transactions( diff --git a/trace_decoder/src/interface.rs b/zero/src/trace_decoder.rs similarity index 97% rename from trace_decoder/src/interface.rs rename to zero/src/trace_decoder.rs index abe3b0af0..3d182cd2c 100644 --- a/trace_decoder/src/interface.rs +++ b/zero/src/trace_decoder.rs @@ -1,6 +1,6 @@ -//! Public types for this crate. +//! Types for what used to be a separate crate. //! -//! These are all in one place because they're about to be heavily refactored in [#401](https://github.com/0xPolygonZero/zk_evm/issues/401). +//! This module is largely technical debt, and is due to be heavily refactored in [#401](https://github.com/0xPolygonZero/zk_evm/issues/401). use std::collections::{BTreeMap, BTreeSet, HashMap}; diff --git a/trace_decoder/src/typed_mpt.rs b/zero/src/typed_mpt.rs similarity index 100% rename from trace_decoder/src/typed_mpt.rs rename to zero/src/typed_mpt.rs diff --git a/trace_decoder/src/cases/.gitattributes b/zero/src/wire_tries/cases/.gitattributes similarity index 100% rename from trace_decoder/src/cases/.gitattributes rename to zero/src/wire_tries/cases/.gitattributes diff --git a/trace_decoder/src/cases/README.md b/zero/src/wire_tries/cases/README.md similarity index 100% rename from trace_decoder/src/cases/README.md rename to zero/src/wire_tries/cases/README.md diff --git a/trace_decoder/src/cases/hermez_cdk_erigon.json b/zero/src/wire_tries/cases/hermez_cdk_erigon.json similarity index 100% rename from trace_decoder/src/cases/hermez_cdk_erigon.json rename to zero/src/wire_tries/cases/hermez_cdk_erigon.json diff --git a/trace_decoder/src/cases/zero_jerigon.json b/zero/src/wire_tries/cases/zero_jerigon.json similarity index 100% rename from trace_decoder/src/cases/zero_jerigon.json rename to zero/src/wire_tries/cases/zero_jerigon.json diff --git a/trace_decoder/src/wire.rs b/zero/src/wire_tries/mod.rs similarity index 94% rename from trace_decoder/src/wire.rs rename to zero/src/wire_tries/mod.rs index 6f56f1e44..bdff10117 100644 --- a/trace_decoder/src/wire.rs +++ b/zero/src/wire_tries/mod.rs @@ -1,3 +1,8 @@ +//! Over RPC, ethereum nodes expose their tries as a series of binary +//! [`Instruction`]s in a node-dependant format. +//! +//! These are parsed into the relevant trie depending on the node +//! //! We support two wire formats: //! - Type 1, based on [this specification](https://gist.github.com/mandrigin/ff7eccf30d0ef9c572bafcb0ab665cff#the-bytes-layout). //! - Type 2, loosely based on [this specification](https://github.com/0xPolygonHermez/cdk-erigon/blob/d1d6b3c7a4c81c46fd995c1baa5c1f8069ff0348/turbo/trie/WITNESS.md) @@ -24,6 +29,9 @@ use winnow::{ Parser as _, }; +pub mod type1; +pub mod type2; + pub fn parse(input: &[u8]) -> anyhow::Result>> { match preceded( one_of((0u8, 1u8)), // header @@ -296,6 +304,20 @@ fn array(input: &mut &[u8]) -> PResult<[u8; N]> { .parse_next(input) } +#[cfg(test)] +#[derive(serde::Deserialize)] +struct Case { + #[serde(with = "crate::hex")] + pub bytes: Vec, + #[serde(deserialize_with = "h256")] + pub expected_state_root: ethereum_types::H256, +} + +#[cfg(test)] +fn h256<'de, D: serde::Deserializer<'de>>(it: D) -> Result { + Ok(ethereum_types::H256(crate::hex::deserialize(it)?)) +} + #[cfg(test)] #[track_caller] fn do_test<'a, T: PartialEq + core::fmt::Debug>( diff --git a/trace_decoder/src/type1.rs b/zero/src/wire_tries/type1.rs similarity index 99% rename from trace_decoder/src/type1.rs rename to zero/src/wire_tries/type1.rs index aeea0dbb6..624b1268c 100644 --- a/trace_decoder/src/type1.rs +++ b/zero/src/wire_tries/type1.rs @@ -12,8 +12,8 @@ use mpt_trie::partial_trie::OnOrphanedHashNode; use nunny::NonEmpty; use u4::U4; +use super::{Instruction, SmtLeaf}; use crate::typed_mpt::{StateMpt, StateTrie as _, StorageTrie, TrieKey}; -use crate::wire::{Instruction, SmtLeaf}; #[derive(Debug, Clone)] pub struct Frontend { @@ -387,7 +387,7 @@ fn test_tries() { .enumerate() { println!("case {}", ix); - let instructions = crate::wire::parse(&case.bytes).unwrap(); + let instructions = super::parse(&case.bytes).unwrap(); let frontend = frontend(instructions).unwrap(); assert_eq!(case.expected_state_root, frontend.state.root()); diff --git a/trace_decoder/src/type2.rs b/zero/src/wire_tries/type2.rs similarity index 98% rename from trace_decoder/src/type2.rs rename to zero/src/wire_tries/type2.rs index dd3e45c4b..12e11bb13 100644 --- a/trace_decoder/src/type2.rs +++ b/zero/src/wire_tries/type2.rs @@ -14,7 +14,7 @@ use itertools::{EitherOrBoth, Itertools as _}; use nunny::NonEmpty; use plonky2::field::types::Field; -use crate::wire::{Instruction, SmtLeaf, SmtLeafType}; +use super::{Instruction, SmtLeaf, SmtLeafType}; type SmtTrie = smt_trie::smt::Smt; @@ -233,7 +233,7 @@ fn test_tries() { .enumerate() { println!("case {}", ix); - let instructions = crate::wire::parse(&case.bytes).unwrap(); + let instructions = super::parse(&case.bytes).unwrap(); let frontend = frontend(instructions).unwrap(); assert_eq!(case.expected_state_root, { let mut it = [0; 32]; diff --git a/trace_decoder/tests/cases/.gitattributes b/zero/tests/cases/.gitattributes similarity index 100% rename from trace_decoder/tests/cases/.gitattributes rename to zero/tests/cases/.gitattributes diff --git a/trace_decoder/tests/cases/b19807080_main.json b/zero/tests/cases/b19807080_main.json similarity index 100% rename from trace_decoder/tests/cases/b19807080_main.json rename to zero/tests/cases/b19807080_main.json diff --git a/trace_decoder/tests/cases/b19807080_main_header.json b/zero/tests/cases/b19807080_main_header.json similarity index 100% rename from trace_decoder/tests/cases/b19807080_main_header.json rename to zero/tests/cases/b19807080_main_header.json diff --git a/trace_decoder/tests/cases/b19840104_main.json b/zero/tests/cases/b19840104_main.json similarity index 100% rename from trace_decoder/tests/cases/b19840104_main.json rename to zero/tests/cases/b19840104_main.json diff --git a/trace_decoder/tests/cases/b19840104_main_header.json b/zero/tests/cases/b19840104_main_header.json similarity index 100% rename from trace_decoder/tests/cases/b19840104_main_header.json rename to zero/tests/cases/b19840104_main_header.json diff --git a/trace_decoder/tests/cases/b20240052_main.json b/zero/tests/cases/b20240052_main.json similarity index 100% rename from trace_decoder/tests/cases/b20240052_main.json rename to zero/tests/cases/b20240052_main.json diff --git a/trace_decoder/tests/cases/b20240052_main_header.json b/zero/tests/cases/b20240052_main_header.json similarity index 100% rename from trace_decoder/tests/cases/b20240052_main_header.json rename to zero/tests/cases/b20240052_main_header.json diff --git a/trace_decoder/tests/cases/b20240058_main.json b/zero/tests/cases/b20240058_main.json similarity index 100% rename from trace_decoder/tests/cases/b20240058_main.json rename to zero/tests/cases/b20240058_main.json diff --git a/trace_decoder/tests/cases/b20240058_main_header.json b/zero/tests/cases/b20240058_main_header.json similarity index 100% rename from trace_decoder/tests/cases/b20240058_main_header.json rename to zero/tests/cases/b20240058_main_header.json diff --git a/trace_decoder/tests/cases/b20472570_main.json b/zero/tests/cases/b20472570_main.json similarity index 100% rename from trace_decoder/tests/cases/b20472570_main.json rename to zero/tests/cases/b20472570_main.json diff --git a/trace_decoder/tests/cases/b20472570_main_header.json b/zero/tests/cases/b20472570_main_header.json similarity index 100% rename from trace_decoder/tests/cases/b20472570_main_header.json rename to zero/tests/cases/b20472570_main_header.json diff --git a/trace_decoder/tests/cases/b28_dev.json b/zero/tests/cases/b28_dev.json similarity index 100% rename from trace_decoder/tests/cases/b28_dev.json rename to zero/tests/cases/b28_dev.json diff --git a/trace_decoder/tests/cases/b28_dev_header.json b/zero/tests/cases/b28_dev_header.json similarity index 100% rename from trace_decoder/tests/cases/b28_dev_header.json rename to zero/tests/cases/b28_dev_header.json diff --git a/trace_decoder/tests/cases/b4_dev.json b/zero/tests/cases/b4_dev.json similarity index 100% rename from trace_decoder/tests/cases/b4_dev.json rename to zero/tests/cases/b4_dev.json diff --git a/trace_decoder/tests/cases/b4_dev_header.json b/zero/tests/cases/b4_dev_header.json similarity index 100% rename from trace_decoder/tests/cases/b4_dev_header.json rename to zero/tests/cases/b4_dev_header.json diff --git a/trace_decoder/tests/common/mod.rs b/zero/tests/common/mod.rs similarity index 98% rename from trace_decoder/tests/common/mod.rs rename to zero/tests/common/mod.rs index 0183c785a..0589eb2ee 100644 --- a/trace_decoder/tests/common/mod.rs +++ b/zero/tests/common/mod.rs @@ -4,8 +4,8 @@ use alloy::rpc::types::Header; use anyhow::{ensure, Context as _}; use camino::Utf8Path; use serde::de::DeserializeOwned; -use trace_decoder::{BlockTrace, OtherBlockData}; use zero::prover::BlockProverInput; +use zero::trace_decoder::{BlockTrace, OtherBlockData}; pub fn cases() -> anyhow::Result> { print!("loading test vectors..."); diff --git a/trace_decoder/tests/consistent-with-header.rs b/zero/tests/consistent-with-header.rs similarity index 96% rename from trace_decoder/tests/consistent-with-header.rs rename to zero/tests/consistent-with-header.rs index 609fd57bb..f0bd27f1f 100644 --- a/trace_decoder/tests/consistent-with-header.rs +++ b/zero/tests/consistent-with-header.rs @@ -10,7 +10,7 @@ use common::{cases, Case}; use itertools::Itertools; use libtest_mimic::{Arguments, Trial}; use mpt_trie::partial_trie::PartialTrie as _; -use trace_decoder::observer::DummyObserver; +use zero::intra_block_tries::DummyObserver; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -24,7 +24,7 @@ fn main() -> anyhow::Result<()> { } in cases()? { trials.push(Trial::test(format!("{name}@{batch_size}"), move || { - let gen_inputs = trace_decoder::entrypoint( + let gen_inputs = zero::intra_block_tries::entrypoint( trace, other.clone(), batch_size, diff --git a/trace_decoder/tests/simulate-execution.rs b/zero/tests/simulate-execution.rs similarity index 76% rename from trace_decoder/tests/simulate-execution.rs rename to zero/tests/simulate-execution.rs index d0476c2b7..b4dd32583 100644 --- a/trace_decoder/tests/simulate-execution.rs +++ b/zero/tests/simulate-execution.rs @@ -8,7 +8,7 @@ use anyhow::Context as _; use common::{cases, Case}; use libtest_mimic::{Arguments, Trial}; use plonky2::field::goldilocks_field::GoldilocksField; -use trace_decoder::observer::DummyObserver; +use zero::intra_block_tries::DummyObserver; fn main() -> anyhow::Result<()> { let mut trials = vec![]; @@ -20,11 +20,15 @@ fn main() -> anyhow::Result<()> { other, } in cases()? { - let gen_inputs = - trace_decoder::entrypoint(trace, other, batch_size, &mut DummyObserver::new()) - .context(format!( - "error in `trace_decoder` for {name} at batch size {batch_size}" - ))?; + let gen_inputs = zero::intra_block_tries::entrypoint( + trace, + other, + batch_size, + &mut DummyObserver::new(), + ) + .context(format!( + "error in `trace_decoder` for {name} at batch size {batch_size}" + ))?; for (ix, gi) in gen_inputs.into_iter().enumerate() { trials.push(Trial::test( format!("{name}@{batch_size}/{ix}"), From 4241f07787dee65cee4bff0108b8ed06075fceda Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 08:46:04 +0200 Subject: [PATCH 03/22] fix: ci --- .github/labeler.yml | 5 ----- .github/workflows/ci.yml | 28 ---------------------------- 2 files changed, 33 deletions(-) diff --git a/.github/labeler.yml b/.github/labeler.yml index baacbe1fa..c1cf50121 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -8,11 +8,6 @@ - changed-files: - any-glob-to-any-file: evm_arithmetization/** -# Add 'crate: trace_decoder' label to any changes within 'trace_decoder' folder. -'crate: trace_decoder': -- changed-files: - - any-glob-to-any-file: trace_decoder/** - # Add 'crate: zero_bin' label to any changes within 'zero' folder. 'crate: zero_bin': - changed-files: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8e8ea6aee..095910b7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -43,34 +43,6 @@ jobs: CARGO_INCREMENTAL: 1 RUST_BACKTRACE: 1 - test_trace_decoder: - name: Test trace_decoder - runs-on: zero-ci - timeout-minutes: 30 - env: - RUST_LOG: info - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 - - if: ${{ ! contains(toJSON(github.event.commits.*.message), '[skip-ci]') }} - steps: - - name: Checkout sources - uses: actions/checkout@v4 - - - name: Install nightly toolchain - uses: dtolnay/rust-toolchain@nightly - - - name: Set up rust cache - uses: Swatinem/rust-cache@v2 - with: - cache-on-failure: true - - - name: build # build separately so test logs are actually nice - run: cargo build --tests --manifest-path trace_decoder/Cargo.toml - - - name: test - run: cargo test --release --manifest-path trace_decoder/Cargo.toml -- --nocapture - test_evm_arithmetization: name: Test evm_arithmetization runs-on: ubuntu-latest From c439a5e006ca21a07e1496fa6d9594494a58af51 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 22:25:29 +0200 Subject: [PATCH 04/22] refactor: rustflags = -Clink-args=-znostart-stop-gc --- .cargo/config.toml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 6340ce34a..e76d6e6bb 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,4 +1,3 @@ [build] -# https://github.com/rust-lang/rust/pull/124129 -# https://github.com/dtolnay/linkme/pull/88 -rustflags = ["-Z", "linker-features=-lld"] +# https://github.com/dtolnay/linkme/issues/94 +rustflags = ["-C", "link-args=-znostart-stop-gc"] From 3a44607712b234d07981ab617afb8113661ffa8a Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 22:49:41 +0200 Subject: [PATCH 05/22] note: test zero fails From 70d93f14e0a2723669f32391cacfd426ed6fb9e8 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 22:54:01 +0200 Subject: [PATCH 06/22] refactor: rustflags = "" --- .cargo/config.toml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index e76d6e6bb..38bd089b6 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,3 +1,2 @@ [build] -# https://github.com/dtolnay/linkme/issues/94 -rustflags = ["-C", "link-args=-znostart-stop-gc"] +rustflags = [] From b9291c301af9b3959ec5902f3ea30d809c849dce Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 22:58:57 +0200 Subject: [PATCH 07/22] note: test zero, docker fails From 15fb7929dcbf878bf27948a6b0c8282312067f24 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 22:59:17 +0200 Subject: [PATCH 08/22] refactor: linkme --feature used_linker --- Cargo.lock | 1 + zero/Cargo.toml | 1 + zero/src/lib.rs | 2 ++ 3 files changed, 4 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 09eef0a79..87702c0d1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5729,6 +5729,7 @@ dependencies = [ "jemallocator", "keccak-hash 0.10.0", "libtest-mimic", + "linkme", "lru", "mpt_trie", "num-traits", diff --git a/zero/Cargo.toml b/zero/Cargo.toml index 9332d7480..e30ccf8d5 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -30,6 +30,7 @@ hashbrown = { workspace = true } hex = { workspace = true } itertools = { workspace = true } keccak-hash = { workspace = true } +linkme = { version = "0.3.28", features = ["used_linker"] } lru = { workspace = true } num-traits = { workspace = true } nunny = { workspace = true } diff --git a/zero/src/lib.rs b/zero/src/lib.rs index 02e599d8f..e9159a5a9 100644 --- a/zero/src/lib.rs +++ b/zero/src/lib.rs @@ -1,3 +1,5 @@ +#![feature(used_with_arg)] + zk_evm_common::check_chain_features!(); pub mod block_interval; From a88af218ed10b476e4e1043e3bf90cf0e8ba9b0c Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 23:28:09 +0200 Subject: [PATCH 09/22] note: this approach would require changes in paladin-core From f71822d29060c42f837e6e2f6bf1d62bc15ddd0d Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 23:28:36 +0200 Subject: [PATCH 10/22] Revert "refactor: linkme --feature used_linker" This reverts commit 15fb7929dcbf878bf27948a6b0c8282312067f24. --- Cargo.lock | 1 - zero/Cargo.toml | 1 - zero/src/lib.rs | 2 -- 3 files changed, 4 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 87702c0d1..09eef0a79 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5729,7 +5729,6 @@ dependencies = [ "jemallocator", "keccak-hash 0.10.0", "libtest-mimic", - "linkme", "lru", "mpt_trie", "num-traits", diff --git a/zero/Cargo.toml b/zero/Cargo.toml index e30ccf8d5..9332d7480 100644 --- a/zero/Cargo.toml +++ b/zero/Cargo.toml @@ -30,7 +30,6 @@ hashbrown = { workspace = true } hex = { workspace = true } itertools = { workspace = true } keccak-hash = { workspace = true } -linkme = { version = "0.3.28", features = ["used_linker"] } lru = { workspace = true } num-traits = { workspace = true } nunny = { workspace = true } diff --git a/zero/src/lib.rs b/zero/src/lib.rs index e9159a5a9..02e599d8f 100644 --- a/zero/src/lib.rs +++ b/zero/src/lib.rs @@ -1,5 +1,3 @@ -#![feature(used_with_arg)] - zk_evm_common::check_chain_features!(); pub mod block_interval; From f823126bc476f80d9742b3fa2f3ff968588976a2 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 23:31:13 +0200 Subject: [PATCH 11/22] refactor: rustflags = -Clink-dead-code --- .cargo/config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 38bd089b6..939a9c2a5 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,2 @@ [build] -rustflags = [] +rustflags = ["-C", "link-dead-code"] From a31e941a3044c2dd745cadf2f0dad5de050657cf Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 23:42:55 +0200 Subject: [PATCH 12/22] note: test zero fails From 68ecd1bfc6aef4dee81e085b95def2961e4f5920 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Wed, 25 Sep 2024 23:47:05 +0200 Subject: [PATCH 13/22] ci: use mold linker for test_zero_bin --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 095910b7f..95c149917 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 - + - uses: rui314/setup-mold@v1 - name: Install nightly toolchain uses: dtolnay/rust-toolchain@nightly From c39e6e8032e0357a5b1ce28eb54a8088ae3a89d7 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 00:00:51 +0200 Subject: [PATCH 14/22] note: test zero fails From f02f64fb5e10e155e321b03c747df26b143aa729 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 00:01:07 +0200 Subject: [PATCH 15/22] Revert "ci: use mold linker for test_zero_bin" This reverts commit 68ecd1bfc6aef4dee81e085b95def2961e4f5920. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 95c149917..095910b7f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -76,7 +76,7 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@v4 - - uses: rui314/setup-mold@v1 + - name: Install nightly toolchain uses: dtolnay/rust-toolchain@nightly From fad3554336b5c4db822565ca1771f8ce377e178d Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 00:52:36 +0200 Subject: [PATCH 16/22] refactor: unset RUSTFLAGS in gha >=[ --- .github/workflows/ci.yml | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 095910b7f..945a42eca 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,11 +37,6 @@ jobs: - name: Test in mpt_trie subdirectory run: cargo test --manifest-path mpt_trie/Cargo.toml - env: - RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 - RUST_LOG: 1 - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 test_evm_arithmetization: name: Test evm_arithmetization @@ -62,11 +57,6 @@ jobs: - name: Test in evm_arithmetization subdirectory run: cargo test --manifest-path evm_arithmetization/Cargo.toml - env: - RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 - RUST_LOG: 1 - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 test_zero_bin: name: Test zero_bin @@ -89,12 +79,6 @@ jobs: run: | cargo test --manifest-path zero/Cargo.toml - env: - RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 - RUST_LOG: 1 - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 - test_zk_evm_proc_macro: name: Test zk_evm_proc_macro runs-on: ubuntu-latest @@ -114,11 +98,6 @@ jobs: - name: Test in proc_macro subdirectory run: cargo test --manifest-path proc_macro/Cargo.toml - env: - RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 - RUST_LOG: 1 - CARGO_INCREMENTAL: 1 - RUST_BACKTRACE: 1 simple_proof_regular: name: Execute bash script to generate and verify a proof for a small block. From 2fa333e0d8f7c9e7622c5a524f7de2f707ba7137 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 01:15:26 +0200 Subject: [PATCH 17/22] note: passes From be6522923c7b95d1bdef9c1c947bf1c56e4edf07 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 01:16:14 +0200 Subject: [PATCH 18/22] run: git checkout origin/develop -- .cargo/config.toml --- .cargo/config.toml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.cargo/config.toml b/.cargo/config.toml index 939a9c2a5..6340ce34a 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -1,2 +1,4 @@ [build] -rustflags = ["-C", "link-dead-code"] +# https://github.com/rust-lang/rust/pull/124129 +# https://github.com/dtolnay/linkme/pull/88 +rustflags = ["-Z", "linker-features=-lld"] From aac87354e889cf6e0b579977ea9925b727753ed2 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 01:29:43 +0200 Subject: [PATCH 19/22] doc: update README.md --- README.md | 23 ++++++++--------------- 1 file changed, 8 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 69fbc79a2..4c0def302 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,6 @@ This repository contains the following Rust crates: * [smt_trie](./smt_trie/README.md): A collection of types and functions to work with Polygon Hermez Sparse Merkle Trees (SMT). -* [trace_decoder](./trace_decoder/Cargo.toml): Flexible protocol designed to process Ethereum clients trace payloads into an IR format that can be -understood by the zkEVM prover. - * [evm_arithmetization](./evm_arithmetization/README.md): Defines all the STARK constraints and recursive circuits to generate succinct proofs of EVM execution. It uses starky and plonky2 as proving backend: https://github.com/0xPolygonZero/plonky2. @@ -32,24 +29,20 @@ TODO: Update mermaid chart with `smt_trie` once type-2 is plugged in. %%{init: {'theme':'dark'}}%% flowchart LR subgraph ps [proving systems] - A1{{plonky2}} - A2{{starky}} + A1{{plonky2}} + A2{{starky}} end ps --> zk_evm - subgraph zk_evm [zk_evm] - B[mpt_trie] - C[evm_arithmetization] - D[trace_decoder] + subgraph zk_evm + B[mpt_trie] + C[evm_arithmetization] - B --> C - B ---> D - C ---> D + B --> C - F{zero-bin} - C --> F - D --> F + F{zero-bin} + C --> F end ``` From e4461fc22d1c5262a25868ec8512536e2c52d712 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 02:05:03 +0200 Subject: [PATCH 20/22] ci: cargo test --release evm_arith... --- .github/workflows/ci.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 945a42eca..30e8626fa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,7 +56,8 @@ jobs: cache-on-failure: true - name: Test in evm_arithmetization subdirectory - run: cargo test --manifest-path evm_arithmetization/Cargo.toml + # we hit timeouts if we `cargo test --debug ...` + run: cargo test --release --manifest-path evm_arithmetization/Cargo.toml test_zero_bin: name: Test zero_bin From 2df30a5d5c6be4587738a6f79c4fcec62cbe5fd9 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 02:59:04 +0200 Subject: [PATCH 21/22] ci: restore RUSTFLAGS for evm_arith tests --- .github/workflows/ci.yml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 30e8626fa..eb614b100 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -56,8 +56,12 @@ jobs: cache-on-failure: true - name: Test in evm_arithmetization subdirectory - # we hit timeouts if we `cargo test --debug ...` - run: cargo test --release --manifest-path evm_arithmetization/Cargo.toml + run: cargo test --manifest-path evm_arithmetization/Cargo.toml + env: + RUSTFLAGS: -Copt-level=3 -Cdebug-assertions -Coverflow-checks=y -Cdebuginfo=0 + RUST_LOG: 1 + CARGO_INCREMENTAL: 1 + RUST_BACKTRACE: 1 test_zero_bin: name: Test zero_bin From 8f45f1e6a179ce1b786949172d9f3ac6fe39b868 Mon Sep 17 00:00:00 2001 From: 0xaatif Date: Thu, 26 Sep 2024 04:03:43 +0200 Subject: [PATCH 22/22] chore: look for other mentions of trace_decoder --- .github/CODEOWNERS | 1 - zero/README.md | 20 -------------------- zero/tests/cases/README.md | 7 +++++++ 3 files changed, 7 insertions(+), 21 deletions(-) create mode 100644 zero/tests/cases/README.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index a3608580d..faa9de096 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -3,5 +3,4 @@ /zero/ @muursh @Nashtare @atanmarko /smt_trie/ @0xaatif @muursh @Nashtare /mpt_trie/ @0xaatif @Nashtare @muursh -/trace_decoder/ @0xaatif @muursh @Nashtare .github/ @0xaatif @atanmarko @muursh @Nashtare diff --git a/zero/README.md b/zero/README.md index 936a73c91..f2db9a0f3 100644 --- a/zero/README.md +++ b/zero/README.md @@ -458,26 +458,6 @@ Filled in: Finally, note that both of these testing scripts force proof generation to be sequential by allowing only one worker. Because of this, this is not a realistic representation of performance but makes the debugging logs much easier to follow. -### Trace decoder tests - -The trace decoder module has some basic regression tests, using the json witness data from the `trace_decoder/tests/data/witnesses` subdirectories. -When needed (e.g. some block with corner-case discovered), additional input witness data should be generated using the following procedure: - -1. Run the `rpc` tool to fetch the block (or multiple blocks) witness: - -```sh -cargo run --package zero --bin rpc fetch --rpc-url --start-block --end-block > ./b_.json -``` - -2. Download the header file for the block (or range of blocks), making the json array of headers: - -```sh -file_name = "b__header.json" -echo "[" > $file_name && cast rpc eth_getBlockByNumber "0x" 'false' --rpc-url >> $file_name && echo "]" >> $file_name -``` - -Move the generated files to the appropriate subdirectory, and they will be automatically included in the test run. - ## License Licensed under either of diff --git a/zero/tests/cases/README.md b/zero/tests/cases/README.md new file mode 100644 index 000000000..798e59f55 --- /dev/null +++ b/zero/tests/cases/README.md @@ -0,0 +1,7 @@ +```bash +cargo run --package zero --bin rpc fetch --rpc-url $rpc_url --start-block $number --end-block $number > ./b$number_$network.json +header_name=b$number_$network_header.json +echo "[" > $file_name +cast rpc eth_getBlockByNumber 0x$number false --rpc-url $rpc_url >> $file_name +echo "]" >> $file_name +```