From fcab695a93526c704ef898a3a8d2b6afb183f579 Mon Sep 17 00:00:00 2001 From: Arsenii Kulikov Date: Mon, 2 Sep 2024 18:20:15 +0400 Subject: [PATCH] refactor: extract `op-reth` binary to separate crate (#10641) Co-authored-by: Loocapro Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com> --- .github/workflows/lint.yml | 3 +- .github/workflows/op-sync.yml | 3 +- Cargo.lock | 31 +++---- Cargo.toml | 3 + Makefile | 10 +-- bin/reth-bench/Cargo.toml | 2 +- bin/reth/Cargo.toml | 44 +--------- bin/reth/src/cli/mod.rs | 25 ++---- .../src/commands/debug_cmd/build_block.rs | 25 +----- bin/reth/src/commands/debug_cmd/execution.rs | 9 +- .../commands/debug_cmd/in_memory_merkle.rs | 16 ++-- bin/reth/src/commands/debug_cmd/merkle.rs | 10 +-- .../src/commands/debug_cmd/replay_engine.rs | 23 +---- bin/reth/src/lib.rs | 2 +- bin/reth/src/macros.rs | 20 ----- bin/reth/src/main.rs | 4 - bin/reth/src/optimism.rs | 84 ------------------- crates/optimism/bin/Cargo.toml | 42 ++++++++++ crates/optimism/bin/src/main.rs | 79 +++++++++++++++++ crates/optimism/chainspec/src/lib.rs | 2 +- crates/optimism/cli/Cargo.toml | 6 +- crates/optimism/cli/src/chainspec.rs | 7 +- crates/optimism/cli/src/lib.rs | 58 ++++++++++++- crates/optimism/node/Cargo.toml | 1 + 24 files changed, 249 insertions(+), 260 deletions(-) delete mode 100644 bin/reth/src/macros.rs delete mode 100644 bin/reth/src/optimism.rs create mode 100644 crates/optimism/bin/Cargo.toml create mode 100644 crates/optimism/bin/src/main.rs diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 27ca0deeb7bc..6e944eb8f077 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -71,7 +71,8 @@ jobs: - uses: Swatinem/rust-cache@v2 with: cache-on-failure: true - - run: cargo hack check + - run: cargo hack check --workspace --exclude op-reth + - run: cargo check -p op-reth --features "optimism" msrv: name: MSRV / ${{ matrix.network }} diff --git a/.github/workflows/op-sync.yml b/.github/workflows/op-sync.yml index 26fa68d7f905..2a223391d711 100644 --- a/.github/workflows/op-sync.yml +++ b/.github/workflows/op-sync.yml @@ -33,8 +33,7 @@ jobs: with: cache-on-failure: true - name: Build op-reth - run: | - cargo install --features asm-keccak,jemalloc,optimism --bin op-reth --path bin/reth + run: make install-op - name: Run sync # https://basescan.org/block/10000 run: | diff --git a/Cargo.lock b/Cargo.lock index c7e48f85c6f1..c9368795c77a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5104,6 +5104,20 @@ dependencies = [ "serde", ] +[[package]] +name = "op-reth" +version = "1.0.6" +dependencies = [ + "clap", + "reth-cli-util", + "reth-node-builder", + "reth-node-optimism", + "reth-optimism-cli", + "reth-optimism-rpc", + "reth-provider", + "tikv-jemallocator", +] + [[package]] name = "opaque-debug" version = "0.3.1" @@ -6040,13 +6054,8 @@ dependencies = [ "aquamarine", "backon", "clap", - "discv5", "eyre", - "fdlimit", "futures", - "itertools 0.13.0", - "libc", - "metrics-process", "reth-basic-payload-builder", "reth-beacon-consensus", "reth-blockchain-tree", @@ -6060,7 +6069,6 @@ dependencies = [ "reth-consensus-common", "reth-db", "reth-db-api", - "reth-db-common", "reth-discv4", "reth-downloaders", "reth-engine-util", @@ -6079,10 +6087,6 @@ dependencies = [ "reth-node-ethereum", "reth-node-events", "reth-node-metrics", - "reth-node-optimism", - "reth-optimism-cli", - "reth-optimism-primitives", - "reth-optimism-rpc", "reth-payload-builder", "reth-payload-primitives", "reth-payload-validator", @@ -6098,21 +6102,17 @@ dependencies = [ "reth-rpc-types", "reth-rpc-types-compat", "reth-stages", - "reth-stages-api", "reth-static-file", - "reth-static-file-types", "reth-tasks", "reth-tracing", "reth-transaction-pool", "reth-trie", "reth-trie-db", - "serde", "serde_json", "similar-asserts", "tempfile", "tikv-jemallocator", "tokio", - "toml", "tracing", ] @@ -7778,6 +7778,7 @@ dependencies = [ "reth-chainspec", "reth-cli", "reth-cli-commands", + "reth-cli-runner", "reth-config", "reth-consensus", "reth-db", @@ -7788,6 +7789,7 @@ dependencies = [ "reth-evm-optimism", "reth-execution-types", "reth-network-p2p", + "reth-node-builder", "reth-node-core", "reth-node-events", "reth-optimism-chainspec", @@ -7799,6 +7801,7 @@ dependencies = [ "reth-stages-types", "reth-static-file", "reth-static-file-types", + "reth-tracing", "tempfile", "tokio", "tokio-util", diff --git a/Cargo.toml b/Cargo.toml index cda62c481f7c..2f1cb72c0e97 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -67,6 +67,7 @@ members = [ "crates/node/core/", "crates/node/events/", "crates/node/metrics", + "crates/optimism/bin", "crates/optimism/chainspec", "crates/optimism/cli", "crates/optimism/consensus", @@ -349,6 +350,7 @@ reth-node-ethereum = { path = "crates/ethereum/node" } reth-node-events = { path = "crates/node/events" } reth-node-metrics = { path = "crates/node/metrics" } reth-node-optimism = { path = "crates/optimism/node" } +op-reth = { path = "crates/optimism/bin" } reth-optimism-chainspec = { path = "crates/optimism/chainspec" } reth-optimism-cli = { path = "crates/optimism/cli" } reth-optimism-consensus = { path = "crates/optimism/consensus" } @@ -563,3 +565,4 @@ serial_test = "3" similar-asserts = "1.5.0" tempfile = "3.8" test-fuzz = "5" +tikv-jemallocator = { version = "0.5.0" } diff --git a/Makefile b/Makefile index 6b43c19e4f42..c0169aa18275 100644 --- a/Makefile +++ b/Makefile @@ -52,7 +52,7 @@ install: ## Build and install the reth binary under `~/.cargo/bin`. .PHONY: install-op install-op: ## Build and install the op-reth binary under `~/.cargo/bin`. - cargo install --path bin/reth --bin op-reth --force --locked \ + cargo install --path crates/optimism/bin --bin op-reth --force --locked \ --features "optimism,$(FEATURES)" \ --profile "$(PROFILE)" \ $(CARGO_INSTALL_EXTRA_FLAGS) @@ -67,14 +67,14 @@ build-debug: ## Build the reth binary into `target/debug` directory. .PHONY: build-op build-op: ## Build the op-reth binary into `target` directory. - cargo build --bin op-reth --features "optimism,$(FEATURES)" --profile "$(PROFILE)" + cargo build --bin op-reth --features "optimism,$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml # Builds the reth binary natively. build-native-%: cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)" op-build-native-%: - cargo build --bin op-reth --target $* --features "optimism,$(FEATURES)" --profile "$(PROFILE)" + cargo build --bin op-reth --target $* --features "optimism,$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml # The following commands use `cross` to build a cross-compile. # @@ -106,7 +106,7 @@ build-%: op-build-%: RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \ - cross build --bin op-reth --target $* --features "optimism,$(FEATURES)" --profile "$(PROFILE)" + cross build --bin op-reth --target $* --features "optimism,$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml # Unfortunately we can't easily use cross to build for Darwin because of licensing issues. # If we wanted to, we would need to build a custom Docker image with the SDK available. @@ -314,7 +314,7 @@ maxperf: ## Builds `reth` with the most aggressive optimisations. .PHONY: maxperf-op maxperf-op: ## Builds `op-reth` with the most aggressive optimisations. - RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --features jemalloc,asm-keccak,optimism --bin op-reth + RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --features jemalloc,asm-keccak,optimism --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml .PHONY: maxperf-no-asm maxperf-no-asm: ## Builds `reth` with the most aggressive optimisations, minus the "asm-keccak" feature. diff --git a/bin/reth-bench/Cargo.toml b/bin/reth-bench/Cargo.toml index 29e9c4963c95..4023c1c17375 100644 --- a/bin/reth-bench/Cargo.toml +++ b/bin/reth-bench/Cargo.toml @@ -72,7 +72,7 @@ clap = { workspace = true, features = ["derive", "env"] } csv = "1.3.0" [target.'cfg(unix)'.dependencies] -tikv-jemallocator = { version = "0.5.0", optional = true } +tikv-jemallocator = { workspace = true, optional = true } libc = "0.2" [dev-dependencies] diff --git a/bin/reth/Cargo.toml b/bin/reth/Cargo.toml index df13b198e467..55f45214a1e1 100644 --- a/bin/reth/Cargo.toml +++ b/bin/reth/Cargo.toml @@ -53,27 +53,18 @@ reth-payload-primitives.workspace = true reth-payload-validator.workspace = true reth-basic-payload-builder.workspace = true reth-static-file.workspace = true -reth-static-file-types = { workspace = true, features = ["clap"] } reth-trie = { workspace = true, features = ["metrics"] } reth-trie-db = { workspace = true, features = ["metrics"] } reth-node-api.workspace = true -reth-node-optimism = { workspace = true, optional = true, features = [ - "optimism", -] } reth-node-core.workspace = true reth-ethereum-payload-builder.workspace = true -reth-db-common.workspace = true reth-node-ethereum.workspace = true reth-node-builder.workspace = true reth-node-events.workspace = true reth-node-metrics.workspace = true reth-consensus.workspace = true -reth-optimism-primitives.workspace = true reth-engine-util.workspace = true reth-prune.workspace = true -reth-stages-api.workspace = true -reth-optimism-cli = { workspace = true, optional = true } -reth-optimism-rpc.workspace = true # crypto alloy-rlp.workspace = true @@ -82,13 +73,7 @@ alloy-rlp.workspace = true tracing.workspace = true # io -fdlimit.workspace = true -serde.workspace = true serde_json.workspace = true -toml = { workspace = true, features = ["display"] } - -# metrics -metrics-process.workspace = true # async tokio = { workspace = true, features = [ @@ -103,20 +88,15 @@ futures.workspace = true aquamarine.workspace = true eyre.workspace = true clap = { workspace = true, features = ["derive", "env"] } -tempfile.workspace = true backon.workspace = true similar-asserts.workspace = true -itertools.workspace = true - -# p2p -discv5.workspace = true [target.'cfg(unix)'.dependencies] -tikv-jemallocator = { version = "0.5.0", optional = true } -libc = "0.2" +tikv-jemallocator = { workspace = true, optional = true } [dev-dependencies] reth-discv4.workspace = true +tempfile.workspace = true [features] default = ["jemalloc"] @@ -134,26 +114,6 @@ min-info-logs = ["tracing/release_max_level_info"] min-debug-logs = ["tracing/release_max_level_debug"] min-trace-logs = ["tracing/release_max_level_trace"] -optimism = [ - "dep:reth-node-optimism", - "dep:reth-optimism-cli", - "reth-beacon-consensus/optimism", - "reth-blockchain-tree/optimism", - "reth-node-core/optimism", - "reth-optimism-cli?/optimism", - "reth-primitives/optimism", - "reth-provider/optimism", - "reth-rpc/optimism", -] - -# no-op feature flag for switching between the `optimism` and default functionality in CI matrices -ethereum = [] - [[bin]] name = "reth" path = "src/main.rs" - -[[bin]] -name = "op-reth" -path = "src/optimism.rs" -required-features = ["optimism"] diff --git a/bin/reth/src/cli/mod.rs b/bin/reth/src/cli/mod.rs index 71f1defc1670..eaa0c77f52a5 100644 --- a/bin/reth/src/cli/mod.rs +++ b/bin/reth/src/cli/mod.rs @@ -3,7 +3,6 @@ use crate::{ args::{utils::chain_help, LogArgs}, commands::debug_cmd, - macros::block_executor, version::{LONG_VERSION, SHORT_VERSION}, }; use clap::{value_parser, Parser, Subcommand}; @@ -18,6 +17,7 @@ use reth_cli_runner::CliRunner; use reth_db::DatabaseEnv; use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::args::utils::DefaultChainSpecParser; +use reth_node_ethereum::EthExecutorProvider; use reth_tracing::FileWorkerGuard; use std::{ffi::OsString, fmt, future::Future, sync::Arc}; use tracing::info; @@ -150,20 +150,13 @@ impl, Ext: clap::Args + fmt::Debug> Cl } Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::InitState(command) => runner.run_blocking_until_ctrl_c(command.execute()), - Commands::Import(command) => runner.run_blocking_until_ctrl_c( - command.execute(|chain_spec| block_executor!(chain_spec)), - ), - #[cfg(feature = "optimism")] - Commands::ImportOp(command) => runner.run_blocking_until_ctrl_c(command.execute()), - #[cfg(feature = "optimism")] - Commands::ImportReceiptsOp(command) => { - runner.run_blocking_until_ctrl_c(command.execute()) + Commands::Import(command) => { + runner.run_blocking_until_ctrl_c(command.execute(EthExecutorProvider::ethereum)) } Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute()), - Commands::Stage(command) => runner.run_command_until_exit(|ctx| { - command.execute(ctx, |chain_spec| block_executor!(chain_spec)) - }), + Commands::Stage(command) => runner + .run_command_until_exit(|ctx| command.execute(ctx, EthExecutorProvider::ethereum)), Commands::P2P(command) => runner.run_until_ctrl_c(command.execute()), #[cfg(feature = "dev")] Commands::TestVectors(command) => runner.run_until_ctrl_c(command.execute()), @@ -199,14 +192,6 @@ pub enum Commands { /// This syncs RLP encoded blocks from a file. #[command(name = "import")] Import(import::ImportCommand), - /// This syncs RLP encoded OP blocks below Bedrock from a file, without executing. - #[cfg(feature = "optimism")] - #[command(name = "import-op")] - ImportOp(reth_optimism_cli::ImportOpCommand), - /// This imports RLP encoded receipts from a file. - #[cfg(feature = "optimism")] - #[command(name = "import-receipts-op")] - ImportReceiptsOp(reth_optimism_cli::ImportReceiptsOpCommand), /// Dumps genesis block JSON configuration to stdout. DumpGenesis(dump_genesis::DumpGenesisCommand), /// Database debugging utilities diff --git a/bin/reth/src/commands/debug_cmd/build_block.rs b/bin/reth/src/commands/debug_cmd/build_block.rs index f09c82b67df7..c6a8bf38ebbd 100644 --- a/bin/reth/src/commands/debug_cmd/build_block.rs +++ b/bin/reth/src/commands/debug_cmd/build_block.rs @@ -1,5 +1,4 @@ //! Command for debugging block building. -use crate::macros::block_executor; use alloy_rlp::Decodable; use clap::Parser; use eyre::Context; @@ -21,6 +20,7 @@ use reth_evm::execute::{BlockExecutorProvider, Executor}; use reth_execution_types::ExecutionOutcome; use reth_fs_util as fs; use reth_node_api::PayloadBuilderAttributes; +use reth_node_ethereum::EthExecutorProvider; use reth_payload_builder::database::CachedReads; use reth_primitives::{ constants::eip4844::LoadKzgSettingsError, revm_primitives::KzgSettings, Address, @@ -122,7 +122,7 @@ impl> Command { let consensus: Arc = Arc::new(EthBeaconConsensus::new(provider_factory.chain_spec())); - let executor = block_executor!(provider_factory.chain_spec()); + let executor = EthExecutorProvider::ethereum(provider_factory.chain_spec()); // configure blockchain tree let tree_externals = @@ -223,17 +223,6 @@ impl> Command { let payload_config = PayloadConfig::new( Arc::clone(&best_block), Bytes::default(), - #[cfg(feature = "optimism")] - reth_node_optimism::OptimismPayloadBuilderAttributes::try_new( - best_block.hash(), - reth_rpc_types::optimism::OptimismPayloadAttributes { - payload_attributes: payload_attrs, - transactions: None, - no_tx_pool: None, - gas_limit: None, - }, - )?, - #[cfg(not(feature = "optimism"))] reth_payload_builder::EthPayloadBuilderAttributes::try_new( best_block.hash(), payload_attrs, @@ -250,13 +239,6 @@ impl> Command { None, ); - #[cfg(feature = "optimism")] - let payload_builder = reth_node_optimism::OptimismPayloadBuilder::new( - reth_node_optimism::OptimismEvmConfig::default(), - ) - .compute_pending_block(); - - #[cfg(not(feature = "optimism"))] let payload_builder = reth_ethereum_payload_builder::EthereumPayloadBuilder::default(); match payload_builder.try_build(args)? { @@ -273,7 +255,8 @@ impl> Command { SealedBlockWithSenders::new(block.clone(), senders).unwrap(); let db = StateProviderDatabase::new(blockchain_db.latest()?); - let executor = block_executor!(provider_factory.chain_spec()).executor(db); + let executor = + EthExecutorProvider::ethereum(provider_factory.chain_spec()).executor(db); let block_execution_output = executor.execute((&block_with_senders.clone().unseal(), U256::MAX).into())?; diff --git a/bin/reth/src/commands/debug_cmd/execution.rs b/bin/reth/src/commands/debug_cmd/execution.rs index 36e24c69d6b5..6eeaec273e6a 100644 --- a/bin/reth/src/commands/debug_cmd/execution.rs +++ b/bin/reth/src/commands/debug_cmd/execution.rs @@ -1,7 +1,6 @@ //! Command for debugging execution. -use std::{path::PathBuf, sync::Arc}; - +use crate::{args::NetworkArgs, utils::get_single_header}; use clap::Parser; use futures::{stream::select as stream_select, StreamExt}; use reth_beacon_consensus::EthBeaconConsensus; @@ -22,6 +21,7 @@ use reth_exex::ExExManagerHandle; use reth_network::{BlockDownloaderProvider, NetworkEventListenerProvider, NetworkHandle}; use reth_network_api::NetworkInfo; use reth_network_p2p::{headers::client::HeadersClient, BlockClient}; +use reth_node_ethereum::EthExecutorProvider; use reth_primitives::{BlockHashOrNumber, BlockNumber, B256}; use reth_provider::{ BlockExecutionWriter, ChainSpecProvider, ProviderFactory, StageCheckpointReader, @@ -33,11 +33,10 @@ use reth_stages::{ }; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; +use std::{path::PathBuf, sync::Arc}; use tokio::sync::watch; use tracing::*; -use crate::{args::NetworkArgs, macros::block_executor, utils::get_single_header}; - /// `reth debug execution` command #[derive(Debug, Parser)] pub struct Command { @@ -84,7 +83,7 @@ impl> Command { let prune_modes = config.prune.clone().map(|prune| prune.segments).unwrap_or_default(); let (tip_tx, tip_rx) = watch::channel(B256::ZERO); - let executor = block_executor!(provider_factory.chain_spec()); + let executor = EthExecutorProvider::ethereum(provider_factory.chain_spec()); let pipeline = Pipeline::builder() .with_tip_sender(tip_tx) diff --git a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs index 2d664deb4ec9..6bbb78d7d9a5 100644 --- a/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs +++ b/bin/reth/src/commands/debug_cmd/in_memory_merkle.rs @@ -1,7 +1,9 @@ //! Command for debugging in-memory merkle trie calculation. -use std::{path::PathBuf, sync::Arc}; - +use crate::{ + args::NetworkArgs, + utils::{get_single_body, get_single_header}, +}; use backon::{ConstantBuilder, Retryable}; use clap::Parser; use reth_chainspec::ChainSpec; @@ -16,6 +18,7 @@ use reth_evm::execute::{BlockExecutorProvider, Executor}; use reth_execution_types::ExecutionOutcome; use reth_network::{BlockDownloaderProvider, NetworkHandle}; use reth_network_api::NetworkInfo; +use reth_node_ethereum::EthExecutorProvider; use reth_primitives::BlockHashOrNumber; use reth_provider::{ writer::UnifiedStorageWriter, AccountExtReader, ChainSpecProvider, HashingWriter, @@ -27,14 +30,9 @@ use reth_stages::StageId; use reth_tasks::TaskExecutor; use reth_trie::StateRoot; use reth_trie_db::DatabaseStateRoot; +use std::{path::PathBuf, sync::Arc}; use tracing::*; -use crate::{ - args::NetworkArgs, - macros::block_executor, - utils::{get_single_body, get_single_header}, -}; - /// `reth debug in-memory-merkle` command /// This debug routine requires that the node is positioned at the block before the target. /// The script will then download the block from p2p network and attempt to calculate and verify @@ -134,7 +132,7 @@ impl> Command { provider_factory.static_file_provider(), )); - let executor = block_executor!(provider_factory.chain_spec()).executor(db); + let executor = EthExecutorProvider::ethereum(provider_factory.chain_spec()).executor(db); let merkle_block_td = provider.header_td_by_number(merkle_block_number)?.unwrap_or_default(); diff --git a/bin/reth/src/commands/debug_cmd/merkle.rs b/bin/reth/src/commands/debug_cmd/merkle.rs index 5e413e568ffb..1822a87ab483 100644 --- a/bin/reth/src/commands/debug_cmd/merkle.rs +++ b/bin/reth/src/commands/debug_cmd/merkle.rs @@ -1,7 +1,5 @@ //! Command for debugging merkle trie calculation. - -use std::{path::PathBuf, sync::Arc}; - +use crate::{args::NetworkArgs, utils::get_single_header}; use backon::{ConstantBuilder, Retryable}; use clap::Parser; use reth_beacon_consensus::EthBeaconConsensus; @@ -18,6 +16,7 @@ use reth_evm::execute::{BatchExecutor, BlockExecutorProvider}; use reth_network::{BlockDownloaderProvider, NetworkHandle}; use reth_network_api::NetworkInfo; use reth_network_p2p::full_block::FullBlockClient; +use reth_node_ethereum::EthExecutorProvider; use reth_primitives::BlockHashOrNumber; use reth_provider::{ writer::UnifiedStorageWriter, BlockNumReader, BlockWriter, ChainSpecProvider, HeaderProvider, @@ -29,10 +28,9 @@ use reth_stages::{ ExecInput, Stage, StageCheckpoint, }; use reth_tasks::TaskExecutor; +use std::{path::PathBuf, sync::Arc}; use tracing::*; -use crate::{args::NetworkArgs, macros::block_executor, utils::get_single_header}; - /// `reth debug merkle` command #[derive(Debug, Parser)] pub struct Command { @@ -96,7 +94,7 @@ impl> Command { ) .await?; - let executor_provider = block_executor!(provider_factory.chain_spec()); + let executor_provider = EthExecutorProvider::ethereum(provider_factory.chain_spec()); // Initialize the fetch client info!(target: "reth::cli", target_block_number=self.to, "Downloading tip of block range"); diff --git a/bin/reth/src/commands/debug_cmd/replay_engine.rs b/bin/reth/src/commands/debug_cmd/replay_engine.rs index 0130b78b0af8..04d1c01b8026 100644 --- a/bin/reth/src/commands/debug_cmd/replay_engine.rs +++ b/bin/reth/src/commands/debug_cmd/replay_engine.rs @@ -1,5 +1,4 @@ -use std::{path::PathBuf, sync::Arc, time::Duration}; - +use crate::args::NetworkArgs; use clap::Parser; use eyre::Context; use reth_basic_payload_builder::{BasicPayloadJobGenerator, BasicPayloadJobGeneratorConfig}; @@ -19,6 +18,7 @@ use reth_engine_util::engine_store::{EngineMessageStore, StoredEngineApiMessage} use reth_fs_util as fs; use reth_network::{BlockDownloaderProvider, NetworkHandle}; use reth_network_api::NetworkInfo; +use reth_node_ethereum::EthExecutorProvider; use reth_payload_builder::{PayloadBuilderHandle, PayloadBuilderService}; use reth_provider::{ providers::BlockchainProvider, CanonStateSubscriptions, ChainSpecProvider, ProviderFactory, @@ -28,11 +28,10 @@ use reth_stages::Pipeline; use reth_static_file::StaticFileProducer; use reth_tasks::TaskExecutor; use reth_transaction_pool::noop::NoopTransactionPool; +use std::{path::PathBuf, sync::Arc, time::Duration}; use tokio::sync::oneshot; use tracing::*; -use crate::{args::NetworkArgs, macros::block_executor}; - /// `reth debug replay-engine` command /// This script will read stored engine API messages and replay them by the timestamp. /// It does not require @@ -82,7 +81,7 @@ impl> Command { let consensus: Arc = Arc::new(EthBeaconConsensus::new(provider_factory.chain_spec())); - let executor = block_executor!(provider_factory.chain_spec()); + let executor = EthExecutorProvider::ethereum(provider_factory.chain_spec()); // Configure blockchain tree let tree_externals = @@ -111,15 +110,8 @@ impl> Command { .await?; // Set up payload builder - #[cfg(not(feature = "optimism"))] let payload_builder = reth_ethereum_payload_builder::EthereumPayloadBuilder::default(); - // Optimism's payload builder is implemented on the OptimismPayloadBuilder type. - #[cfg(feature = "optimism")] - let payload_builder = reth_node_optimism::OptimismPayloadBuilder::new( - reth_node_optimism::OptimismEvmConfig::default(), - ); - let payload_generator = BasicPayloadJobGenerator::with_builder( blockchain_db.clone(), NoopTransactionPool::default(), @@ -129,13 +121,6 @@ impl> Command { payload_builder, ); - #[cfg(feature = "optimism")] - let (payload_service, payload_builder): ( - _, - PayloadBuilderHandle, - ) = PayloadBuilderService::new(payload_generator, blockchain_db.canonical_state_stream()); - - #[cfg(not(feature = "optimism"))] let (payload_service, payload_builder): ( _, PayloadBuilderHandle, diff --git a/bin/reth/src/lib.rs b/bin/reth/src/lib.rs index 020f6458089a..654ac1201a94 100644 --- a/bin/reth/src/lib.rs +++ b/bin/reth/src/lib.rs @@ -27,11 +27,11 @@ html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256", issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/" )] +#![cfg_attr(not(test), warn(unused_crate_dependencies))] #![cfg_attr(docsrs, feature(doc_cfg, doc_auto_cfg))] pub mod cli; pub mod commands; -mod macros; /// Re-exported utils. pub mod utils { diff --git a/bin/reth/src/macros.rs b/bin/reth/src/macros.rs deleted file mode 100644 index 7ff81a0f9058..000000000000 --- a/bin/reth/src/macros.rs +++ /dev/null @@ -1,20 +0,0 @@ -//! Helper macros - -/// Creates the block executor type based on the configured feature. -/// -/// Note(mattsse): This is incredibly horrible and will be replaced -#[cfg(not(feature = "optimism"))] -macro_rules! block_executor { - ($chain_spec:expr) => { - reth_node_ethereum::EthExecutorProvider::ethereum($chain_spec) - }; -} - -#[cfg(feature = "optimism")] -macro_rules! block_executor { - ($chain_spec:expr) => { - reth_node_optimism::OpExecutorProvider::optimism($chain_spec) - }; -} - -pub(crate) use block_executor; diff --git a/bin/reth/src/main.rs b/bin/reth/src/main.rs index 1040a28a43b1..6952d117b76e 100644 --- a/bin/reth/src/main.rs +++ b/bin/reth/src/main.rs @@ -5,9 +5,6 @@ #[global_allocator] static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; -#[cfg(all(feature = "optimism", not(test)))] -compile_error!("Cannot build the `reth` binary with the `optimism` feature flag enabled. Did you mean to build `op-reth`?"); - /// clap [Args] for Engine related arguments. use clap::Args; @@ -20,7 +17,6 @@ pub struct EngineArgs { pub experimental: bool, } -#[cfg(not(feature = "optimism"))] fn main() { use clap::Parser; use reth::{args::utils::DefaultChainSpecParser, cli::Cli}; diff --git a/bin/reth/src/optimism.rs b/bin/reth/src/optimism.rs deleted file mode 100644 index cab039a5bc51..000000000000 --- a/bin/reth/src/optimism.rs +++ /dev/null @@ -1,84 +0,0 @@ -#![allow(missing_docs, rustdoc::missing_crate_level_docs)] - -use clap::Parser; -use reth::cli::Cli; -use reth_node_builder::EngineNodeLauncher; -use reth_node_optimism::{args::RollupArgs, node::OptimismAddOns, OptimismNode}; -use reth_optimism_rpc::eth::rpc::SequencerClient; -use reth_provider::providers::BlockchainProvider2; - -// We use jemalloc for performance reasons -#[cfg(all(feature = "jemalloc", unix))] -#[global_allocator] -static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - -#[cfg(not(feature = "optimism"))] -compile_error!("Cannot build the `op-reth` binary with the `optimism` feature flag disabled. Did you mean to build `reth`?"); - -#[cfg(feature = "optimism")] -fn main() { - use reth::args::utils::DefaultChainSpecParser; - - reth_cli_util::sigsegv_handler::install(); - - // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. - if std::env::var_os("RUST_BACKTRACE").is_none() { - std::env::set_var("RUST_BACKTRACE", "1"); - } - - if let Err(err) = - Cli::::parse().run(|builder, rollup_args| async move { - let enable_engine2 = rollup_args.experimental; - let sequencer_http_arg = rollup_args.sequencer_http.clone(); - match enable_engine2 { - true => { - let handle = builder - .with_types_and_provider::>() - .with_components(OptimismNode::components(rollup_args)) - .with_add_ons::() - .extend_rpc_modules(move |ctx| { - // register sequencer tx forwarder - if let Some(sequencer_http) = sequencer_http_arg { - ctx.registry - .eth_api() - .set_sequencer_client(SequencerClient::new(sequencer_http)); - } - - Ok(()) - }) - .launch_with_fn(|builder| { - let launcher = EngineNodeLauncher::new( - builder.task_executor().clone(), - builder.config().datadir(), - ); - builder.launch_with(launcher) - }) - .await?; - - handle.node_exit_future.await - } - false => { - let handle = builder - .node(OptimismNode::new(rollup_args.clone())) - .extend_rpc_modules(move |ctx| { - // register sequencer tx forwarder - if let Some(sequencer_http) = sequencer_http_arg { - ctx.registry - .eth_api() - .set_sequencer_client(SequencerClient::new(sequencer_http)); - } - - Ok(()) - }) - .launch() - .await?; - - handle.node_exit_future.await - } - } - }) - { - eprintln!("Error: {err:?}"); - std::process::exit(1); - } -} diff --git a/crates/optimism/bin/Cargo.toml b/crates/optimism/bin/Cargo.toml new file mode 100644 index 000000000000..97c001f74451 --- /dev/null +++ b/crates/optimism/bin/Cargo.toml @@ -0,0 +1,42 @@ +[package] +name = "op-reth" +version.workspace = true +edition.workspace = true +rust-version.workspace = true +license.workspace = true +homepage.workspace = true +repository.workspace = true +exclude.workspace = true + +[dependencies] +reth-node-builder.workspace = true +reth-cli-util.workspace = true +reth-optimism-cli.workspace = true +reth-provider.workspace = true +reth-optimism-rpc.workspace = true +reth-node-optimism.workspace = true + +clap = { workspace = true, features = ["derive", "env"] } + +[target.'cfg(unix)'.dependencies] +tikv-jemallocator = { workspace = true, optional = true } + +[lints] +workspace = true + +[features] +default = ["jemalloc"] + +jemalloc = ["dep:tikv-jemallocator"] +jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"] + +asm-keccak = ["reth-node-optimism/asm-keccak"] + +optimism = [ + "reth-optimism-cli/optimism", + "reth-node-optimism/optimism", +] + +[[bin]] +name = "op-reth" +path = "src/main.rs" diff --git a/crates/optimism/bin/src/main.rs b/crates/optimism/bin/src/main.rs new file mode 100644 index 000000000000..7b91bca6a51c --- /dev/null +++ b/crates/optimism/bin/src/main.rs @@ -0,0 +1,79 @@ +#![cfg_attr(not(test), warn(unused_crate_dependencies))] +#![allow(missing_docs, rustdoc::missing_crate_level_docs)] +// The `optimism` feature must be enabled to use this crate. +#![cfg(feature = "optimism")] + +use clap::Parser; +use reth_node_builder::EngineNodeLauncher; +use reth_node_optimism::{args::RollupArgs, node::OptimismAddOns, OptimismNode}; +use reth_optimism_cli::Cli; +use reth_optimism_rpc::eth::rpc::SequencerClient; +use reth_provider::providers::BlockchainProvider2; + +// We use jemalloc for performance reasons +#[cfg(all(feature = "jemalloc", unix))] +#[global_allocator] +static ALLOC: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +fn main() { + reth_cli_util::sigsegv_handler::install(); + + // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided. + if std::env::var_os("RUST_BACKTRACE").is_none() { + std::env::set_var("RUST_BACKTRACE", "1"); + } + + if let Err(err) = Cli::::parse().run(|builder, rollup_args| async move { + let enable_engine2 = rollup_args.experimental; + let sequencer_http_arg = rollup_args.sequencer_http.clone(); + match enable_engine2 { + true => { + let handle = builder + .with_types_and_provider::>() + .with_components(OptimismNode::components(rollup_args)) + .with_add_ons::() + .extend_rpc_modules(move |ctx| { + // register sequencer tx forwarder + if let Some(sequencer_http) = sequencer_http_arg { + ctx.registry + .eth_api() + .set_sequencer_client(SequencerClient::new(sequencer_http)); + } + + Ok(()) + }) + .launch_with_fn(|builder| { + let launcher = EngineNodeLauncher::new( + builder.task_executor().clone(), + builder.config().datadir(), + ); + builder.launch_with(launcher) + }) + .await?; + + handle.node_exit_future.await + } + false => { + let handle = builder + .node(OptimismNode::new(rollup_args.clone())) + .extend_rpc_modules(move |ctx| { + // register sequencer tx forwarder + if let Some(sequencer_http) = sequencer_http_arg { + ctx.registry + .eth_api() + .set_sequencer_client(SequencerClient::new(sequencer_http)); + } + + Ok(()) + }) + .launch() + .await?; + + handle.node_exit_future.await + } + } + }) { + eprintln!("Error: {err:?}"); + std::process::exit(1); + } +} diff --git a/crates/optimism/chainspec/src/lib.rs b/crates/optimism/chainspec/src/lib.rs index fd290448d18b..952fee7625dc 100644 --- a/crates/optimism/chainspec/src/lib.rs +++ b/crates/optimism/chainspec/src/lib.rs @@ -28,7 +28,7 @@ use derive_more::{Constructor, Deref, Into}; use reth_chainspec::ChainSpec; /// OP stack chain spec type. -#[derive(Debug, Deref, Into, Constructor)] +#[derive(Debug, Clone, Deref, Into, Constructor)] pub struct OpChainSpec { /// [`ChainSpec`]. pub inner: ChainSpec, diff --git a/crates/optimism/cli/Cargo.toml b/crates/optimism/cli/Cargo.toml index 9f4be1586ba4..6eb896edd432 100644 --- a/crates/optimism/cli/Cargo.toml +++ b/crates/optimism/cli/Cargo.toml @@ -37,6 +37,10 @@ reth-errors.workspace = true reth-config.workspace = true reth-evm-optimism.workspace = true reth-cli.workspace = true +reth-cli-runner.workspace = true +reth-node-builder.workspace = true +reth-tracing.workspace = true + # eth alloy-primitives.workspace = true @@ -68,4 +72,4 @@ reth-db-common.workspace = true "reth-primitives/optimism", "reth-evm-optimism/optimism", "reth-provider/optimism", - ] \ No newline at end of file + ] diff --git a/crates/optimism/cli/src/chainspec.rs b/crates/optimism/cli/src/chainspec.rs index af4cbc8956e1..03d78cba0a3c 100644 --- a/crates/optimism/cli/src/chainspec.rs +++ b/crates/optimism/cli/src/chainspec.rs @@ -1,5 +1,6 @@ use std::sync::Arc; +use reth_chainspec::ChainSpec; use reth_cli::chainspec::ChainSpecParser; use reth_node_core::args::utils::parse_custom_chain_spec; use reth_optimism_chainspec::{ @@ -26,7 +27,7 @@ fn chain_value_parser(s: &str) -> eyre::Result, eyre::Error> { pub struct OpChainSpecParser; impl ChainSpecParser for OpChainSpecParser { - type ChainSpec = OpChainSpec; + type ChainSpec = ChainSpec; const SUPPORTED_CHAINS: &'static [&'static str] = &[ "dev", @@ -38,8 +39,8 @@ impl ChainSpecParser for OpChainSpecParser { "base-sepolia", ]; - fn parse(s: &str) -> eyre::Result> { - chain_value_parser(s) + fn parse(s: &str) -> eyre::Result> { + chain_value_parser(s).map(|s| Arc::new(Arc::unwrap_or_clone(s).inner)) } } diff --git a/crates/optimism/cli/src/lib.rs b/crates/optimism/cli/src/lib.rs index 2314fe36466c..9019f2f8ce2f 100644 --- a/crates/optimism/cli/src/lib.rs +++ b/crates/optimism/cli/src/lib.rs @@ -34,15 +34,22 @@ use std::{ffi::OsString, fmt, sync::Arc}; use chainspec::OpChainSpecParser; use clap::{command, value_parser, Parser}; use commands::Commands; +use futures_util::Future; use reth_chainspec::ChainSpec; use reth_cli::chainspec::ChainSpecParser; use reth_cli_commands::node::NoArgs; +use reth_cli_runner::CliRunner; +use reth_db::DatabaseEnv; +use reth_evm_optimism::OpExecutorProvider; +use reth_node_builder::{NodeBuilder, WithLaunchContext}; use reth_node_core::{ args::{utils::chain_help, LogArgs}, version::{LONG_VERSION, SHORT_VERSION}, }; +use reth_tracing::FileWorkerGuard; +use tracing::info; -/// The main reth cli interface. +/// The main op-reth cli interface. /// /// This is the entrypoint to the executable. #[derive(Debug, Parser)] @@ -100,3 +107,52 @@ impl Cli { Self::try_parse_from(itr) } } + +impl Cli { + /// Execute the configured cli command. + /// + /// This accepts a closure that is used to launch the node via the + /// [`NodeCommand`](reth_cli_commands::node::NodeCommand). + pub fn run(mut self, launcher: L) -> eyre::Result<()> + where + L: FnOnce(WithLaunchContext>>, Ext) -> Fut, + Fut: Future>, + { + // add network name to logs dir + self.logs.log_file_directory = + self.logs.log_file_directory.join(self.chain.chain.to_string()); + + let _guard = self.init_tracing()?; + info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory); + + let runner = CliRunner::default(); + match self.command { + Commands::Node(command) => { + runner.run_command_until_exit(|ctx| command.execute(ctx, launcher)) + } + Commands::Init(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::InitState(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::ImportOp(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::ImportReceiptsOp(command) => { + runner.run_blocking_until_ctrl_c(command.execute()) + } + Commands::DumpGenesis(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::Db(command) => runner.run_blocking_until_ctrl_c(command.execute()), + Commands::Stage(command) => runner + .run_command_until_exit(|ctx| command.execute(ctx, OpExecutorProvider::optimism)), + Commands::P2P(command) => runner.run_until_ctrl_c(command.execute()), + Commands::Config(command) => runner.run_until_ctrl_c(command.execute()), + Commands::Recover(command) => runner.run_command_until_exit(|ctx| command.execute(ctx)), + Commands::Prune(command) => runner.run_until_ctrl_c(command.execute()), + } + } + + /// Initializes tracing with the configured options. + /// + /// If file logging is enabled, this function returns a guard that must be kept alive to ensure + /// that all logs are flushed to disk. + pub fn init_tracing(&self) -> eyre::Result> { + let guard = self.logs.init_tracing()?; + Ok(guard) + } +} diff --git a/crates/optimism/node/Cargo.toml b/crates/optimism/node/Cargo.toml index 670b392dd647..3e963a238ee0 100644 --- a/crates/optimism/node/Cargo.toml +++ b/crates/optimism/node/Cargo.toml @@ -80,4 +80,5 @@ optimism = [ "reth-auto-seal-consensus/optimism", "reth-optimism-rpc/optimism" ] +asm-keccak = ["reth-primitives/asm-keccak"] test-utils = ["reth-node-builder/test-utils"]