feat(starknet_l1_provider): add scraper

config/sequencer/default_config.json

@@ -1,4 +1,14 @@
 {
+  "base_layer_config.node_url": {
+    "description": "A required param! Ethereum node URL. A schema to match to Infura node: https://mainnet.infura.io/v3/<your_api_key>, but any other node can be used.",
+    "param_type": "String",
+    "privacy": "Private"
+  },
+  "base_layer_config.starknet_contract_address": {
+    "description": "Starknet contract address in ethereum.",
+    "privacy": "Public",
+    "value": "0xc662c410C0ECf747543f5bA90660f6ABeBD9C8c4"
+  },
   "batcher_config.block_builder_config.bouncer_config.block_max_capacity.builtin_count.add_mod": {
     "description": "Max number of add mod builtin usage in a block.",
     "privacy": "Public",

crates/papyrus_base_layer/Cargo.toml

@@ -32,6 +32,7 @@ tempfile = { workspace = true, optional = true }
 thiserror.workspace = true
 tokio = { workspace = true, features = ["full", "sync"] }
 url = { workspace = true, features = ["serde"] }
+validator.workspace = true
 
 [dev-dependencies]
 ethers-core.workspace = true

crates/papyrus_base_layer/src/constants.rs

@@ -0,0 +1 @@
+pub type EventIdentifier = &'static str;

crates/papyrus_base_layer/src/ethereum_base_layer_contract.rs

@@ -18,6 +18,7 @@ use starknet_api::block::{BlockHash, BlockHashAndNumber, BlockNumber};
 use starknet_api::hash::StarkHash;
 use starknet_api::StarknetApiError;
 use url::Url;
+use validator::Validate;
 
 use crate::{BaseLayerContract, L1Event};
 
@@ -116,7 +117,7 @@ pub enum EthereumBaseLayerError {
     UnhandledL1Event(alloy_primitives::Log),
 }
 
-#[derive(Clone, Debug, Deserialize, PartialEq, Serialize)]
+#[derive(Clone, Debug, Deserialize, PartialEq, Serialize, Validate)]
 pub struct EthereumBaseLayerConfig {
     pub node_url: Url,
     pub starknet_contract_address: EthereumContractAddress,

crates/papyrus_base_layer/src/lib.rs

@@ -1,10 +1,14 @@
+use std::error::Error;
+use std::fmt::{Debug, Display};
+
 use async_trait::async_trait;
 use serde::{Deserialize, Serialize};
 use starknet_api::block::BlockHashAndNumber;
 use starknet_api::core::{ContractAddress, EntryPointSelector, EthAddress, Nonce};
 use starknet_api::transaction::fields::{Calldata, Fee};
 use starknet_api::transaction::L1HandlerTransaction;
 
+pub mod constants;
 pub mod ethereum_base_layer_contract;
 
 pub(crate) mod eth_events;
@@ -18,7 +22,7 @@ mod base_layer_test;
 /// Interface for getting data from the Starknet base contract.
 #[async_trait]
 pub trait BaseLayerContract {
-    type Error;
+    type Error: Error + Display + Debug;
 
     /// Get the latest Starknet block that is proved on the base layer.
     /// Optionally, require minimum confirmations.

crates/starknet_integration_tests/src/config_utils.rs

@@ -3,13 +3,13 @@ use std::io::Write;
 use std::path::PathBuf;
 
 use papyrus_config::dumping::{combine_config_map_and_pointers, SerializeConfig};
-use serde_json::{Map, Value};
+use serde_json::Value;
 use starknet_sequencer_node::config::node_config::{
     SequencerNodeConfig,
     CONFIG_NON_POINTERS_WHITELIST,
     CONFIG_POINTERS,
 };
-use starknet_sequencer_node::config::test_utils::RequiredParams;
+use starknet_sequencer_node::config::test_utils::{config_to_preset, RequiredParams};
 use tracing::info;
 
 // TODO(Tsabary): Move here all config-related functions from "integration_test_utils.rs".
@@ -56,43 +56,6 @@ fn dump_json_data(json_data: Value, path: &str, dir: PathBuf) -> PathBuf {
     temp_dir_path
 }
 
-/// Transforms a nested JSON dictionary object into a simplified JSON dictionary object by
-/// extracting specific values from the inner dictionaries.
-///
-/// # Parameters
-/// - `config_map`: A reference to a `serde_json::Value` that must be a JSON dictionary object. Each
-///   key in the object maps to another JSON dictionary object.
-///
-/// # Returns
-/// - A `serde_json::Value` dictionary object where:
-///   - Each key is preserved from the top-level dictionary.
-///   - Each value corresponds to the `"value"` field of the nested JSON dictionary under the
-///     original key.
-///
-/// # Panics
-/// This function panics if the provided `config_map` is not a JSON dictionary object.
-fn config_to_preset(config_map: &Value) -> Value {
-    // Ensure the config_map is a JSON object.
-    if let Value::Object(map) = config_map {
-        let mut result = Map::new();
-
-        for (key, value) in map {
-            if let Value::Object(inner_map) = value {
-                // Extract the value.
-                if let Some(inner_value) = inner_map.get("value") {
-                    // Add it to the result map
-                    result.insert(key.clone(), inner_value.clone());
-                }
-            }
-        }
-
-        // Return the transformed result as a JSON object.
-        Value::Object(result)
-    } else {
-        panic!("Config map is not a JSON object: {:?}", config_map);
-    }
-}
-
 /// Merges required parameters into an existing preset JSON object.
 ///
 /// # Parameters

crates/starknet_integration_tests/src/utils.rs

@@ -35,7 +35,10 @@ use starknet_monitoring_endpoint::config::MonitoringEndpointConfig;
 use starknet_sequencer_infra::test_utils::{get_available_socket, AvailablePorts};
 use starknet_sequencer_node::config::component_config::ComponentConfig;
 use starknet_sequencer_node::config::node_config::SequencerNodeConfig;
-use starknet_sequencer_node::config::test_utils::RequiredParams;
+use starknet_sequencer_node::config::test_utils::{
+    EthereumBaseLayerConfigRequiredParams,
+    RequiredParams,
+};
 use starknet_state_sync::config::StateSyncConfig;
 use starknet_types_core::felt::Felt;
 use url::Url;
@@ -95,6 +98,9 @@ pub async fn create_node_config(
             validator_id,
             // TODO(dvir): change this to real value when add recorder to integration tests.
             recorder_url: Url::parse("https://recorder_url").expect("The URL is valid"),
+            base_layer_config: EthereumBaseLayerConfigRequiredParams {
+                node_url: Url::parse("https://node_url").expect("The URL is valid"),
+            },
         },
     )
 }

crates/starknet_l1_provider/Cargo.toml

@@ -18,6 +18,7 @@ starknet_api.workspace = true
 starknet_l1_provider_types.workspace = true
 starknet_sequencer_infra.workspace = true
 thiserror.workspace = true
+tokio.workspace = true
 tracing.workspace = true
 validator.workspace = true
 

crates/starknet_l1_provider/src/l1_scraper.rs

@@ -0,0 +1,79 @@
+use std::time::Duration;
+
+use papyrus_base_layer::constants::EventIdentifier;
+use papyrus_base_layer::BaseLayerContract;
+use papyrus_config::converters::deserialize_seconds_to_duration;
+use papyrus_config::validators::validate_ascii;
+use serde::{Deserialize, Serialize};
+use starknet_api::core::ChainId;
+use starknet_l1_provider_types::SharedL1ProviderClient;
+use thiserror::Error;
+use tokio::time::sleep;
+use tracing::error;
+use validator::Validate;
+
+type L1ScraperResult<T, B> = Result<T, L1ScraperError<B>>;
+
+pub struct L1Scraper<B: BaseLayerContract> {
+    pub config: L1ScraperConfig,
+    pub base_layer: B,
+    pub last_block_number_processed: u64,
+    pub l1_provider_client: SharedL1ProviderClient,
+    _tracked_event_identifiers: Vec<EventIdentifier>,
+}
+
+impl<B: BaseLayerContract + Send + Sync> L1Scraper<B> {
+    pub fn new(
+        config: L1ScraperConfig,
+        l1_provider_client: SharedL1ProviderClient,
+        base_layer: B,
+        events_identifiers_to_track: &[EventIdentifier],
+    ) -> Self {
+        Self {
+            l1_provider_client,
+            base_layer,
+            last_block_number_processed: config.l1_block_to_start_scraping_from,
+            config,
+            _tracked_event_identifiers: events_identifiers_to_track.to_vec(),
+        }
+    }
+
+    pub async fn fetch_events(&mut self) -> L1ScraperResult<(), B> {
+        todo!()
+    }
+
+    async fn _run(&mut self) -> L1ScraperResult<(), B> {
+        loop {
+            sleep(self.config.polling_interval).await;
+            // TODO: retry.
+            self.fetch_events().await?;
+        }
+    }
+}
+
+#[derive(Clone, Debug, Serialize, Deserialize, Validate, PartialEq)]
+pub struct L1ScraperConfig {
+    pub l1_block_to_start_scraping_from: u64,
+    #[validate(custom = "validate_ascii")]
+    pub chain_id: ChainId,
+    pub finality: u64,
+    #[serde(deserialize_with = "deserialize_seconds_to_duration")]
+    pub polling_interval: Duration,
+}
+
+impl Default for L1ScraperConfig {
+    fn default() -> Self {
+        Self {
+            l1_block_to_start_scraping_from: 0,
+            chain_id: ChainId::Mainnet,
+            finality: 0,
+            polling_interval: Duration::from_secs(1),
+        }
+    }
+}
+
+#[derive(Error, Debug)]
+pub enum L1ScraperError<T: BaseLayerContract + Send + Sync> {
+    #[error("Base layer error: {0}")]
+    BaseLayer(T::Error),
+}

crates/starknet_l1_provider/src/lib.rs

@@ -1,5 +1,6 @@
 pub mod communication;
 pub mod errors;
+pub mod l1_scraper;
 
 #[cfg(test)]
 pub mod test_utils;