diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 9527a5f..676ad1b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -4,7 +4,6 @@ on: push: branches: [ "master" ] pull_request: - branches: [ "master" ] env: CARGO_TERM_COLOR: always diff --git a/.github/workflows/update-fixtures.yml b/.github/workflows/update-fixtures.yml new file mode 100644 index 0000000..7adba9c --- /dev/null +++ b/.github/workflows/update-fixtures.yml @@ -0,0 +1,38 @@ +name: Update Test Fixtures + +on: + schedule: + # Runs on the first day of every month at 00:00 UTC + - cron: '0 0 1 * *' + # Allow manual triggering + workflow_dispatch: + +jobs: + update-fixtures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Install Rust + uses: dtolnay/rust-toolchain@stable + + - name: Cache dependencies + uses: Swatinem/rust-cache@v2 + + - name: Build and run update script + run: cargo run --release + env: + PANDAOPS_CLIENT_ID: ${{ secrets.PANDAOPS_CLIENT_ID }} + PANDAOPS_CLIENT_SECRET: ${{ secrets.PANDAOPS_CLIENT_SECRET }} + + - name: Create Pull Request + uses: peter-evans/create-pull-request@v5 + with: + commit-message: 'chore: update test fixtures for month' + title: 'Update test fixtures [automated]' + body: | + This PR updates the test fixtures with the latest data. + + Generated automatically by GitHub Actions. + branch: update-fixtures + base: master diff --git a/.gitignore b/.gitignore index a3d72b1..5bfcf40 100644 --- a/.gitignore +++ b/.gitignore @@ -18,4 +18,8 @@ Cargo.lock # misc .git .env -venv \ No newline at end of file +venv + +# Added by cargo + +/target diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..2b6582a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "fixture-updater" +version = "0.1.0" +edition = "2021" + +[dependencies] +anyhow = "1.0" +chrono = "0.4" +dotenv = "0.15" +ethportal-api = "0.4.0" +reqwest = { version = "0.11", features = ["json"] } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +serde_yaml = "0.9" +ssz_types = "0.8.0" +tokio = { version = "1.0", features = ["full"] } +tracing = "0.1.36" +tracing-subscriber = "0.3" diff --git a/src/client.rs b/src/client.rs new file mode 100644 index 0000000..dff3418 --- /dev/null +++ b/src/client.rs @@ -0,0 +1,246 @@ +use anyhow::{ensure, Result}; +use ethportal_api::{ + consensus::{ + beacon_state::BeaconStateDeneb, + historical_summaries::{HistoricalSummariesStateProof, HistoricalSummariesWithProof}, + }, + light_client::{ + bootstrap::LightClientBootstrapDeneb, finality_update::LightClientFinalityUpdateDeneb, + optimistic_update::LightClientOptimisticUpdateDeneb, update::LightClientUpdateDeneb, + }, + types::{ + consensus::fork::ForkName, + content_key::beacon::{ + HistoricalSummariesWithProofKey, LightClientBootstrapKey, LightClientFinalityUpdateKey, + LightClientOptimisticUpdateKey, LightClientUpdatesByRangeKey, + }, + content_value::beacon::{ + ForkVersionedHistoricalSummariesWithProof, ForkVersionedLightClientUpdate, + LightClientUpdatesByRange, + }, + }, + utils::bytes::hex_decode, + BeaconContentKey, BeaconContentValue, +}; +use reqwest::Client; +use serde_yaml::Value; +use ssz_types::VariableList; +use tracing::info; + +use crate::fixture::FixtureEntry; + +// Pandaops consensus endpoint. +const BASE_CL_ENDPOINT: &str = "https://nimbus-geth.mainnet.eu1.ethpandaops.io"; +// The number of slots in an epoch. +const SLOTS_PER_EPOCH: u64 = 32; +/// The number of slots in a sync committee period. +const SLOTS_PER_PERIOD: u64 = SLOTS_PER_EPOCH * 256; +// Beacon chain mainnet genesis time: Tue Dec 01 2020 12:00:23 GMT+0000 +pub const BEACON_GENESIS_TIME: u64 = 1606824023; +/// The historical summaries proof always has a length of 5 hashes. +const HISTORICAL_SUMMARIES_PROOF_LENGTH: usize = 5; + +pub struct BeaconClient { + client: Client, +} + +impl BeaconClient { + pub fn new(client_id: &str, client_secret: &str) -> Result { + info!("Creating new BeaconClient"); + let mut headers = reqwest::header::HeaderMap::new(); + headers.insert( + reqwest::header::CONTENT_TYPE, + reqwest::header::HeaderValue::from_static("application/json"), + ); + headers.insert( + "CF-Access-Client-ID", + reqwest::header::HeaderValue::from_str(client_id)?, + ); + headers.insert( + "CF-Access-Client-Secret", + reqwest::header::HeaderValue::from_str(client_secret)?, + ); + + let client = Client::builder() + .default_headers(headers) + .build() + .map_err(|_| anyhow::anyhow!("Failed to build HTTP client"))?; + + Ok(Self { client }) + } + + async fn get_finalized_root(&self) -> Result { + info!("Fetching finalized root"); + let url = format!("{}/eth/v1/beacon/blocks/finalized/root", BASE_CL_ENDPOINT); + let response = self.client.get(url).send().await?; + let json_data = response.error_for_status()?.json::().await?; + Ok(json_data["data"]["root"].as_str().unwrap().to_string()) + } + + pub async fn get_light_client_bootstrap(&self) -> Result { + info!("Fetching light client bootstrap data"); + let block_root = self.get_finalized_root().await?; + let url = format!( + "{}/eth/v1/beacon/light_client/bootstrap/{}", + BASE_CL_ENDPOINT, block_root + ); + let response = self.client.get(url).send().await?; + let json_data = response + .error_for_status()? + .json::() + .await?; + let result: serde_json::Value = json_data["data"].clone(); + let content_key = BeaconContentKey::LightClientBootstrap(LightClientBootstrapKey { + block_hash: <[u8; 32]>::try_from(hex_decode(&block_root)?).unwrap(), + }); + let bootstrap: LightClientBootstrapDeneb = serde_json::from_value(result.clone())?; + let content_value = BeaconContentValue::LightClientBootstrap(bootstrap.into()); + + Ok(FixtureEntry::new( + "Light Client Bootstrap", + content_key, + content_value, + )) + } + + pub async fn get_light_client_finality_update(&self) -> Result { + info!("Fetching light client finality update"); + let url = format!( + "{}/eth/v1/beacon/light_client/finality_update", + BASE_CL_ENDPOINT + ); + let response = self.client.get(url).send().await?; + let json_data = response + .error_for_status()? + .json::() + .await?; + let result: serde_json::Value = json_data["data"].clone(); + let update: LightClientFinalityUpdateDeneb = serde_json::from_value(result.clone())?; + let new_finalized_slot = update.finalized_header.beacon.slot; + let content_key = BeaconContentKey::LightClientFinalityUpdate( + LightClientFinalityUpdateKey::new(new_finalized_slot), + ); + let content_value = BeaconContentValue::LightClientFinalityUpdate(update.into()); + + Ok(FixtureEntry::new( + "Light Client Finality Update", + content_key, + content_value, + )) + } + + pub async fn get_light_client_optimistic_update(&self) -> Result { + info!("Fetching light client optimistic update"); + let url = format!( + "{}/eth/v1/beacon/light_client/optimistic_update", + BASE_CL_ENDPOINT + ); + let response = self.client.get(url).send().await?; + let json_data = response + .error_for_status()? + .json::() + .await?; + let result: serde_json::Value = json_data["data"].clone(); + let update: LightClientOptimisticUpdateDeneb = serde_json::from_value(result.clone())?; + let content_key = BeaconContentKey::LightClientOptimisticUpdate( + LightClientOptimisticUpdateKey::new(update.signature_slot), + ); + let content_value = BeaconContentValue::LightClientOptimisticUpdate(update.into()); + + Ok(FixtureEntry::new( + "Light Client Optimistic Update", + content_key, + content_value, + )) + } + + pub async fn get_light_client_updates_by_range(&self) -> Result { + info!("Fetching light client updates by range"); + let start_period = get_start_period().await?; + let count = 1; + + let url = format!( + "{}/eth/v1/beacon/light_client/updates?start_period={}&count={}", + BASE_CL_ENDPOINT, start_period, count + ); + let response = self.client.get(url).send().await?; + let json_data = response + .error_for_status()? + .json::() + .await?; + let update: LightClientUpdateDeneb = serde_json::from_value(json_data[0]["data"].clone())?; + let fork_versioned_update = ForkVersionedLightClientUpdate { + fork_name: ForkName::Deneb, + update: update.into(), + }; + let content_value = BeaconContentValue::LightClientUpdatesByRange( + LightClientUpdatesByRange(VariableList::from(vec![fork_versioned_update])), + ); + let content_key = + BeaconContentKey::LightClientUpdatesByRange(LightClientUpdatesByRangeKey { + start_period, + count, + }); + + Ok(FixtureEntry::new( + "Light Client Updates By Range", + content_key, + content_value, + )) + } + + pub async fn get_historical_summaries_with_proof(&self) -> Result { + info!("Fetching historical summaries with proof"); + let url = format!("{}/eth/v2/debug/beacon/states/finalized", BASE_CL_ENDPOINT); + let response = self.client.get(url).send().await?; + let json_data = response.error_for_status()?.text().await?; + let beacon_state_val: serde_json::Value = serde_json::from_str(&json_data)?; + let beacon_state: BeaconStateDeneb = + serde_json::from_value(beacon_state_val["data"].clone())?; + let state_epoch = beacon_state.slot / SLOTS_PER_EPOCH; + let historical_summaries_proof = beacon_state.build_historical_summaries_proof(); + + ensure!( + historical_summaries_proof.len() == HISTORICAL_SUMMARIES_PROOF_LENGTH, + "Historical summaries proof length is not 5", + ); + + let historical_summaries = beacon_state.historical_summaries; + let historical_summaries_with_proof = ForkVersionedHistoricalSummariesWithProof { + fork_name: ForkName::Deneb, + historical_summaries_with_proof: HistoricalSummariesWithProof { + epoch: state_epoch, + historical_summaries, + proof: HistoricalSummariesStateProof::from(historical_summaries_proof), + }, + }; + let content_key = + BeaconContentKey::HistoricalSummariesWithProof(HistoricalSummariesWithProofKey { + epoch: state_epoch, + }); + let content_value = + BeaconContentValue::HistoricalSummariesWithProof(historical_summaries_with_proof); + + Ok(FixtureEntry::new( + "Historical Summaries With Proof", + content_key, + content_value, + )) + } +} + +async fn get_start_period() -> Result { + let now = std::time::SystemTime::now(); + let expected_current_period = + expected_current_slot(BEACON_GENESIS_TIME, now) / SLOTS_PER_PERIOD; + Ok(expected_current_period) +} + +fn expected_current_slot(genesis_time: u64, now: std::time::SystemTime) -> u64 { + let now = now + .duration_since(std::time::UNIX_EPOCH) + .expect("Time went backwards"); + let since_genesis = now - std::time::Duration::from_secs(genesis_time); + + since_genesis.as_secs() / 12 +} diff --git a/src/fixture.rs b/src/fixture.rs new file mode 100644 index 0000000..e631e7c --- /dev/null +++ b/src/fixture.rs @@ -0,0 +1,38 @@ +use chrono::{DateTime, Utc}; +use ethportal_api::{ + types::content_value::ContentValue, utils::bytes::hex_encode, BeaconContentKey, + BeaconContentValue, OverlayContentKey, +}; + +#[derive(Debug)] +pub struct FixtureEntry { + data_type: &'static str, + content_key: BeaconContentKey, + content_value: BeaconContentValue, + updated_at: DateTime, +} + +impl FixtureEntry { + pub fn new( + data_type: &'static str, + content_key: BeaconContentKey, + content_value: BeaconContentValue, + ) -> Self { + Self { + data_type, + content_key, + content_value, + updated_at: Utc::now(), + } + } + + pub fn to_yaml_string(&self) -> String { + format!( + "# {}\n# Last updated: {}\n- content_key: \"{}\"\n content_value: \"{}\"\n", + self.data_type, + self.updated_at.format("%Y-%m-%d"), + self.content_key.to_hex(), + hex_encode(self.content_value.encode()), + ) + } +} diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..3f91822 --- /dev/null +++ b/src/main.rs @@ -0,0 +1,80 @@ +mod client; +mod fixture; + +use std::{env, fs, path::PathBuf}; + +use anyhow::{Context, Result}; +use client::BeaconClient; +use tracing::info; + +// This script fetches various Ethereum beacon chain light client data and historical summaries +// from a Consensus Layer endpoint and saves them as YAML test fixtures. +// +// # Usage +// ```bash +// # Set required environment variables +// export PANDAOPS_CLIENT_ID="your_client_id" +// export PANDAOPS_CLIENT_SECRET="your_client_secret" +// +// cargo run +// ``` +// +// This script is run once a month automatically as a github workflow, and +// creates a pr with the updated test fixtures. +// You can trigger this workflow to run manually by going to the actions tab in Github. +// +// This code is largely based off the portal-bridge codebase. +// todo: add bootstraps older than 4 months for tests + +#[tokio::main] +async fn main() -> Result<()> { + tracing_subscriber::fmt::init(); + info!("Starting fixture update process"); + dotenv::dotenv().ok(); + + let client_id = env::var("PANDAOPS_CLIENT_ID").context("PANDAOPS_CLIENT_ID not found")?; + let client_secret = + env::var("PANDAOPS_CLIENT_SECRET").context("PANDAOPS_CLIENT_SECRET not found")?; + + let client = BeaconClient::new(&client_id, &client_secret)?; + // Update latest test fixtures + let latest_data = fetch_latest_data(&client).await?; + update_fixture(&latest_data, "test_data.yaml")?; + + info!("Successfully updated fixture"); + Ok(()) +} + +async fn fetch_latest_data(client: &BeaconClient) -> Result { + let bootstrap = client.get_light_client_bootstrap().await?; + let finality = client.get_light_client_finality_update().await?; + let optimistic = client.get_light_client_optimistic_update().await?; + let updates = client.get_light_client_updates_by_range().await?; + let historical = client.get_historical_summaries_with_proof().await?; + + let yaml_str = format!( + "# Deneb test data for Portal Hive Beacon tests\n# Last updated: {}\n\n{}\n{}\n{}\n{}\n{}\n", + chrono::Utc::now().format("%Y-%m-%d"), + bootstrap.to_yaml_string(), + finality.to_yaml_string(), + optimistic.to_yaml_string(), + updates.to_yaml_string(), + historical.to_yaml_string(), + ); + + Ok(yaml_str) +} + +fn update_fixture(data: &str, file_name: &str) -> Result<()> { + let fixture_path = PathBuf::from("tests/mainnet/beacon_chain/hive").join(file_name); + + if !fixture_path.exists() { + return Err(anyhow::anyhow!( + "Fixture path not found: {:?}", + fixture_path + )); + } + + fs::write(&fixture_path, data)?; + Ok(()) +}