From 0b7a426946f0fe4590f0bec3c74df6f205ca498c Mon Sep 17 00:00:00 2001 From: Jimmy Chen Date: Thu, 10 Aug 2023 23:32:49 +1000 Subject: [PATCH] Builder flow for Deneb & Blobs (#4428) * Add Deneb builder flow types with generics * Update validator client `get_blinded_blocks` call to support Deneb * `produceBlindedBlock` endpoint updates: - Handle new Deneb BuilderBid response from builder endpoint (new BlindedBlobsBundle type) - Build BlockContents response (containing kzg_commitments, proof and blinded_blob_sidecars) * Appease Clippy lint * Partial implementation of submit blinded block & blobs. Refactor existing `BlobSidecar` related types to support blinded blobs. * Add associated types for BlockProposal * Rename `AbstractSidecar` to `Sidecar` * Remove blob cache as it's no longer necessary * Remove unnecessary enum variant * Clean up * Hanlde unblinded blobs and publish full block contents * Fix tests * Add local EL blobs caching in blinded flow * Remove BlockProposal and move associated Sidecar trait to AbstractExecPayload to simplify changes * add blob roots associated type * move raw blobs associated type to sidecar trait * Fix todos and improve error handling * Consolidate BlobsBundle from `execution_layer` into `consensus/types` * Rename RawBlobs, Blobs, and BlobRoots * Use `BlobRoots` type alias * Update error message. Co-authored-by: realbigsean * update builder bid type # Conflicts: # consensus/types/src/builder_bid.rs * Fix lint * remove generic from builder bid --------- Co-authored-by: realbigsean --- Cargo.lock | 2 +- beacon_node/beacon_chain/src/beacon_chain.rs | 119 ++++---- beacon_node/beacon_chain/src/blob_cache.rs | 35 --- beacon_node/beacon_chain/src/builder.rs | 2 - beacon_node/beacon_chain/src/errors.rs | 4 +- beacon_node/beacon_chain/src/lib.rs | 1 - beacon_node/beacon_chain/src/test_utils.rs | 62 +++-- .../beacon_chain/tests/block_verification.rs | 1 + beacon_node/builder_client/src/lib.rs | 12 +- beacon_node/execution_layer/src/engine_api.rs | 18 +- .../src/engine_api/json_structures.rs | 15 +- beacon_node/execution_layer/src/lib.rs | 220 ++++++++------- .../execution_layer/src/payload_cache.rs | 13 +- .../test_utils/execution_block_generator.rs | 16 +- .../src/test_utils/mock_execution_layer.rs | 2 +- .../http_api/src/build_block_contents.rs | 51 +++- beacon_node/http_api/src/lib.rs | 20 +- beacon_node/http_api/src/publish_blocks.rs | 55 ++-- beacon_node/http_api/tests/tests.rs | 36 ++- .../network/src/sync/block_lookups/tests.rs | 7 +- common/eth2/Cargo.toml | 1 + common/eth2/src/lib.rs | 6 +- common/eth2/src/types.rs | 193 +++++++++++-- consensus/types/Cargo.toml | 1 - consensus/types/src/blob_sidecar.rs | 258 +++++++++++++++++- consensus/types/src/builder_bid.rs | 141 +++++----- consensus/types/src/lib.rs | 10 +- consensus/types/src/payload.rs | 99 ++++++- consensus/types/src/signed_blob.rs | 48 +++- validator_client/src/block_service.rs | 50 ++-- validator_client/src/signing_method.rs | 2 +- validator_client/src/validator_store.rs | 24 +- 32 files changed, 1026 insertions(+), 498 deletions(-) delete mode 100644 beacon_node/beacon_chain/src/blob_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 6959799be29..9f68cb61a88 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2151,6 +2151,7 @@ dependencies = [ "serde", "serde_json", "slashing_protection", + "ssz_types", "store", "tokio", "types", @@ -8500,7 +8501,6 @@ dependencies = [ "serde", "serde_derive", "serde_json", - "serde_with", "serde_yaml", "slog", "smallvec 1.11.0", diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 0a89b739103..88f1c5ddd3e 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -7,7 +7,6 @@ use crate::attester_cache::{AttesterCache, AttesterCacheKey}; use crate::beacon_block_streamer::{BeaconBlockStreamer, CheckEarlyAttesterCache}; use crate::beacon_proposer_cache::compute_proposer_duties_from_head; use crate::beacon_proposer_cache::BeaconProposerCache; -use crate::blob_cache::BlobCache; use crate::blob_verification::{self, GossipBlobError, GossipVerifiedBlob}; use crate::block_times_cache::BlockTimesCache; use crate::block_verification::POS_PANDA_BANNER; @@ -67,8 +66,10 @@ use crate::validator_monitor::{ HISTORIC_EPOCHS as VALIDATOR_MONITOR_HISTORIC_EPOCHS, }; use crate::validator_pubkey_cache::ValidatorPubkeyCache; -use crate::{kzg_utils, AvailabilityPendingExecutedBlock}; -use crate::{metrics, BeaconChainError, BeaconForkChoiceStore, BeaconSnapshot, CachedHead}; +use crate::{ + kzg_utils, metrics, AvailabilityPendingExecutedBlock, BeaconChainError, BeaconForkChoiceStore, + BeaconSnapshot, CachedHead, +}; use eth2::types::{EventKind, SseBlock, SseExtendedPayloadAttributes, SyncDuty}; use execution_layer::{ BlockProposalContents, BuilderParams, ChainHealth, ExecutionLayer, FailedCondition, @@ -118,7 +119,7 @@ use task_executor::{ShutdownReason, TaskExecutor}; use tokio_stream::Stream; use tree_hash::TreeHash; use types::beacon_state::CloneConfig; -use types::blob_sidecar::{BlobSidecarList, FixedBlobSidecarList}; +use types::blob_sidecar::{BlobItems, BlobSidecarList, FixedBlobSidecarList}; use types::consts::deneb::MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; use types::*; @@ -473,12 +474,15 @@ pub struct BeaconChain { pub validator_monitor: RwLock>, /// The slot at which blocks are downloaded back to. pub genesis_backfill_slot: Slot, - pub proposal_blob_cache: BlobCache, pub data_availability_checker: Arc>, pub kzg: Option::Kzg>>>, } -type BeaconBlockAndState = (BeaconBlock, BeaconState); +type BeaconBlockAndState = ( + BeaconBlock, + BeaconState, + Option>::Sidecar>>, +); impl FinalizationAndCanonicity { pub fn is_finalized(self) -> bool { @@ -4978,67 +4982,52 @@ impl BeaconChain { let blobs_verification_timer = metrics::start_timer(&metrics::BLOCK_PRODUCTION_BLOBS_VERIFICATION_TIMES); - if let (Some(blobs), Some(proofs)) = (blobs_opt, proofs_opt) { - let kzg = self - .kzg - .as_ref() - .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; - let beacon_block_root = block.canonical_root(); - let expected_kzg_commitments = block.body().blob_kzg_commitments().map_err(|_| { - BlockProductionError::InvalidBlockVariant( - "DENEB block does not contain kzg commitments".to_string(), - ) - })?; - - if expected_kzg_commitments.len() != blobs.len() { - return Err(BlockProductionError::MissingKzgCommitment(format!( - "Missing KZG commitment for slot {}. Expected {}, got: {}", - slot, - blobs.len(), - expected_kzg_commitments.len() - ))); - } + let maybe_sidecar_list = match (blobs_opt, proofs_opt) { + (Some(blobs_or_blobs_roots), Some(proofs)) => { + let expected_kzg_commitments = + block.body().blob_kzg_commitments().map_err(|_| { + BlockProductionError::InvalidBlockVariant( + "deneb block does not contain kzg commitments".to_string(), + ) + })?; - let kzg_proofs = Vec::from(proofs); + if expected_kzg_commitments.len() != blobs_or_blobs_roots.len() { + return Err(BlockProductionError::MissingKzgCommitment(format!( + "Missing KZG commitment for slot {}. Expected {}, got: {}", + block.slot(), + blobs_or_blobs_roots.len(), + expected_kzg_commitments.len() + ))); + } - kzg_utils::validate_blobs::( - kzg.as_ref(), - expected_kzg_commitments, - &blobs, - &kzg_proofs, - ) - .map_err(BlockProductionError::KzgError)?; - - let blob_sidecars = BlobSidecarList::from( - blobs - .into_iter() - .enumerate() - .map(|(blob_index, blob)| { - let kzg_commitment = expected_kzg_commitments - .get(blob_index) - .expect("KZG commitment should exist for blob"); - - let kzg_proof = kzg_proofs - .get(blob_index) - .expect("KZG proof should exist for blob"); - - Ok(Arc::new(BlobSidecar { - block_root: beacon_block_root, - index: blob_index as u64, - slot, - block_parent_root: block.parent_root(), - proposer_index, - blob, - kzg_commitment: *kzg_commitment, - kzg_proof: *kzg_proof, - })) - }) - .collect::, BlockProductionError>>()?, - ); + let kzg_proofs = Vec::from(proofs); + + if let Some(blobs) = blobs_or_blobs_roots.blobs() { + let kzg = self + .kzg + .as_ref() + .ok_or(BlockProductionError::TrustedSetupNotInitialized)?; + kzg_utils::validate_blobs::( + kzg, + expected_kzg_commitments, + blobs, + &kzg_proofs, + ) + .map_err(BlockProductionError::KzgError)?; + } - self.proposal_blob_cache - .put(beacon_block_root, blob_sidecars); - } + Some( + Sidecar::build_sidecar( + blobs_or_blobs_roots, + &block, + expected_kzg_commitments, + kzg_proofs, + ) + .map_err(BlockProductionError::FailedToBuildBlobSidecars)?, + ) + } + _ => None, + }; drop(blobs_verification_timer); @@ -5052,7 +5041,7 @@ impl BeaconChain { "slot" => block.slot() ); - Ok((block, state)) + Ok((block, state, maybe_sidecar_list)) } /// This method must be called whenever an execution engine indicates that a payload is diff --git a/beacon_node/beacon_chain/src/blob_cache.rs b/beacon_node/beacon_chain/src/blob_cache.rs deleted file mode 100644 index 64f113c285c..00000000000 --- a/beacon_node/beacon_chain/src/blob_cache.rs +++ /dev/null @@ -1,35 +0,0 @@ -use lru::LruCache; -use parking_lot::Mutex; -use types::{BlobSidecarList, EthSpec, Hash256}; - -pub const DEFAULT_BLOB_CACHE_SIZE: usize = 10; - -/// A cache blobs by beacon block root. -pub struct BlobCache { - blobs: Mutex>>, -} - -#[derive(Hash, PartialEq, Eq)] -struct BlobCacheId(Hash256); - -impl Default for BlobCache { - fn default() -> Self { - BlobCache { - blobs: Mutex::new(LruCache::new(DEFAULT_BLOB_CACHE_SIZE)), - } - } -} - -impl BlobCache { - pub fn put( - &self, - beacon_block: Hash256, - blobs: BlobSidecarList, - ) -> Option> { - self.blobs.lock().put(BlobCacheId(beacon_block), blobs) - } - - pub fn pop(&self, root: &Hash256) -> Option> { - self.blobs.lock().pop(&BlobCacheId(*root)) - } -} diff --git a/beacon_node/beacon_chain/src/builder.rs b/beacon_node/beacon_chain/src/builder.rs index 55fc24feaf8..45c4f42411d 100644 --- a/beacon_node/beacon_chain/src/builder.rs +++ b/beacon_node/beacon_chain/src/builder.rs @@ -1,5 +1,4 @@ use crate::beacon_chain::{CanonicalHead, BEACON_CHAIN_DB_KEY, ETH1_CACHE_DB_KEY, OP_POOL_DB_KEY}; -use crate::blob_cache::BlobCache; use crate::data_availability_checker::DataAvailabilityChecker; use crate::eth1_chain::{CachingEth1Backend, SszEth1}; use crate::eth1_finalization_cache::Eth1FinalizationCache; @@ -889,7 +888,6 @@ where DataAvailabilityChecker::new(slot_clock, kzg.clone(), store, self.spec) .map_err(|e| format!("Error initializing DataAvailabiltyChecker: {:?}", e))?, ), - proposal_blob_cache: BlobCache::default(), kzg, }; diff --git a/beacon_node/beacon_chain/src/errors.rs b/beacon_node/beacon_chain/src/errors.rs index 32f7e63a935..8b4493d49d4 100644 --- a/beacon_node/beacon_chain/src/errors.rs +++ b/beacon_node/beacon_chain/src/errors.rs @@ -275,20 +275,22 @@ pub enum BlockProductionError { blob_block_hash: ExecutionBlockHash, payload_block_hash: ExecutionBlockHash, }, - NoBlobsCached, FailedToReadFinalizedBlock(store::Error), MissingFinalizedBlock(Hash256), BlockTooLarge(usize), ShuttingDown, + MissingBlobs, MissingSyncAggregate, MissingExecutionPayload, MissingKzgCommitment(String), + MissingKzgProof(String), TokioJoin(tokio::task::JoinError), BeaconChain(BeaconChainError), InvalidPayloadFork, TrustedSetupNotInitialized, InvalidBlockVariant(String), KzgError(kzg::Error), + FailedToBuildBlobSidecars(String), } easy_from_to!(BlockProcessingError, BlockProductionError); diff --git a/beacon_node/beacon_chain/src/lib.rs b/beacon_node/beacon_chain/src/lib.rs index 05d41b016d7..4efc776b2c6 100644 --- a/beacon_node/beacon_chain/src/lib.rs +++ b/beacon_node/beacon_chain/src/lib.rs @@ -7,7 +7,6 @@ mod beacon_chain; mod beacon_fork_choice_store; pub mod beacon_proposer_cache; mod beacon_snapshot; -pub mod blob_cache; pub mod blob_verification; pub mod block_reward; mod block_times_cache; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index b8e32ef7fbe..1172746825b 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -15,7 +15,7 @@ use crate::{ StateSkipConfig, }; use bls::get_withdrawal_credentials; -use eth2::types::BlockContentsTuple; +use eth2::types::SignedBlockContentsTuple; use execution_layer::test_utils::generate_genesis_header; use execution_layer::{ auth::JwtKey, @@ -50,6 +50,7 @@ use state_processing::{ use std::borrow::Cow; use std::collections::{HashMap, HashSet}; use std::fmt; +use std::marker::PhantomData; use std::str::FromStr; use std::sync::Arc; use std::time::Duration; @@ -817,9 +818,28 @@ where &self, state: BeaconState, slot: Slot, - ) -> (BlockContentsTuple>, BeaconState) { + ) -> ( + SignedBlockContentsTuple>, + BeaconState, + ) { let (unblinded, new_state) = self.make_block(state, slot).await; - ((unblinded.0.into(), unblinded.1), new_state) + let maybe_blinded_blob_sidecars = unblinded.1.map(|blob_sidecar_list| { + VariableList::new( + blob_sidecar_list + .into_iter() + .map(|blob_sidecar| { + let blinded_sidecar: BlindedBlobSidecar = blob_sidecar.message.into(); + SignedSidecar { + message: Arc::new(blinded_sidecar), + signature: blob_sidecar.signature, + _phantom: PhantomData, + } + }) + .collect(), + ) + .unwrap() + }); + ((unblinded.0.into(), maybe_blinded_blob_sidecars), new_state) } /// Returns a newly created block, signed by the proposer for the given slot. @@ -827,7 +847,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> (BlockContentsTuple>, BeaconState) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -845,7 +865,7 @@ where let randao_reveal = self.sign_randao_reveal(&state, proposer_index, slot); - let (block, state) = self + let (block, state, maybe_blob_sidecars) = self .chain .produce_block_on_state( state, @@ -865,18 +885,14 @@ where &self.spec, ); - let block_contents: BlockContentsTuple> = match &signed_block { + let block_contents: SignedBlockContentsTuple> = match &signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { - let signed_blobs: SignedBlobSidecarList = Vec::from(blobs) + if let Some(blobs) = maybe_blob_sidecars { + let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { blob.sign( @@ -911,7 +927,7 @@ where &self, mut state: BeaconState, slot: Slot, - ) -> (BlockContentsTuple>, BeaconState) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -931,7 +947,7 @@ where let pre_state = state.clone(); - let (block, state) = self + let (block, state, maybe_blob_sidecars) = self .chain .produce_block_on_state( state, @@ -951,18 +967,14 @@ where &self.spec, ); - let block_contents: BlockContentsTuple> = match &signed_block { + let block_contents: SignedBlockContentsTuple> = match &signed_block { SignedBeaconBlock::Base(_) | SignedBeaconBlock::Altair(_) | SignedBeaconBlock::Merge(_) | SignedBeaconBlock::Capella(_) => (signed_block, None), SignedBeaconBlock::Deneb(_) => { - if let Some(blobs) = self - .chain - .proposal_blob_cache - .pop(&signed_block.canonical_root()) - { - let signed_blobs: SignedBlobSidecarList = Vec::from(blobs) + if let Some(blobs) = maybe_blob_sidecars { + let signed_blobs: SignedSidecarList> = Vec::from(blobs) .into_iter() .map(|blob| { blob.sign( @@ -1778,7 +1790,7 @@ where state: BeaconState, slot: Slot, block_modifier: impl FnOnce(&mut BeaconBlock), - ) -> (BlockContentsTuple>, BeaconState) { + ) -> (SignedBlockContentsTuple>, BeaconState) { assert_ne!(slot, 0, "can't produce a block at slot 0"); assert!(slot >= state.slot()); @@ -1876,7 +1888,7 @@ where &self, slot: Slot, block_root: Hash256, - block_contents: BlockContentsTuple>, + block_contents: SignedBlockContentsTuple>, ) -> Result> { self.set_current_slot(slot); let (block, blobs) = block_contents; @@ -1906,7 +1918,7 @@ where pub async fn process_block_result( &self, - block_contents: BlockContentsTuple>, + block_contents: SignedBlockContentsTuple>, ) -> Result> { let (block, blobs) = block_contents; // Note: we are just dropping signatures here and skipping signature verification. @@ -1991,7 +2003,7 @@ where ) -> Result< ( SignedBeaconBlockHash, - BlockContentsTuple>, + SignedBlockContentsTuple>, BeaconState, ), BlockError, diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index 370d078b5a0..164707005c3 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -126,6 +126,7 @@ async fn get_chain_segment_with_signed_blobs() -> ( .get(&BlobSignatureKey::new(block_root, blob_index)) .unwrap() .clone(), + _phantom: PhantomData, } }) .collect::>(); diff --git a/beacon_node/builder_client/src/lib.rs b/beacon_node/builder_client/src/lib.rs index 9a842340195..aebc3f5e2be 100644 --- a/beacon_node/builder_client/src/lib.rs +++ b/beacon_node/builder_client/src/lib.rs @@ -1,8 +1,8 @@ use eth2::types::builder_bid::SignedBuilderBid; +use eth2::types::payload::FullPayloadContents; use eth2::types::{ - AbstractExecPayload, BlindedPayload, EthSpec, ExecutionBlockHash, ExecutionPayload, - ForkVersionedResponse, PublicKeyBytes, SignedBlockContents, SignedValidatorRegistrationData, - Slot, + BlindedPayload, EthSpec, ExecutionBlockHash, ForkVersionedResponse, PublicKeyBytes, + SignedBlockContents, SignedValidatorRegistrationData, Slot, }; pub use eth2::Error; use eth2::{ok_or_error, StatusCode}; @@ -141,7 +141,7 @@ impl BuilderHttpClient { pub async fn post_builder_blinded_blocks( &self, blinded_block: &SignedBlockContents>, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.server.full.clone(); path.path_segments_mut() @@ -163,12 +163,12 @@ impl BuilderHttpClient { } /// `GET /eth/v1/builder/header` - pub async fn get_builder_header>( + pub async fn get_builder_header( &self, slot: Slot, parent_hash: ExecutionBlockHash, pubkey: &PublicKeyBytes, - ) -> Result>>, Error> { + ) -> Result>>, Error> { let mut path = self.server.full.clone(); path.path_segments_mut() diff --git a/beacon_node/execution_layer/src/engine_api.rs b/beacon_node/execution_layer/src/engine_api.rs index ed3cc330a39..8952b15ed0f 100644 --- a/beacon_node/execution_layer/src/engine_api.rs +++ b/beacon_node/execution_layer/src/engine_api.rs @@ -20,16 +20,14 @@ use state_processing::per_block_processing::deneb::deneb::kzg_commitment_to_vers use std::convert::TryFrom; use strum::IntoStaticStr; use superstruct::superstruct; -use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; pub use types::{ Address, BeaconBlockRef, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadHeader, ExecutionPayloadRef, FixedVector, ForkName, Hash256, Transactions, Uint256, VariableList, Withdrawal, Withdrawals, }; use types::{ - BeaconStateError, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, - KzgProofs, VersionedHash, + BeaconStateError, BlobsBundle, ExecutionPayloadCapella, ExecutionPayloadDeneb, + ExecutionPayloadMerge, KzgProofs, VersionedHash, }; pub mod auth; @@ -64,7 +62,6 @@ pub enum Error { IncorrectStateVariant, RequiredMethodUnsupported(&'static str), UnsupportedForkVariant(String), - BadConversion(String), RlpDecoderError(rlp::DecoderError), BlobTxConversionError(BlobTxConversionError), } @@ -416,7 +413,7 @@ pub struct GetPayloadResponse { pub execution_payload: ExecutionPayloadDeneb, pub block_value: Uint256, #[superstruct(only(Deneb))] - pub blobs_bundle: BlobsBundleV1, + pub blobs_bundle: BlobsBundle, #[superstruct(only(Deneb), partial_getter(copy))] pub should_override_builder: bool, } @@ -452,7 +449,7 @@ impl From> for ExecutionPayload { } impl From> - for (ExecutionPayload, Uint256, Option>) + for (ExecutionPayload, Uint256, Option>) { fn from(response: GetPayloadResponse) -> Self { match response { @@ -575,13 +572,6 @@ impl ExecutionPayloadBodyV1 { } } -#[derive(Clone, Default, Debug, PartialEq)] -pub struct BlobsBundleV1 { - pub commitments: KzgCommitments, - pub proofs: KzgProofs, - pub blobs: Blobs, -} - #[superstruct( variants(Merge, Capella, Deneb), variant_attributes(derive(Clone, Debug, PartialEq),), diff --git a/beacon_node/execution_layer/src/engine_api/json_structures.rs b/beacon_node/execution_layer/src/engine_api/json_structures.rs index bce4e686f92..3f4f1eb96b8 100644 --- a/beacon_node/execution_layer/src/engine_api/json_structures.rs +++ b/beacon_node/execution_layer/src/engine_api/json_structures.rs @@ -3,10 +3,11 @@ use serde::{Deserialize, Serialize}; use strum::EnumString; use superstruct::superstruct; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; +use types::blob_sidecar::BlobsList; use types::{ - EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, - ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, VariableList, Withdrawal, + BlobsBundle, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, + ExecutionPayloadDeneb, ExecutionPayloadMerge, FixedVector, Transactions, Unsigned, + VariableList, Withdrawal, }; #[derive(Debug, PartialEq, Serialize, Deserialize)] @@ -441,11 +442,11 @@ pub struct JsonBlobsBundleV1 { pub commitments: KzgCommitments, pub proofs: KzgProofs, #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] - pub blobs: Blobs, + pub blobs: BlobsList, } -impl From> for JsonBlobsBundleV1 { - fn from(blobs_bundle: BlobsBundleV1) -> Self { +impl From> for JsonBlobsBundleV1 { + fn from(blobs_bundle: BlobsBundle) -> Self { Self { commitments: blobs_bundle.commitments, proofs: blobs_bundle.proofs, @@ -453,7 +454,7 @@ impl From> for JsonBlobsBundleV1 { } } } -impl From> for BlobsBundleV1 { +impl From> for BlobsBundle { fn from(json_blobs_bundle: JsonBlobsBundleV1) -> Self { Self { commitments: json_blobs_bundle.commitments, diff --git a/beacon_node/execution_layer/src/lib.rs b/beacon_node/execution_layer/src/lib.rs index e997fc596ae..6675b7826b7 100644 --- a/beacon_node/execution_layer/src/lib.rs +++ b/beacon_node/execution_layer/src/lib.rs @@ -13,8 +13,8 @@ pub use engine_api::*; pub use engine_api::{http, http::deposit_methods, http::HttpJsonRpc}; use engines::{Engine, EngineError}; pub use engines::{EngineState, ForkchoiceState}; -use eth2::types::SignedBlockContents; -use eth2::types::{builder_bid::SignedBuilderBid, ForkVersionedResponse}; +use eth2::types::{builder_bid::SignedBuilderBid, BlobsBundle, ForkVersionedResponse}; +use eth2::types::{FullPayloadContents, SignedBlockContents}; use ethers_core::abi::ethereum_types::FromStrRadixErr; use ethers_core::types::Transaction as EthersTransaction; use fork_choice::ForkchoiceUpdateParameters; @@ -41,12 +41,13 @@ use tokio::{ use tokio_stream::wrappers::WatchStream; use tree_hash::TreeHash; use types::beacon_block_body::KzgCommitments; -use types::blob_sidecar::Blobs; -use types::KzgProofs; +use types::blob_sidecar::BlobItems; +use types::builder_bid::BuilderBid; use types::{ AbstractExecPayload, BeaconStateError, BlindedPayload, BlockType, ChainSpec, Epoch, ExecPayload, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadMerge, }; +use types::{KzgProofs, Sidecar}; use types::{ProposerPreparationData, PublicKeyBytes, Signature, Slot, Transaction}; mod block_hash; @@ -86,6 +87,40 @@ pub enum ProvenancedPayload

{ Builder(P), } +impl> TryFrom> + for ProvenancedPayload> +{ + type Error = Error; + + fn try_from(value: BuilderBid) -> Result { + let block_proposal_contents = match value { + BuilderBid::Merge(builder_bid) => BlockProposalContents::Payload { + payload: ExecutionPayloadHeader::Merge(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, + block_value: builder_bid.value, + }, + BuilderBid::Capella(builder_bid) => BlockProposalContents::Payload { + payload: ExecutionPayloadHeader::Capella(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, + block_value: builder_bid.value, + }, + BuilderBid::Deneb(builder_bid) => BlockProposalContents::PayloadAndBlobs { + payload: ExecutionPayloadHeader::Deneb(builder_bid.header) + .try_into() + .map_err(|_| Error::InvalidPayloadConversion)?, + block_value: builder_bid.value, + kzg_commitments: builder_bid.blinded_blobs_bundle.commitments, + blobs: BlobItems::try_from_blob_roots(builder_bid.blinded_blobs_bundle.blob_roots) + .map_err(Error::InvalidBlobConversion)?, + proofs: builder_bid.blinded_blobs_bundle.proofs, + }, + }; + Ok(ProvenancedPayload::Builder(block_proposal_contents)) + } +} + #[derive(Debug)] pub enum Error { NoEngine, @@ -107,6 +142,8 @@ pub enum Error { InvalidJWTSecret(String), InvalidForkForPayload, InvalidPayloadBody(String), + InvalidPayloadConversion, + InvalidBlobConversion(String), BeaconStateError(BeaconStateError), } @@ -131,28 +168,31 @@ pub enum BlockProposalContents> { payload: Payload, block_value: Uint256, kzg_commitments: KzgCommitments, - blobs: Blobs, + blobs: >::BlobItems, proofs: KzgProofs, }, } -impl> From> +impl> TryFrom> for BlockProposalContents { - fn from(response: GetPayloadResponse) -> Self { + type Error = Error; + + fn try_from(response: GetPayloadResponse) -> Result { let (execution_payload, block_value, maybe_bundle) = response.into(); match maybe_bundle { - Some(bundle) => Self::PayloadAndBlobs { + Some(bundle) => Ok(Self::PayloadAndBlobs { payload: execution_payload.into(), block_value, kzg_commitments: bundle.commitments, - blobs: bundle.blobs, + blobs: BlobItems::try_from_blobs(bundle.blobs) + .map_err(Error::InvalidBlobConversion)?, proofs: bundle.proofs, - }, - None => Self::Payload { + }), + None => Ok(Self::Payload { payload: execution_payload.into(), block_value, - }, + }), } } } @@ -164,7 +204,7 @@ impl> BlockProposalContents ( Payload, Option>, - Option>, + Option<>::BlobItems>, Option>, ) { match self { @@ -184,47 +224,20 @@ impl> BlockProposalContents &Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, } } pub fn to_payload(self) -> Payload { match self { - Self::Payload { - payload, - block_value: _, - } => payload, - Self::PayloadAndBlobs { - payload, - block_value: _, - kzg_commitments: _, - blobs: _, - proofs: _, - } => payload, + Self::Payload { payload, .. } => payload, + Self::PayloadAndBlobs { payload, .. } => payload, } } pub fn block_value(&self) -> &Uint256 { match self { - Self::Payload { - payload: _, - block_value, - } => block_value, - Self::PayloadAndBlobs { - payload: _, - block_value, - kzg_commitments: _, - blobs: _, - proofs: _, - } => block_value, + Self::Payload { block_value, .. } => block_value, + Self::PayloadAndBlobs { block_value, .. } => block_value, } } pub fn default_at_fork(fork_name: ForkName) -> Result { @@ -238,7 +251,7 @@ impl> BlockProposalContents BlockProposalContents::PayloadAndBlobs { payload: Payload::default_at_fork(fork_name)?, block_value: Uint256::zero(), - blobs: VariableList::default(), + blobs: Payload::default_blobs_at_fork(fork_name)?, kzg_commitments: VariableList::default(), proofs: VariableList::default(), }, @@ -285,6 +298,8 @@ pub enum FailedCondition { EpochsSinceFinalization, } +type PayloadContentsRefTuple<'a, T> = (ExecutionPayloadRef<'a, T>, Option<&'a BlobsBundle>); + struct Inner { engine: Arc, builder: Option, @@ -488,12 +503,28 @@ impl ExecutionLayer { } /// Cache a full payload, keyed on the `tree_hash_root` of the payload - fn cache_payload(&self, payload: ExecutionPayloadRef) -> Option> { - self.inner.payload_cache.put(payload.clone_from_ref()) + fn cache_payload( + &self, + payload_and_blobs: PayloadContentsRefTuple, + ) -> Option> { + let (payload_ref, maybe_json_blobs_bundle) = payload_and_blobs; + + let payload = payload_ref.clone_from_ref(); + let maybe_blobs_bundle = maybe_json_blobs_bundle + .cloned() + .map(|blobs_bundle| BlobsBundle { + commitments: blobs_bundle.commitments, + proofs: blobs_bundle.proofs, + blobs: blobs_bundle.blobs, + }); + + self.inner + .payload_cache + .put(FullPayloadContents::new(payload, maybe_blobs_bundle)) } /// Attempt to retrieve a full payload from the payload cache by the payload root - pub fn get_payload_by_root(&self, root: &Hash256) -> Option> { + pub fn get_payload_by_root(&self, root: &Hash256) -> Option> { self.inner.payload_cache.get(root) } @@ -791,7 +822,8 @@ impl ExecutionLayer { current_fork, ) .await - .map(|get_payload_response| ProvenancedPayload::Local(get_payload_response.into())) + .and_then(GetPayloadResponse::try_into) + .map(ProvenancedPayload::Local) } }; @@ -856,7 +888,7 @@ impl ExecutionLayer { let ((relay_result, relay_duration), (local_result, local_duration)) = tokio::join!( timed_future(metrics::GET_BLINDED_PAYLOAD_BUILDER, async { builder - .get_builder_header::(slot, parent_hash, &pubkey) + .get_builder_header::(slot, parent_hash, &pubkey) .await }), timed_future(metrics::GET_BLINDED_PAYLOAD_LOCAL, async { @@ -874,7 +906,7 @@ impl ExecutionLayer { self.log(), "Requested blinded execution payload"; "relay_fee_recipient" => match &relay_result { - Ok(Some(r)) => format!("{:?}", r.data.message.header.fee_recipient()), + Ok(Some(r)) => format!("{:?}", r.data.message.header().fee_recipient()), Ok(None) => "empty response".to_string(), Err(_) => "request failed".to_string(), }, @@ -897,7 +929,7 @@ impl ExecutionLayer { "local_block_hash" => ?local.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local.into())) + Ok(ProvenancedPayload::Local(local.try_into()?)) } (Ok(None), Ok(local)) => { info!( @@ -907,10 +939,10 @@ impl ExecutionLayer { "local_block_hash" => ?local.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local.into())) + Ok(ProvenancedPayload::Local(local.try_into()?)) } (Ok(Some(relay)), Ok(local)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -920,21 +952,21 @@ impl ExecutionLayer { "parent_hash" => ?parent_hash, ); - let relay_value = relay.data.message.value; + let relay_value = relay.data.message.value(); let local_value = *local.block_value(); if !self.inner.always_prefer_builder_payload { - if local_value >= relay_value { + if local_value >= *relay_value { info!( self.log(), "Local block is more profitable than relay block"; "local_block_value" => %local_value, "relay_value" => %relay_value ); - return Ok(ProvenancedPayload::Local(local.into())); + return Ok(ProvenancedPayload::Local(local.try_into()?)); } else if local.should_override_builder().unwrap_or(false) { let percentage_difference = - percentage_difference_u256(local_value, relay_value); + percentage_difference_u256(local_value, *relay_value); if percentage_difference.map_or(false, |percentage| { percentage < self @@ -947,7 +979,7 @@ impl ExecutionLayer { "local_block_value" => %local_value, "relay_value" => %relay_value ); - return Ok(ProvenancedPayload::Local(local.into())); + return Ok(ProvenancedPayload::Local(local.try_into()?)); } } else { info!( @@ -968,12 +1000,7 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::try_from(relay.data.message)?), Err(reason) if !reason.payload_invalid() => { info!( self.log(), @@ -983,7 +1010,7 @@ impl ExecutionLayer { "relay_block_hash" => ?header.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local.into())) + Ok(ProvenancedPayload::Local(local.try_into()?)) } Err(reason) => { metrics::inc_counter_vec( @@ -998,12 +1025,12 @@ impl ExecutionLayer { "relay_block_hash" => ?header.block_hash(), "parent_hash" => ?parent_hash, ); - Ok(ProvenancedPayload::Local(local.into())) + Ok(ProvenancedPayload::Local(local.try_into()?)) } } } (Ok(Some(relay)), Err(local_error)) => { - let header = &relay.data.message.header; + let header = &relay.data.message.header(); info!( self.log(), @@ -1022,20 +1049,12 @@ impl ExecutionLayer { current_fork, spec, ) { - Ok(()) => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Ok(()) => Ok(ProvenancedPayload::try_from(relay.data.message)?), // If the payload is valid then use it. The local EE failed // to produce a payload so we have no alternative. - Err(e) if !e.payload_invalid() => Ok(ProvenancedPayload::Builder( - BlockProposalContents::Payload { - payload: relay.data.message.header, - block_value: relay.data.message.value, - }, - )), + Err(e) if !e.payload_invalid() => { + Ok(ProvenancedPayload::try_from(relay.data.message)?) + } Err(reason) => { metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_GET_PAYLOAD_BUILDER_REJECTIONS, @@ -1103,7 +1122,8 @@ impl ExecutionLayer { current_fork, ) .await - .map(|get_payload_response| ProvenancedPayload::Local(get_payload_response.into())) + .and_then(GetPayloadResponse::try_into) + .map(ProvenancedPayload::Local) } /// Get a full payload without caching its result in the execution layer's payload cache. @@ -1148,7 +1168,10 @@ impl ExecutionLayer { payload_attributes: &PayloadAttributes, forkchoice_update_params: ForkchoiceUpdateParameters, current_fork: ForkName, - f: fn(&ExecutionLayer, ExecutionPayloadRef) -> Option>, + cache_fn: fn( + &ExecutionLayer, + PayloadContentsRefTuple, + ) -> Option>, ) -> Result, Error> { self.engine() .request(move |engine| async move { @@ -1227,7 +1250,7 @@ impl ExecutionLayer { "suggested_fee_recipient" => ?payload_attributes.suggested_fee_recipient(), ); } - if f(self, payload_response.execution_payload_ref()).is_some() { + if cache_fn(self, (payload_response.execution_payload_ref(), payload_response.blobs_bundle().ok())).is_some() { warn!( self.log(), "Duplicate payload cached, this might indicate redundant proposal \ @@ -1859,7 +1882,7 @@ impl ExecutionLayer { &self, block_root: Hash256, block: &SignedBlockContents>, - ) -> Result, Error> { + ) -> Result, Error> { debug!( self.log(), "Sending block to builder"; @@ -1878,11 +1901,12 @@ impl ExecutionLayer { .await; match &payload_result { - Ok(payload) => { + Ok(unblinded_response) => { metrics::inc_counter_vec( &metrics::EXECUTION_LAYER_BUILDER_REVEAL_PAYLOAD_OUTCOME, &[metrics::SUCCESS], ); + let payload = unblinded_response.payload_ref(); info!( self.log(), "Builder successfully revealed payload"; @@ -2025,8 +2049,8 @@ impl fmt::Display for InvalidBuilderPayload { } /// Perform some cursory, non-exhaustive validation of the bid returned from the builder. -fn verify_builder_bid>( - bid: &ForkVersionedResponse>, +fn verify_builder_bid( + bid: &ForkVersionedResponse>, parent_hash: ExecutionBlockHash, payload_attributes: &PayloadAttributes, block_number: Option, @@ -2035,11 +2059,11 @@ fn verify_builder_bid>( spec: &ChainSpec, ) -> Result<(), Box> { let is_signature_valid = bid.data.verify_signature(spec); - let header = &bid.data.message.header; - let payload_value = bid.data.message.value; + let header = &bid.data.message.header(); + let payload_value = bid.data.message.value(); // Avoid logging values that we can't represent with our Prometheus library. - let payload_value_gwei = bid.data.message.value / 1_000_000_000; + let payload_value_gwei = bid.data.message.value() / 1_000_000_000; if payload_value_gwei <= Uint256::from(i64::max_value()) { metrics::set_gauge_vec( &metrics::EXECUTION_LAYER_PAYLOAD_BIDS, @@ -2053,12 +2077,12 @@ fn verify_builder_bid>( .ok() .cloned() .map(|withdrawals| Withdrawals::::from(withdrawals).tree_hash_root()); - let payload_withdrawals_root = header.withdrawals_root().ok(); + let payload_withdrawals_root = header.withdrawals_root().ok().copied(); - if payload_value < profit_threshold { + if *payload_value < profit_threshold { Err(Box::new(InvalidBuilderPayload::LowValue { profit_threshold, - payload_value, + payload_value: *payload_value, })) } else if header.parent_hash() != parent_hash { Err(Box::new(InvalidBuilderPayload::ParentHash { @@ -2088,7 +2112,7 @@ fn verify_builder_bid>( } else if !is_signature_valid { Err(Box::new(InvalidBuilderPayload::Signature { signature: bid.data.signature.clone(), - pubkey: bid.data.message.pubkey, + pubkey: *bid.data.message.pubkey(), })) } else if payload_withdrawals_root != expected_withdrawals_root { Err(Box::new(InvalidBuilderPayload::WithdrawalsRoot { @@ -2197,8 +2221,8 @@ fn ethers_tx_to_ssz( fn noop( _: &ExecutionLayer, - _: ExecutionPayloadRef, -) -> Option> { + _: PayloadContentsRefTuple, +) -> Option> { None } diff --git a/beacon_node/execution_layer/src/payload_cache.rs b/beacon_node/execution_layer/src/payload_cache.rs index 1722edff465..1155b1ca3a4 100644 --- a/beacon_node/execution_layer/src/payload_cache.rs +++ b/beacon_node/execution_layer/src/payload_cache.rs @@ -1,13 +1,14 @@ +use eth2::types::FullPayloadContents; use lru::LruCache; use parking_lot::Mutex; use tree_hash::TreeHash; -use types::{EthSpec, ExecutionPayload, Hash256}; +use types::{EthSpec, Hash256}; pub const DEFAULT_PAYLOAD_CACHE_SIZE: usize = 10; /// A cache mapping execution payloads by tree hash roots. pub struct PayloadCache { - payloads: Mutex>>, + payloads: Mutex>>, } #[derive(Hash, PartialEq, Eq)] @@ -22,16 +23,16 @@ impl Default for PayloadCache { } impl PayloadCache { - pub fn put(&self, payload: ExecutionPayload) -> Option> { - let root = payload.tree_hash_root(); + pub fn put(&self, payload: FullPayloadContents) -> Option> { + let root = payload.payload_ref().tree_hash_root(); self.payloads.lock().put(PayloadCacheId(root), payload) } - pub fn pop(&self, root: &Hash256) -> Option> { + pub fn pop(&self, root: &Hash256) -> Option> { self.payloads.lock().pop(&PayloadCacheId(*root)) } - pub fn get(&self, hash: &Hash256) -> Option> { + pub fn get(&self, hash: &Hash256) -> Option> { self.payloads.lock().get(&PayloadCacheId(*hash)).cloned() } } diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index e6fa17349c7..af9bc266aa2 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -6,7 +6,7 @@ use crate::{ }, ExecutionBlock, PayloadAttributes, PayloadId, PayloadStatusV1, PayloadStatusV1Status, }, - random_valid_tx, BlobsBundleV1, ExecutionBlockWithTransactions, + random_valid_tx, ExecutionBlockWithTransactions, }; use kzg::Kzg; use rand::thread_rng; @@ -16,9 +16,9 @@ use std::sync::Arc; use tree_hash::TreeHash; use tree_hash_derive::TreeHash; use types::{ - BlobSidecar, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, ExecutionPayloadCapella, - ExecutionPayloadDeneb, ExecutionPayloadHeader, ExecutionPayloadMerge, ForkName, Hash256, - Transactions, Uint256, + BlobSidecar, BlobsBundle, ChainSpec, EthSpec, ExecutionBlockHash, ExecutionPayload, + ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadHeader, ExecutionPayloadMerge, + ForkName, Hash256, Transactions, Uint256, }; use super::DEFAULT_TERMINAL_BLOCK; @@ -128,7 +128,7 @@ pub struct ExecutionBlockGenerator { /* * deneb stuff */ - pub blobs_bundles: HashMap>, + pub blobs_bundles: HashMap>, pub kzg: Option>>, } @@ -406,7 +406,7 @@ impl ExecutionBlockGenerator { self.payload_ids.get(id).cloned() } - pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { + pub fn get_blobs_bundle(&mut self, id: &PayloadId) -> Option> { self.blobs_bundles.get(id).cloned() } @@ -630,8 +630,8 @@ impl ExecutionBlockGenerator { pub fn generate_random_blobs( n_blobs: usize, kzg: &Kzg, -) -> Result<(BlobsBundleV1, Transactions), String> { - let mut bundle = BlobsBundleV1::::default(); +) -> Result<(BlobsBundle, Transactions), String> { + let mut bundle = BlobsBundle::::default(); let mut transactions = vec![]; for blob_index in 0..n_blobs { let random_valid_sidecar = BlobSidecar::::random_valid(&mut thread_rng(), kzg)?; diff --git a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs index d82aca3bc40..5aa3b2f8b9a 100644 --- a/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs +++ b/beacon_node/execution_layer/src/test_utils/mock_execution_layer.rs @@ -202,7 +202,7 @@ impl MockExecutionLayer { assert_eq!( self.el .get_payload_by_root(&payload_header.tree_hash_root()), - Some(payload.clone()) + Some(FullPayloadContents::Payload(payload.clone())) ); // TODO: again consider forks diff --git a/beacon_node/http_api/src/build_block_contents.rs b/beacon_node/http_api/src/build_block_contents.rs index d6c5ada0071..c8f28fa9ae3 100644 --- a/beacon_node/http_api/src/build_block_contents.rs +++ b/beacon_node/http_api/src/build_block_contents.rs @@ -1,22 +1,25 @@ -use beacon_chain::{BeaconChain, BeaconChainTypes, BlockProductionError}; -use eth2::types::{BeaconBlockAndBlobSidecars, BlockContents}; -use std::sync::Arc; -use types::{AbstractExecPayload, BeaconBlock, ForkName}; +use beacon_chain::BlockProductionError; +use eth2::types::{BeaconBlockAndBlobSidecars, BlindedBeaconBlockAndBlobSidecars, BlockContents}; +use types::{ + BeaconBlock, BlindedBlobSidecarList, BlindedPayload, BlobSidecarList, EthSpec, ForkName, + FullPayload, +}; type Error = warp::reject::Rejection; +type FullBlockContents = BlockContents>; +type BlindedBlockContents = BlockContents>; -pub fn build_block_contents>( +pub fn build_block_contents( fork_name: ForkName, - chain: Arc>, - block: BeaconBlock, -) -> Result, Error> { + block: BeaconBlock>, + maybe_blobs: Option>, +) -> Result, Error> { match fork_name { ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { Ok(BlockContents::Block(block)) } ForkName::Deneb => { - let block_root = &block.canonical_root(); - if let Some(blob_sidecars) = chain.proposal_blob_cache.pop(block_root) { + if let Some(blob_sidecars) = maybe_blobs { let block_and_blobs = BeaconBlockAndBlobSidecars { block, blob_sidecars, @@ -25,7 +28,33 @@ pub fn build_block_contents( + fork_name: ForkName, + block: BeaconBlock>, + maybe_blobs: Option>, +) -> Result, Error> { + match fork_name { + ForkName::Base | ForkName::Altair | ForkName::Merge | ForkName::Capella => { + Ok(BlockContents::Block(block)) + } + ForkName::Deneb => { + if let Some(blinded_blob_sidecars) = maybe_blobs { + let block_and_blobs = BlindedBeaconBlockAndBlobSidecars { + blinded_block: block, + blinded_blob_sidecars, + }; + + Ok(BlockContents::BlindedBlockAndBlobSidecars(block_and_blobs)) + } else { + Err(warp_utils::reject::block_production_error( + BlockProductionError::MissingBlobs, )) } } diff --git a/beacon_node/http_api/src/lib.rs b/beacon_node/http_api/src/lib.rs index 4b95a04a4ee..300ebfa4957 100644 --- a/beacon_node/http_api/src/lib.rs +++ b/beacon_node/http_api/src/lib.rs @@ -1440,14 +1440,14 @@ pub fn serve( .and(network_tx_filter.clone()) .and(log_filter.clone()) .and_then( - |block: SignedBlockContents>, + |block_contents: SignedBlockContents>, task_spawner: TaskSpawner, chain: Arc>, network_tx: UnboundedSender>, log: Logger| { task_spawner.spawn_async_with_rejection(Priority::P0, async move { publish_blocks::publish_blinded_block( - block, + block_contents, chain, &network_tx, log, @@ -3065,7 +3065,7 @@ pub fn serve( ProduceBlockVerification::VerifyRandao }; - let (block, _) = chain + let (block, _, maybe_blobs) = chain .produce_block_with_verification::>( randao_reveal, slot, @@ -3080,9 +3080,9 @@ pub fn serve( .map_err(inconsistent_fork_rejection)?; let block_contents = - build_block_contents::build_block_contents(fork_name, chain, block); + build_block_contents::build_block_contents(fork_name, block, maybe_blobs)?; - fork_versioned_response(endpoint_version, fork_name, block_contents?) + fork_versioned_response(endpoint_version, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) .map(|res| add_consensus_version_header(res, fork_name)) }) @@ -3129,7 +3129,7 @@ pub fn serve( ProduceBlockVerification::VerifyRandao }; - let (block, _) = chain + let (block, _, maybe_blobs) = chain .produce_block_with_verification::>( randao_reveal, slot, @@ -3143,8 +3143,14 @@ pub fn serve( .fork_name(&chain.spec) .map_err(inconsistent_fork_rejection)?; + let block_contents = build_block_contents::build_blinded_block_contents( + fork_name, + block, + maybe_blobs, + )?; + // Pose as a V2 endpoint so we return the fork `version`. - fork_versioned_response(V2, fork_name, block) + fork_versioned_response(V2, fork_name, block_contents) .map(|response| warp::reply::json(&response).into_response()) .map(|res| add_consensus_version_header(res, fork_name)) }) diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 38c40b890db..3fe1f37c976 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -7,7 +7,7 @@ use beacon_chain::{ IntoGossipVerifiedBlockContents, NotifyExecutionLayer, }; use eth2::types::BroadcastValidation; -use eth2::types::SignedBlockContents; +use eth2::types::{FullPayloadContents, SignedBlockContents}; use execution_layer::ProvenancedPayload; use lighthouse_network::PubsubMessage; use network::NetworkMessage; @@ -267,15 +267,15 @@ pub async fn publish_block( - block: SignedBlockContents>, + block_contents: SignedBlockContents>, chain: Arc>, network_tx: &UnboundedSender>, log: Logger, validation_level: BroadcastValidation, ) -> Result<(), Rejection> { - let block_root = block.signed_block().canonical_root(); + let block_root = block_contents.signed_block().canonical_root(); let full_block: ProvenancedBlock> = - reconstruct_block(chain.clone(), block_root, block, log.clone()).await?; + reconstruct_block(chain.clone(), block_root, block_contents, log.clone()).await?; publish_block::( Some(block_root), full_block, @@ -293,25 +293,21 @@ pub async fn publish_blinded_block( pub async fn reconstruct_block( chain: Arc>, block_root: Hash256, - block: SignedBlockContents>, + block_contents: SignedBlockContents>, log: Logger, ) -> Result>, Rejection> { - let full_payload_opt = if let Ok(payload_header) = - block.signed_block().message().body().execution_payload() - { + let block = block_contents.signed_block(); + let full_payload_opt = if let Ok(payload_header) = block.message().body().execution_payload() { let el = chain.execution_layer.as_ref().ok_or_else(|| { warp_utils::reject::custom_server_error("Missing execution layer".to_string()) })?; // If the execution block hash is zero, use an empty payload. - let full_payload = if payload_header.block_hash() == ExecutionBlockHash::zero() { + let full_payload_contents = if payload_header.block_hash() == ExecutionBlockHash::zero() { let payload = FullPayload::default_at_fork( - chain.spec.fork_name_at_epoch( - block - .signed_block() - .slot() - .epoch(T::EthSpec::slots_per_epoch()), - ), + chain + .spec + .fork_name_at_epoch(block.slot().epoch(T::EthSpec::slots_per_epoch())), ) .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -319,7 +315,7 @@ pub async fn reconstruct_block( )) })? .into(); - ProvenancedPayload::Local(payload) + ProvenancedPayload::Local(FullPayloadContents::Payload(payload)) // If we already have an execution payload with this transactions root cached, use it. } else if let Some(cached_payload) = el.get_payload_by_root(&payload_header.tree_hash_root()) @@ -336,14 +332,14 @@ pub async fn reconstruct_block( late_block_logging( &chain, timestamp_now(), - block.signed_block().message(), + block.message(), block_root, "builder", &log, ); let full_payload = el - .propose_blinded_beacon_block(block_root, &block) + .propose_blinded_beacon_block(block_root, &block_contents) .await .map_err(|e| { warp_utils::reject::custom_server_error(format!( @@ -355,7 +351,7 @@ pub async fn reconstruct_block( ProvenancedPayload::Builder(full_payload) }; - Some(full_payload) + Some(full_payload_contents) } else { None }; @@ -363,23 +359,14 @@ pub async fn reconstruct_block( match full_payload_opt { // A block without a payload is pre-merge and we consider it locally // built. - None => block - .deconstruct() - .0 - .try_into_full_block(None) - .map(SignedBlockContents::Block) + None => block_contents + .try_into_full_block_and_blobs(None) .map(ProvenancedBlock::local), - Some(ProvenancedPayload::Local(full_payload)) => block - .deconstruct() - .0 - .try_into_full_block(Some(full_payload)) - .map(SignedBlockContents::Block) + Some(ProvenancedPayload::Local(full_payload_contents)) => block_contents + .try_into_full_block_and_blobs(Some(full_payload_contents)) .map(ProvenancedBlock::local), - Some(ProvenancedPayload::Builder(full_payload)) => block - .deconstruct() - .0 - .try_into_full_block(Some(full_payload)) - .map(SignedBlockContents::Block) + Some(ProvenancedPayload::Builder(full_payload_contents)) => block_contents + .try_into_full_block_and_blobs(Some(full_payload_contents)) .map(ProvenancedBlock::builder), } .ok_or_else(|| { diff --git a/beacon_node/http_api/tests/tests.rs b/beacon_node/http_api/tests/tests.rs index 402f0707281..de2cd4aeaf4 100644 --- a/beacon_node/http_api/tests/tests.rs +++ b/beacon_node/http_api/tests/tests.rs @@ -2576,12 +2576,16 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); + let signed_block_contents = + SignedBlockContents::::Block(signed_block.clone()); self.client - .post_beacon_blinded_blocks(&signed_block) + .post_beacon_blinded_blocks(&signed_block_contents) .await .unwrap(); @@ -2636,7 +2640,9 @@ impl ApiTester { .get_validator_blinded_blocks::(slot, &randao_reveal, None) .await .unwrap() - .data; + .data + .deconstruct() + .0; let signed_block = block.sign(&sk, &fork, genesis_validators_root, &self.chain.spec); @@ -2659,7 +2665,7 @@ impl ApiTester { for _ in 0..E::slots_per_epoch() { let slot = self.chain.slot().unwrap(); - let block = self + let block_contents = self .client .get_validator_blinded_blocks_modular::( slot, @@ -2670,7 +2676,7 @@ impl ApiTester { .await .unwrap() .data; - assert_eq!(block.slot(), slot); + assert_eq!(block_contents.block().slot(), slot); self.chain.slot_clock.set_slot(slot.as_u64() + 1); } @@ -3206,6 +3212,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3246,6 +3253,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3289,6 +3297,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3338,6 +3347,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3386,6 +3396,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3433,6 +3444,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3479,6 +3491,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3515,6 +3528,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3552,6 +3566,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3595,6 +3610,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3624,6 +3640,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3673,6 +3690,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3712,6 +3730,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3755,6 +3774,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3796,6 +3816,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3833,6 +3854,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3870,6 +3892,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3907,6 +3930,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3957,6 +3981,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() @@ -3999,6 +4024,7 @@ impl ApiTester { .await .unwrap() .data + .block() .body() .execution_payload() .unwrap() diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index b0217e5b74b..5793adfbb57 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -12,7 +12,6 @@ use beacon_chain::builder::Witness; use beacon_chain::eth1_chain::CachingEth1Backend; use beacon_chain::test_utils::{build_log, BeaconChainHarness, EphemeralHarnessType}; use beacon_processor::WorkEvent; -use execution_layer::BlobsBundleV1; use lighthouse_network::rpc::RPCResponseErrorCode; use lighthouse_network::{NetworkGlobals, Request}; use slot_clock::{ManualSlotClock, SlotClock, TestingSlotClock}; @@ -21,8 +20,8 @@ use tokio::sync::mpsc; use types::{ map_fork_name, map_fork_name_with, test_utils::{SeedableRng, TestRandom, XorShiftRng}, - BeaconBlock, BlobSidecar, EthSpec, ForkName, FullPayloadDeneb, MinimalEthSpec as E, - SignedBeaconBlock, + BeaconBlock, BlobSidecar, BlobsBundle, EthSpec, ForkName, FullPayloadDeneb, + MinimalEthSpec as E, SignedBeaconBlock, }; type T = Witness, E, MemoryStore, MemoryStore>; @@ -126,7 +125,7 @@ impl TestRig { } message.body.blob_kzg_commitments = bundle.commitments.clone(); - let BlobsBundleV1 { + let BlobsBundle { commitments, proofs, blobs, diff --git a/common/eth2/Cargo.toml b/common/eth2/Cargo.toml index d15c56d9b05..d0cb62da29c 100644 --- a/common/eth2/Cargo.toml +++ b/common/eth2/Cargo.toml @@ -9,6 +9,7 @@ edition = "2021" [dependencies] serde = { version = "1.0.116", features = ["derive"] } serde_json = "1.0.58" +ssz_types = "0.5.4" types = { path = "../../consensus/types" } reqwest = { version = "0.11.0", features = ["json", "stream"] } lighthouse_network = { path = "../../beacon_node/lighthouse_network" } diff --git a/common/eth2/src/lib.rs b/common/eth2/src/lib.rs index 820ca23bdb9..7c9a8a23b7a 100644 --- a/common/eth2/src/lib.rs +++ b/common/eth2/src/lib.rs @@ -727,7 +727,7 @@ impl BeaconNodeHttpClient { /// Returns `Ok(None)` on a 404 error. pub async fn post_beacon_blinded_blocks>( &self, - block: &SignedBeaconBlock, + block: &SignedBlockContents, ) -> Result<(), Error> { let mut path = self.eth_path(V1)?; @@ -1648,7 +1648,7 @@ impl BeaconNodeHttpClient { slot: Slot, randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, - ) -> Result>, Error> { + ) -> Result>, Error> { self.get_validator_blinded_blocks_modular( slot, randao_reveal, @@ -1668,7 +1668,7 @@ impl BeaconNodeHttpClient { randao_reveal: &SignatureBytes, graffiti: Option<&Graffiti>, skip_randao_verification: SkipRandaoVerification, - ) -> Result>, Error> { + ) -> Result>, Error> { let mut path = self.eth_path(V1)?; path.path_segments_mut() diff --git a/common/eth2/src/types.rs b/common/eth2/src/types.rs index cba01fa2660..a10e7b22486 100644 --- a/common/eth2/src/types.rs +++ b/common/eth2/src/types.rs @@ -1361,28 +1361,41 @@ mod tests { } /// A wrapper over a [`BeaconBlock`] or a [`BeaconBlockAndBlobSidecars`]. -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Debug, Serialize, Deserialize)] #[serde(untagged)] #[serde(bound = "T: EthSpec")] pub enum BlockContents> { BlockAndBlobSidecars(BeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(BlindedBeaconBlockAndBlobSidecars), Block(BeaconBlock), } +pub type BlockContentsTuple = ( + BeaconBlock, + Option>::Sidecar>>, +); + impl> BlockContents { pub fn block(&self) -> &BeaconBlock { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => &block_and_sidecars.block, + BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.blinded_block + } BlockContents::Block(block) => block, } } - pub fn deconstruct(self) -> (BeaconBlock, Option>) { + pub fn deconstruct(self) -> BlockContentsTuple { match self { BlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.block, Some(block_and_sidecars.blob_sidecars), ), + BlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.blinded_block, + Some(block_and_sidecars.blinded_blob_sidecars), + ), BlockContents::Block(block) => (block, None), } } @@ -1415,14 +1428,17 @@ impl> Into> fn into(self) -> BeaconBlock { match self { Self::BlockAndBlobSidecars(block_and_sidecars) => block_and_sidecars.block, + Self::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + block_and_sidecars.blinded_block + } Self::Block(block) => block, } } } -pub type BlockContentsTuple = ( +pub type SignedBlockContentsTuple = ( SignedBeaconBlock, - Option>, + Option>::Sidecar>>, ); /// A wrapper over a [`SignedBeaconBlock`] or a [`SignedBeaconBlockAndBlobSidecars`]. @@ -1432,21 +1448,29 @@ pub type BlockContentsTuple = ( #[ssz(enum_behaviour = "transparent")] pub enum SignedBlockContents = FullPayload> { BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars), + BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars), Block(SignedBeaconBlock), } impl> SignedBlockContents { pub fn new( block: SignedBeaconBlock, - blobs: Option>, + blobs: Option>, ) -> Self { - if let Some(blobs) = blobs { - Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { - signed_block: block, - signed_blob_sidecars: blobs, - }) - } else { - Self::Block(block) + match (Payload::block_type(), blobs) { + (BlockType::Blinded, Some(blobs)) => { + Self::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block: block, + signed_blob_sidecars: blobs, + }) + } + (BlockType::Full, Some(blobs)) => { + Self::BlindedBlockAndBlobSidecars(SignedBlindedBeaconBlockAndBlobSidecars { + signed_blinded_block: block, + signed_blinded_blob_sidecars: blobs, + }) + } + (_, None) => Self::Block(block), } } @@ -1462,30 +1486,88 @@ impl> SignedBlockContents { &block_and_sidecars.signed_block } + SignedBlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => { + &block_and_sidecars.signed_blinded_block + } SignedBlockContents::Block(block) => block, } } - pub fn blobs_cloned(&self) -> Option> { + pub fn blobs_cloned(&self) -> Option> { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => { Some(block_and_sidecars.signed_blob_sidecars.clone()) } SignedBlockContents::Block(_block) => None, + SignedBlockContents::BlindedBlockAndBlobSidecars(_) => None, } } - pub fn deconstruct(self) -> BlockContentsTuple { + pub fn deconstruct(self) -> SignedBlockContentsTuple { match self { SignedBlockContents::BlockAndBlobSidecars(block_and_sidecars) => ( block_and_sidecars.signed_block, Some(block_and_sidecars.signed_blob_sidecars), ), + SignedBlockContents::BlindedBlockAndBlobSidecars(block_and_sidecars) => ( + block_and_sidecars.signed_blinded_block, + Some(block_and_sidecars.signed_blinded_blob_sidecars), + ), SignedBlockContents::Block(block) => (block, None), } } } +impl SignedBlockContents> { + pub fn try_into_full_block_and_blobs( + self, + maybe_full_payload_contents: Option>, + ) -> Option>> { + match self { + SignedBlockContents::BlindedBlockAndBlobSidecars(blinded_block_and_blob_sidecars) => { + maybe_full_payload_contents.and_then(|full_payload_contents| { + match full_payload_contents.deconstruct() { + (full_payload, Some(blobs_bundle)) => { + let maybe_full_block = blinded_block_and_blob_sidecars + .signed_blinded_block + .try_into_full_block(Some(full_payload)); + let full_blob_sidecars: Vec<_> = blinded_block_and_blob_sidecars + .signed_blinded_blob_sidecars + .into_iter() + .zip(blobs_bundle.blobs) + .map(|(blinded_blob_sidecar, blob)| { + blinded_blob_sidecar.into_full_blob_sidecars(blob) + }) + .collect(); + + maybe_full_block.map(|signed_block| { + SignedBlockContents::BlockAndBlobSidecars( + SignedBeaconBlockAndBlobSidecars { + signed_block, + signed_blob_sidecars: VariableList::from( + full_blob_sidecars, + ), + }, + ) + }) + } + // Can't build full block contents without full blobs + _ => None, + } + }) + } + SignedBlockContents::Block(blinded_block) => { + let full_payload_opt = maybe_full_payload_contents.map(|o| o.deconstruct().0); + blinded_block + .try_into_full_block(full_payload_opt) + .map(SignedBlockContents::Block) + } + // Unexpected scenario for blinded block proposal + SignedBlockContents::BlockAndBlobSidecars(_) => None, + } + } +} + impl> TryFrom> for SignedBlockContents { @@ -1503,18 +1585,26 @@ impl> TryFrom> From> +impl> From> for SignedBlockContents { - fn from(block_contents_tuple: BlockContentsTuple) -> Self { + fn from(block_contents_tuple: SignedBlockContentsTuple) -> Self { match block_contents_tuple { (signed_block, None) => SignedBlockContents::Block(signed_block), - (signed_block, Some(signed_blob_sidecars)) => { - SignedBlockContents::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { - signed_block, - signed_blob_sidecars, - }) - } + (signed_block, Some(signed_blob_sidecars)) => match Payload::block_type() { + BlockType::Blinded => SignedBlockContents::BlindedBlockAndBlobSidecars( + SignedBlindedBeaconBlockAndBlobSidecars { + signed_blinded_block: signed_block, + signed_blinded_blob_sidecars: signed_blob_sidecars, + }, + ), + BlockType::Full => { + SignedBlockContents::BlockAndBlobSidecars(SignedBeaconBlockAndBlobSidecars { + signed_block, + signed_blob_sidecars, + }) + } + }, } } } @@ -1523,14 +1613,14 @@ impl> From> { pub signed_block: SignedBeaconBlock, - pub signed_blob_sidecars: SignedBlobSidecarList, + pub signed_blob_sidecars: SignedSidecarList, } #[derive(Debug, Clone, Serialize, Deserialize, Encode)] #[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] pub struct BeaconBlockAndBlobSidecars> { pub block: BeaconBlock, - pub blob_sidecars: BlobSidecarList, + pub blob_sidecars: SidecarList, } impl> ForkVersionDeserialize @@ -1541,12 +1631,13 @@ impl> ForkVersionDeserialize fork_name: ForkName, ) -> Result { #[derive(Deserialize)] - #[serde(bound = "T: EthSpec")] - struct Helper { + #[serde(bound = "T: EthSpec, S: Sidecar")] + struct Helper> { block: serde_json::Value, - blob_sidecars: BlobSidecarList, + blob_sidecars: SidecarList, } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; + let helper: Helper = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; Ok(Self { block: BeaconBlock::deserialize_by_fork::<'de, D>(helper.block, fork_name)?, @@ -1554,3 +1645,49 @@ impl> ForkVersionDeserialize }) } } + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec")] +pub struct SignedBlindedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload = BlindedPayload, +> { + pub signed_blinded_block: SignedBeaconBlock, + pub signed_blinded_blob_sidecars: SignedSidecarList, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Encode)] +#[serde(bound = "T: EthSpec, Payload: AbstractExecPayload")] +pub struct BlindedBeaconBlockAndBlobSidecars< + T: EthSpec, + Payload: AbstractExecPayload = BlindedPayload, +> { + pub blinded_block: BeaconBlock, + pub blinded_blob_sidecars: SidecarList, +} + +impl> ForkVersionDeserialize + for BlindedBeaconBlockAndBlobSidecars +{ + fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( + value: serde_json::value::Value, + fork_name: ForkName, + ) -> Result { + #[derive(Deserialize)] + #[serde(bound = "T: EthSpec, S: Sidecar")] + struct Helper> { + blinded_block: serde_json::Value, + blinded_blob_sidecars: SidecarList, + } + let helper: Helper = + serde_json::from_value(value).map_err(serde::de::Error::custom)?; + + Ok(Self { + blinded_block: BeaconBlock::deserialize_by_fork::<'de, D>( + helper.blinded_block, + fork_name, + )?, + blinded_blob_sidecars: helper.blinded_blob_sidecars, + }) + } +} diff --git a/consensus/types/Cargo.toml b/consensus/types/Cargo.toml index 9c105404b19..1b8aa203775 100644 --- a/consensus/types/Cargo.toml +++ b/consensus/types/Cargo.toml @@ -51,7 +51,6 @@ superstruct = "0.6.0" metastruct = "0.1.0" serde_json = "1.0.74" smallvec = "1.8.0" -serde_with = "1.13.0" maplit = "1.0.2" strum = { version = "0.24.0", features = ["derive"] } diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 214e7676403..b51cc0f323f 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,17 +1,109 @@ -use crate::test_utils::TestRandom; -use crate::{Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, SignedBlobSidecar, SignedRoot, Slot}; -use bls::SecretKey; +use std::fmt::Debug; +use std::hash::Hash; +use std::marker::PhantomData; +use std::sync::Arc; + use derivative::Derivative; use kzg::{Kzg, KzgCommitment, KzgPreset, KzgProof}; use rand::Rng; +use serde::de::DeserializeOwned; use serde_derive::{Deserialize, Serialize}; -use ssz::Encode; +use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use ssz_types::{FixedVector, VariableList}; -use std::sync::Arc; -use test_random_derive::TestRandom; +use tree_hash::TreeHash; use tree_hash_derive::TreeHash; +use bls::SecretKey; +use test_random_derive::TestRandom; + +use crate::beacon_block_body::KzgCommitments; +use crate::test_utils::TestRandom; +use crate::{ + AbstractExecPayload, BeaconBlock, Blob, ChainSpec, Domain, EthSpec, Fork, Hash256, + SignedBlobSidecar, SignedRoot, Slot, +}; + +pub trait Sidecar: + serde::Serialize + + Clone + + DeserializeOwned + + Encode + + Decode + + Hash + + TreeHash + + TestRandom + + Debug + + SignedRoot + + Sync + + Send + + for<'a> arbitrary::Arbitrary<'a> +{ + type BlobItems: BlobItems; + fn slot(&self) -> Slot; + fn build_sidecar>( + blob_items: Self::BlobItems, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String>; +} + +pub trait BlobItems: Sync + Send + Sized { + fn try_from_blob_roots(roots: BlobRootsList) -> Result; + fn try_from_blobs(blobs: BlobsList) -> Result; + fn len(&self) -> usize; + fn is_empty(&self) -> bool; + fn blobs(&self) -> Option<&BlobsList>; +} + +impl BlobItems for BlobsList { + fn try_from_blob_roots(_roots: BlobRootsList) -> Result { + Err("Unexpected conversion from blob roots to blobs".to_string()) + } + + fn try_from_blobs(blobs: BlobsList) -> Result { + Ok(blobs) + } + + fn len(&self) -> usize { + VariableList::len(self) + } + + fn is_empty(&self) -> bool { + VariableList::is_empty(self) + } + + fn blobs(&self) -> Option<&BlobsList> { + Some(self) + } +} + +impl BlobItems for BlobRootsList { + fn try_from_blob_roots(roots: BlobRootsList) -> Result { + Ok(roots) + } + + fn try_from_blobs(_blobs: BlobsList) -> Result { + // It is possible to convert from blobs to blob roots, however this should be done using + // `From` or `Into` instead of this generic implementation; this function implementation + // should be unreachable, and attempt to use this indicates a bug somewhere. + Err("Unexpected conversion from blob to blob roots".to_string()) + } + + fn len(&self) -> usize { + VariableList::len(self) + } + + fn is_empty(&self) -> bool { + VariableList::is_empty(self) + } + + fn blobs(&self) -> Option<&BlobsList> { + None + } +} + /// Container of the data that identifies an individual blob. #[derive( Serialize, Deserialize, Encode, Decode, TreeHash, Copy, Clone, Debug, PartialEq, Eq, Hash, @@ -63,6 +155,67 @@ pub struct BlobSidecar { pub kzg_proof: KzgProof, } +impl Sidecar for BlobSidecar { + type BlobItems = BlobsList; + + fn slot(&self) -> Slot { + self.slot + } + + fn build_sidecar>( + blobs: BlobsList, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + let blob_sidecars = BlobSidecarList::from( + blobs + .into_iter() + .enumerate() + .map(|(blob_index, blob)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .ok_or("KZG commitment should exist for blob")?; + + let kzg_proof = kzg_proofs + .get(blob_index) + .ok_or("KZG proof should exist for blob")?; + + Ok(Arc::new(BlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } +} + +impl From>> for BlindedBlobSidecar { + fn from(blob_sidecar: Arc>) -> Self { + BlindedBlobSidecar { + block_root: blob_sidecar.block_root, + index: blob_sidecar.index, + slot: blob_sidecar.slot, + block_parent_root: blob_sidecar.block_parent_root, + proposer_index: blob_sidecar.proposer_index, + blob_root: blob_sidecar.blob.tree_hash_root(), + kzg_commitment: blob_sidecar.kzg_commitment, + kzg_proof: blob_sidecar.kzg_proof, + } + } +} + impl PartialOrd for BlobSidecar { fn partial_cmp(&self, other: &Self) -> Option { self.index.partial_cmp(&other.index) @@ -75,11 +228,6 @@ impl Ord for BlobSidecar { } } -pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; -pub type FixedBlobSidecarList = - FixedVector>>, ::MaxBlobsPerBlock>; -pub type Blobs = VariableList, ::MaxBlobsPerBlock>; - impl SignedRoot for BlobSidecar {} impl BlobSidecar { @@ -153,6 +301,94 @@ impl BlobSidecar { SignedBlobSidecar { message: self, signature, + _phantom: PhantomData, } } } + +#[derive( + Debug, + Clone, + Serialize, + Deserialize, + Encode, + Decode, + TreeHash, + Default, + TestRandom, + Derivative, + arbitrary::Arbitrary, +)] +#[derivative(PartialEq, Eq, Hash)] +pub struct BlindedBlobSidecar { + pub block_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub index: u64, + pub slot: Slot, + pub block_parent_root: Hash256, + #[serde(with = "serde_utils::quoted_u64")] + pub proposer_index: u64, + pub blob_root: Hash256, + pub kzg_commitment: KzgCommitment, + pub kzg_proof: KzgProof, +} + +impl SignedRoot for BlindedBlobSidecar {} + +impl Sidecar for BlindedBlobSidecar { + type BlobItems = BlobRootsList; + + fn slot(&self) -> Slot { + self.slot + } + + fn build_sidecar>( + blob_roots: BlobRootsList, + block: &BeaconBlock, + expected_kzg_commitments: &KzgCommitments, + kzg_proofs: Vec, + ) -> Result, String> { + let beacon_block_root = block.canonical_root(); + let slot = block.slot(); + + let blob_sidecars = BlindedBlobSidecarList::::from( + blob_roots + .into_iter() + .enumerate() + .map(|(blob_index, blob_root)| { + let kzg_commitment = expected_kzg_commitments + .get(blob_index) + .ok_or("KZG commitment should exist for blob")?; + + let kzg_proof = kzg_proofs.get(blob_index).ok_or(format!( + "Missing KZG proof for slot {} blob index: {}", + slot, blob_index + ))?; + + Ok(Arc::new(BlindedBlobSidecar { + block_root: beacon_block_root, + index: blob_index as u64, + slot, + block_parent_root: block.parent_root(), + proposer_index: block.proposer_index(), + blob_root, + kzg_commitment: *kzg_commitment, + kzg_proof: *kzg_proof, + })) + }) + .collect::, String>>()?, + ); + + Ok(blob_sidecars) + } +} + +pub type SidecarList = VariableList, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = SidecarList>; +pub type BlindedBlobSidecarList = SidecarList; + +pub type FixedBlobSidecarList = + FixedVector>>, ::MaxBlobsPerBlock>; + +pub type BlobsList = VariableList, ::MaxBlobsPerBlock>; +pub type BlobRootsList = VariableList::MaxBlobsPerBlock>; diff --git a/consensus/types/src/builder_bid.rs b/consensus/types/src/builder_bid.rs index 8723c2afed9..e0c0bb306e8 100644 --- a/consensus/types/src/builder_bid.rs +++ b/consensus/types/src/builder_bid.rs @@ -1,76 +1,97 @@ +use crate::beacon_block_body::KzgCommitments; use crate::{ - AbstractExecPayload, ChainSpec, EthSpec, ExecPayload, ExecutionPayloadHeader, ForkName, - ForkVersionDeserialize, SignedRoot, Uint256, + BlobRootsList, ChainSpec, EthSpec, ExecutionPayloadHeaderCapella, ExecutionPayloadHeaderDeneb, + ExecutionPayloadHeaderMerge, ExecutionPayloadHeaderRef, ForkName, ForkVersionDeserialize, + KzgProofs, SignedRoot, Uint256, }; use bls::PublicKeyBytes; use bls::Signature; -use serde::{Deserialize as De, Deserializer, Serialize as Ser, Serializer}; +use serde::Deserializer; use serde_derive::{Deserialize, Serialize}; -use serde_with::{serde_as, DeserializeAs, SerializeAs}; -use std::marker::PhantomData; +use superstruct::superstruct; use tree_hash_derive::TreeHash; -#[serde_as] #[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] -#[serde(bound = "E: EthSpec, Payload: ExecPayload")] -pub struct BuilderBid> { - #[serde_as(as = "BlindedPayloadAsHeader")] - pub header: Payload, +#[serde(bound = "E: EthSpec")] +pub struct BlindedBlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + pub blob_roots: BlobRootsList, +} + +#[superstruct( + variants(Merge, Capella, Deneb), + variant_attributes( + derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone), + serde(bound = "E: EthSpec", deny_unknown_fields) + ), + map_ref_into(ExecutionPayloadHeaderRef) +)] +#[derive(PartialEq, Debug, Serialize, Deserialize, TreeHash, Clone)] +#[serde(bound = "E: EthSpec", deny_unknown_fields, untagged)] +#[tree_hash(enum_behaviour = "transparent")] +pub struct BuilderBid { + #[superstruct(only(Merge), partial_getter(rename = "header_merge"))] + pub header: ExecutionPayloadHeaderMerge, + #[superstruct(only(Capella), partial_getter(rename = "header_capella"))] + pub header: ExecutionPayloadHeaderCapella, + #[superstruct(only(Deneb), partial_getter(rename = "header_deneb"))] + pub header: ExecutionPayloadHeaderDeneb, + #[superstruct(only(Deneb))] + pub blinded_blobs_bundle: BlindedBlobsBundle, #[serde(with = "serde_utils::quoted_u256")] pub value: Uint256, pub pubkey: PublicKeyBytes, - #[serde(skip)] - #[tree_hash(skip_hashing)] - _phantom_data: PhantomData, } -impl> SignedRoot for BuilderBid {} +impl BuilderBid { + pub fn header(&self) -> ExecutionPayloadHeaderRef<'_, E> { + self.to_ref().header() + } +} + +impl<'a, E: EthSpec> BuilderBidRef<'a, E> { + pub fn header(&self) -> ExecutionPayloadHeaderRef<'a, E> { + map_builder_bid_ref_into_execution_payload_header_ref!(&'a _, self, |bid, cons| cons( + &bid.header + )) + } +} + +impl SignedRoot for BuilderBid {} /// Validator registration, for use in interacting with servers implementing the builder API. #[derive(PartialEq, Debug, Serialize, Deserialize, Clone)] -#[serde(bound = "E: EthSpec, Payload: ExecPayload")] -pub struct SignedBuilderBid> { - pub message: BuilderBid, +#[serde(bound = "E: EthSpec")] +pub struct SignedBuilderBid { + pub message: BuilderBid, pub signature: Signature, } -impl> ForkVersionDeserialize - for BuilderBid -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( +impl ForkVersionDeserialize for BuilderBid { + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { - let convert_err = |_| { - serde::de::Error::custom( - "BuilderBid failed to deserialize: unable to convert payload header to payload", - ) - }; - - #[derive(Deserialize)] - struct Helper { - header: serde_json::Value, - #[serde(with = "serde_utils::quoted_u256")] - value: Uint256, - pubkey: PublicKeyBytes, - } - let helper: Helper = serde_json::from_value(value).map_err(serde::de::Error::custom)?; - let payload_header = - ExecutionPayloadHeader::deserialize_by_fork::<'de, D>(helper.header, fork_name)?; + let convert_err = + |e| serde::de::Error::custom(format!("BuilderBid failed to deserialize: {:?}", e)); - Ok(Self { - header: Payload::try_from(payload_header).map_err(convert_err)?, - value: helper.value, - pubkey: helper.pubkey, - _phantom_data: Default::default(), + Ok(match fork_name { + ForkName::Merge => Self::Merge(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Capella => Self::Capella(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Deneb => Self::Deneb(serde_json::from_value(value).map_err(convert_err)?), + ForkName::Base | ForkName::Altair => { + return Err(serde::de::Error::custom(format!( + "BuilderBid failed to deserialize: unsupported fork '{}'", + fork_name + ))); + } }) } } -impl> ForkVersionDeserialize - for SignedBuilderBid -{ - fn deserialize_by_fork<'de, D: serde::Deserializer<'de>>( +impl ForkVersionDeserialize for SignedBuilderBid { + fn deserialize_by_fork<'de, D: Deserializer<'de>>( value: serde_json::value::Value, fork_name: ForkName, ) -> Result { @@ -88,34 +109,10 @@ impl> ForkVersionDeserialize } } -struct BlindedPayloadAsHeader(PhantomData); - -impl> SerializeAs for BlindedPayloadAsHeader { - fn serialize_as(source: &Payload, serializer: S) -> Result - where - S: Serializer, - { - source.to_execution_payload_header().serialize(serializer) - } -} - -impl<'de, E: EthSpec, Payload: AbstractExecPayload> DeserializeAs<'de, Payload> - for BlindedPayloadAsHeader -{ - fn deserialize_as(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let payload_header = ExecutionPayloadHeader::deserialize(deserializer)?; - Payload::try_from(payload_header) - .map_err(|_| serde::de::Error::custom("unable to convert payload header to payload")) - } -} - -impl> SignedBuilderBid { +impl SignedBuilderBid { pub fn verify_signature(&self, spec: &ChainSpec) -> bool { self.message - .pubkey + .pubkey() .decompress() .map(|pubkey| { let domain = spec.get_builder_domain(); diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 3e6ced0d686..5008bdf6320 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -119,7 +119,10 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{BeaconTreeHashCache, Error as BeaconStateError, *}; -pub use crate::blob_sidecar::{BlobSidecar, BlobSidecarList}; +pub use crate::blob_sidecar::{ + BlindedBlobSidecar, BlindedBlobSidecarList, BlobRootsList, BlobSidecar, BlobSidecarList, + BlobsList, Sidecar, SidecarList, +}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; @@ -158,8 +161,9 @@ pub use crate::participation_flags::ParticipationFlags; pub use crate::participation_list::ParticipationList; pub use crate::payload::{ AbstractExecPayload, BlindedPayload, BlindedPayloadCapella, BlindedPayloadDeneb, - BlindedPayloadMerge, BlindedPayloadRef, BlockType, ExecPayload, FullPayload, - FullPayloadCapella, FullPayloadDeneb, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, + BlindedPayloadMerge, BlindedPayloadRef, BlobsBundle, BlockType, ExecPayload, + ExecutionPayloadAndBlobs, FullPayload, FullPayloadCapella, FullPayloadContents, + FullPayloadDeneb, FullPayloadMerge, FullPayloadRef, OwnedExecPayload, }; pub use crate::pending_attestation::PendingAttestation; pub use crate::preset::{AltairPreset, BasePreset, BellatrixPreset, CapellaPreset}; diff --git a/consensus/types/src/payload.rs b/consensus/types/src/payload.rs index fd15bb5d5dc..8016aaad774 100644 --- a/consensus/types/src/payload.rs +++ b/consensus/types/src/payload.rs @@ -1,7 +1,9 @@ +use crate::beacon_block_body::KzgCommitments; use crate::{test_utils::TestRandom, *}; use derivative::Derivative; use serde::de::DeserializeOwned; -use serde::{Deserialize, Serialize}; +use serde::{Deserialize, Deserializer, Serialize}; +use serde_json::Value; use ssz::{Decode, Encode}; use ssz_derive::{Decode, Encode}; use std::borrow::Cow; @@ -83,6 +85,8 @@ pub trait AbstractExecPayload: + TryInto + TryInto { + type Sidecar: Sidecar; + type Ref<'a>: ExecPayload + Copy + From<&'a Self::Merge> @@ -103,6 +107,9 @@ pub trait AbstractExecPayload: + TryFrom>; fn default_at_fork(fork_name: ForkName) -> Result; + fn default_blobs_at_fork( + fork_name: ForkName, + ) -> Result<>::BlobItems, Error>; } #[superstruct( @@ -379,6 +386,7 @@ impl<'b, T: EthSpec> ExecPayload for FullPayloadRef<'b, T> { } impl AbstractExecPayload for FullPayload { + type Sidecar = BlobSidecar; type Ref<'a> = FullPayloadRef<'a, T>; type Merge = FullPayloadMerge; type Capella = FullPayloadCapella; @@ -392,6 +400,9 @@ impl AbstractExecPayload for FullPayload { ForkName::Deneb => Ok(FullPayloadDeneb::default().into()), } } + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { + Ok(VariableList::default()) + } } impl From> for FullPayload { @@ -897,6 +908,8 @@ impl AbstractExecPayload for BlindedPayload { type Capella = BlindedPayloadCapella; type Deneb = BlindedPayloadDeneb; + type Sidecar = BlindedBlobSidecar; + fn default_at_fork(fork_name: ForkName) -> Result { match fork_name { ForkName::Base | ForkName::Altair => Err(Error::IncorrectStateVariant), @@ -905,6 +918,9 @@ impl AbstractExecPayload for BlindedPayload { ForkName::Deneb => Ok(BlindedPayloadDeneb::default().into()), } } + fn default_blobs_at_fork(_fork_name: ForkName) -> Result, Error> { + Ok(VariableList::default()) + } } impl From> for BlindedPayload { @@ -955,3 +971,84 @@ impl From> for ExecutionPayloadHeader { } } } + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(untagged)] +#[serde(bound = "E: EthSpec")] +pub enum FullPayloadContents { + Payload(ExecutionPayload), + PayloadAndBlobs(ExecutionPayloadAndBlobs), +} + +impl FullPayloadContents { + pub fn new( + execution_payload: ExecutionPayload, + maybe_blobs: Option>, + ) -> Self { + match maybe_blobs { + None => Self::Payload(execution_payload), + Some(blobs_bundle) => Self::PayloadAndBlobs(ExecutionPayloadAndBlobs { + execution_payload, + blobs_bundle, + }), + } + } + + pub fn payload_ref(&self) -> &ExecutionPayload { + match self { + FullPayloadContents::Payload(payload) => payload, + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => { + &payload_and_blobs.execution_payload + } + } + } + + pub fn block_hash(&self) -> ExecutionBlockHash { + self.payload_ref().block_hash() + } + + pub fn deconstruct(self) -> (ExecutionPayload, Option>) { + match self { + FullPayloadContents::Payload(payload) => (payload, None), + FullPayloadContents::PayloadAndBlobs(payload_and_blobs) => ( + payload_and_blobs.execution_payload, + Some(payload_and_blobs.blobs_bundle), + ), + } + } +} + +impl ForkVersionDeserialize for FullPayloadContents { + fn deserialize_by_fork<'de, D: Deserializer<'de>>( + value: Value, + fork_name: ForkName, + ) -> Result { + match fork_name { + ForkName::Merge | ForkName::Capella => serde_json::from_value(value) + .map(Self::Payload) + .map_err(serde::de::Error::custom), + ForkName::Deneb => serde_json::from_value(value) + .map(Self::PayloadAndBlobs) + .map_err(serde::de::Error::custom), + ForkName::Base | ForkName::Altair => Err(serde::de::Error::custom(format!( + "FullPayloadContents deserialization for {fork_name} not implemented" + ))), + } + } +} + +#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct ExecutionPayloadAndBlobs { + pub execution_payload: ExecutionPayload, + pub blobs_bundle: BlobsBundle, +} + +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[serde(bound = "E: EthSpec")] +pub struct BlobsBundle { + pub commitments: KzgCommitments, + pub proofs: KzgProofs, + #[serde(with = "ssz_types::serde_utils::list_of_hex_fixed_vec")] + pub blobs: BlobsList, +} diff --git a/consensus/types/src/signed_blob.rs b/consensus/types/src/signed_blob.rs index aaab02ca783..7d1c553d923 100644 --- a/consensus/types/src/signed_blob.rs +++ b/consensus/types/src/signed_blob.rs @@ -1,12 +1,13 @@ use crate::{ - test_utils::TestRandom, BlobSidecar, ChainSpec, Domain, EthSpec, Fork, Hash256, Signature, - SignedRoot, SigningData, + test_utils::TestRandom, BlindedBlobSidecar, Blob, BlobSidecar, ChainSpec, Domain, EthSpec, + Fork, Hash256, Sidecar, Signature, SignedRoot, SigningData, }; use bls::PublicKey; use derivative::Derivative; use serde_derive::{Deserialize, Serialize}; use ssz_derive::{Decode, Encode}; use ssz_types::VariableList; +use std::marker::PhantomData; use std::sync::Arc; use test_random_derive::TestRandom; use tree_hash::TreeHash; @@ -25,16 +26,45 @@ use tree_hash_derive::TreeHash; Derivative, arbitrary::Arbitrary, )] -#[serde(bound = "T: EthSpec")] -#[arbitrary(bound = "T: EthSpec")] -#[derivative(Hash(bound = "T: EthSpec"))] -pub struct SignedBlobSidecar { - pub message: Arc>, +#[serde(bound = "T: EthSpec, S: Sidecar")] +#[arbitrary(bound = "T: EthSpec, S: Sidecar")] +#[derivative(Hash(bound = "T: EthSpec, S: Sidecar"))] +pub struct SignedSidecar> { + pub message: Arc, pub signature: Signature, + #[ssz(skip_serializing, skip_deserializing)] + #[tree_hash(skip_hashing)] + #[serde(skip)] + #[arbitrary(default)] + pub _phantom: PhantomData, } -pub type SignedBlobSidecarList = - VariableList, ::MaxBlobsPerBlock>; +impl SignedSidecar { + pub fn into_full_blob_sidecars(self, blob: Blob) -> SignedSidecar> { + let blinded_sidecar = self.message; + SignedSidecar { + message: Arc::new(BlobSidecar { + block_root: blinded_sidecar.block_root, + index: blinded_sidecar.index, + slot: blinded_sidecar.slot, + block_parent_root: blinded_sidecar.block_parent_root, + proposer_index: blinded_sidecar.proposer_index, + blob, + kzg_commitment: blinded_sidecar.kzg_commitment, + kzg_proof: blinded_sidecar.kzg_proof, + }), + signature: self.signature, + _phantom: PhantomData, + } + } +} + +/// List of Signed Sidecars that implements `Sidecar`. +pub type SignedSidecarList = + VariableList, ::MaxBlobsPerBlock>; +pub type SignedBlobSidecarList = SignedSidecarList>; + +pub type SignedBlobSidecar = SignedSidecar>; impl SignedBlobSidecar { /// Verify `self.signature`. diff --git a/validator_client/src/block_service.rs b/validator_client/src/block_service.rs index e59838991cb..08e35e3a08f 100644 --- a/validator_client/src/block_service.rs +++ b/validator_client/src/block_service.rs @@ -525,7 +525,7 @@ impl BlockService { Some(blob_sidecars) => { match self_ref .validator_store - .sign_blobs(*validator_pubkey_ref, blob_sidecars) + .sign_blobs::(*validator_pubkey_ref, blob_sidecars) .await { Ok(signed_blobs) => Some(signed_blobs), @@ -620,16 +620,15 @@ impl BlockService { &metrics::BLOCK_SERVICE_TIMES, &[metrics::BLINDED_BEACON_BLOCK_HTTP_POST], ); - todo!("need to be adjusted for blobs"); - // beacon_node - // .post_beacon_blinded_blocks(signed_block_contents.signed_block()) - // .await - // .map_err(|e| { - // BlockError::Irrecoverable(format!( - // "Error from beacon node when publishing block: {:?}", - // e - // )) - // })? + beacon_node + .post_beacon_blinded_blocks(signed_block_contents) + .await + .map_err(|e| { + BlockError::Irrecoverable(format!( + "Error from beacon node when publishing block: {:?}", + e + )) + })? } } Ok::<_, BlockError>(()) @@ -665,21 +664,20 @@ impl BlockService { &metrics::BLOCK_SERVICE_TIMES, &[metrics::BLINDED_BEACON_BLOCK_HTTP_GET], ); - todo!("implement blinded flow for blobs"); - // beacon_node - // .get_validator_blinded_blocks::( - // slot, - // randao_reveal_ref, - // graffiti.as_ref(), - // ) - // .await - // .map_err(|e| { - // BlockError::Recoverable(format!( - // "Error from beacon node when producing block: {:?}", - // e - // )) - // })? - // .data + beacon_node + .get_validator_blinded_blocks::( + slot, + randao_reveal_ref, + graffiti.as_ref(), + ) + .await + .map_err(|e| { + BlockError::Recoverable(format!( + "Error from beacon node when producing block: {:?}", + e + )) + })? + .data } }; diff --git a/validator_client/src/signing_method.rs b/validator_client/src/signing_method.rs index 5291ad6ddc0..96bfd2511f1 100644 --- a/validator_client/src/signing_method.rs +++ b/validator_client/src/signing_method.rs @@ -37,7 +37,7 @@ pub enum Error { pub enum SignableMessage<'a, T: EthSpec, Payload: AbstractExecPayload = FullPayload> { RandaoReveal(Epoch), BeaconBlock(&'a BeaconBlock), - BlobSidecar(&'a BlobSidecar), + BlobSidecar(&'a Payload::Sidecar), AttestationData(&'a AttestationData), SignedAggregateAndProof(&'a AggregateAndProof), SelectionProof(Slot), diff --git a/validator_client/src/validator_store.rs b/validator_client/src/validator_store.rs index 9492ad588f9..ffe7fd08328 100644 --- a/validator_client/src/validator_store.rs +++ b/validator_client/src/validator_store.rs @@ -20,10 +20,10 @@ use std::sync::Arc; use task_executor::TaskExecutor; use types::{ attestation::Error as AttestationError, graffiti::GraffitiString, AbstractExecPayload, Address, - AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, BlobSidecarList, ChainSpec, - ContributionAndProof, Domain, Epoch, EthSpec, Fork, ForkName, Graffiti, Hash256, Keypair, - PublicKeyBytes, SelectionProof, Signature, SignedAggregateAndProof, SignedBeaconBlock, - SignedBlobSidecar, SignedBlobSidecarList, SignedContributionAndProof, SignedRoot, + AggregateAndProof, Attestation, BeaconBlock, BlindedPayload, ChainSpec, ContributionAndProof, + Domain, Epoch, EthSpec, Fork, ForkName, Graffiti, Hash256, Keypair, PublicKeyBytes, + SelectionProof, Sidecar, SidecarList, Signature, SignedAggregateAndProof, SignedBeaconBlock, + SignedContributionAndProof, SignedRoot, SignedSidecar, SignedSidecarList, SignedValidatorRegistrationData, SignedVoluntaryExit, Slot, SyncAggregatorSelectionData, SyncCommitteeContribution, SyncCommitteeMessage, SyncSelectionProof, SyncSubnetId, ValidatorRegistrationData, VoluntaryExit, @@ -566,22 +566,21 @@ impl ValidatorStore { } } - pub async fn sign_blobs( + pub async fn sign_blobs>( &self, validator_pubkey: PublicKeyBytes, - blob_sidecars: BlobSidecarList, - ) -> Result, Error> { + blob_sidecars: SidecarList, + ) -> Result, Error> { let mut signed_blob_sidecars = Vec::new(); - for blob_sidecar in blob_sidecars.into_iter() { - let slot = blob_sidecar.slot; + let slot = blob_sidecar.slot(); let signing_epoch = slot.epoch(E::slots_per_epoch()); let signing_context = self.signing_context(Domain::BlobSidecar, signing_epoch); let signing_method = self.doppelganger_checked_signing_method(validator_pubkey)?; let signature = signing_method - .get_signature::>( - SignableMessage::BlobSidecar(&blob_sidecar), + .get_signature::( + SignableMessage::BlobSidecar(blob_sidecar.as_ref()), signing_context, &self.spec, &self.task_executor, @@ -590,9 +589,10 @@ impl ValidatorStore { metrics::inc_counter_vec(&metrics::SIGNED_BLOBS_TOTAL, &[metrics::SUCCESS]); - signed_blob_sidecars.push(SignedBlobSidecar { + signed_blob_sidecars.push(SignedSidecar { message: blob_sidecar, signature, + _phantom: PhantomData, }); }