From 25feedfde348b530c4fa2348cc71a06b746898ed Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Thu, 29 Aug 2024 16:11:19 -0700 Subject: [PATCH 1/9] First pass --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +- .../beacon_chain/src/blob_verification.rs | 10 ++- .../beacon_chain/src/block_verification.rs | 4 +- .../src/block_verification_types.rs | 17 ------ .../src/data_availability_checker.rs | 7 +-- .../overflow_lru_cache.rs | 47 ++++++++------ beacon_node/beacon_chain/src/kzg_utils.rs | 7 ++- .../src/observed_data_sidecars.rs | 14 +++-- beacon_node/beacon_chain/src/test_utils.rs | 10 +-- beacon_node/client/src/builder.rs | 3 +- .../test_utils/execution_block_generator.rs | 3 +- beacon_node/http_api/src/block_id.rs | 3 +- beacon_node/http_api/src/publish_blocks.rs | 8 ++- .../lighthouse_network/src/rpc/methods.rs | 14 ++++- .../lighthouse_network/src/rpc/protocol.rs | 6 +- .../src/sync/block_sidecar_coupling.rs | 16 +++-- beacon_node/network/src/sync/manager.rs | 2 + .../network/src/sync/network_context.rs | 34 +++++++++-- beacon_node/store/src/hot_cold_store.rs | 18 +++++- .../store/src/impls/execution_payload.rs | 3 +- .../src/per_block_processing.rs | 6 +- consensus/types/src/beacon_block_body.rs | 2 - consensus/types/src/blob_sidecar.rs | 33 +++++----- consensus/types/src/chain_spec.rs | 26 ++++++++ consensus/types/src/data_column_sidecar.rs | 14 ++--- consensus/types/src/eth_spec.rs | 12 +--- consensus/types/src/lib.rs | 4 +- consensus/types/src/preset.rs | 3 - consensus/types/src/runtime_var_list.rs | 61 +++++++++++++++++++ 29 files changed, 262 insertions(+), 130 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index 9d744003360..e1cf40f65f6 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1249,7 +1249,10 @@ impl BeaconChain { pub fn get_blobs(&self, block_root: &Hash256) -> Result, Error> { match self.store.get_blobs(block_root)? { Some(blobs) => Ok(blobs), - None => Ok(BlobSidecarList::default()), + None => Ok(BlobSidecarList::empty( + // TODO(pawan): fix this + self.spec.max_blobs_per_block(Epoch::new(0)) as usize, + )), } } diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 99fc5d9d0c0..4a39a9d2531 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -17,7 +17,8 @@ use std::time::Duration; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; use types::{ - BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, SignedBeaconBlockHeader, Slot, + BeaconStateError, BlobSidecar, Epoch, EthSpec, Hash256, RuntimeVariableList, + SignedBeaconBlockHeader, Slot, }; /// An error occurred while validating a gossip blob. @@ -172,10 +173,7 @@ impl From for GossipBlobError { } } -pub type GossipVerifiedBlobList = VariableList< - GossipVerifiedBlob, - <::EthSpec as EthSpec>::MaxBlobsPerBlock, ->; +pub type GossipVerifiedBlobList = RuntimeVariableList>; /// A wrapper around a `BlobSidecar` that indicates it has been approved for re-gossiping on /// the p2p network. @@ -399,7 +397,7 @@ pub fn validate_blob_sidecar_for_gossip( // since we only subscribe to `MaxBlobsPerBlock` subnets over gossip network. // We include this check only for completeness. // Getting this error would imply something very wrong with our networking decoding logic. - if blob_index >= T::EthSpec::max_blobs_per_block() as u64 { + if blob_index >= chain.spec.max_blobs_per_block(blob_epoch) { return Err(GossipBlobError::InvalidSubnet { expected: subnet, received: blob_index, diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index d9662d59f9e..6582096befb 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -779,7 +779,9 @@ fn build_gossip_verified_blobs( GossipVerifiedBlob::new(Arc::new(blob), i as u64, chain)?; gossip_verified_blobs.push(gossip_verified_blob); } - let gossip_verified_blobs = VariableList::from(gossip_verified_blobs); + let max_len = chain.spec.max_blobs_per_block(block.epoch()) as usize; + let gossip_verified_blobs = + RuntimeVariableList::from_vec(gossip_verified_blobs, max_len); Ok::<_, BlockContentsError>(gossip_verified_blobs) }) .transpose() diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index b271f0a2f98..67643318f2c 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -181,23 +181,6 @@ impl RpcBlock { }) } - pub fn new_from_fixed( - block_root: Hash256, - block: Arc>, - blobs: FixedBlobSidecarList, - ) -> Result { - let filtered = blobs - .into_iter() - .filter_map(|b| b.clone()) - .collect::>(); - let blobs = if filtered.is_empty() { - None - } else { - Some(VariableList::from(filtered)) - }; - Self::new(Some(block_root), block, blobs) - } - #[allow(clippy::type_complexity)] pub fn deconstruct( self, diff --git a/beacon_node/beacon_chain/src/data_availability_checker.rs b/beacon_node/beacon_chain/src/data_availability_checker.rs index 470cee713fa..cef7c528f6a 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker.rs @@ -200,7 +200,7 @@ impl DataAvailabilityChecker { .ok_or(AvailabilityCheckError::SlotClockError)?; let verified_blobs = - KzgVerifiedBlobList::new(Vec::from(blobs).into_iter().flatten(), kzg, seen_timestamp) + KzgVerifiedBlobList::new(blobs.into_vec().into_iter().flatten(), kzg, seen_timestamp) .map_err(AvailabilityCheckError::Kzg)?; self.availability_cache @@ -384,14 +384,13 @@ impl DataAvailabilityChecker { blocks: Vec>, ) -> Result>, AvailabilityCheckError> { let mut results = Vec::with_capacity(blocks.len()); - let all_blobs: BlobSidecarList = blocks + let all_blobs = blocks .iter() .filter(|block| self.blobs_required_for_block(block.as_block())) // this clone is cheap as it's cloning an Arc .filter_map(|block| block.blobs().cloned()) .flatten() - .collect::>() - .into(); + .collect::>(); // verify kzg for all blobs at once if !all_blobs.is_empty() { diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 4863982b552..9c33051ae02 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -16,9 +16,10 @@ use std::collections::HashSet; use std::num::NonZeroUsize; use std::sync::Arc; use types::blob_sidecar::BlobIdentifier; +use types::runtime_var_list::RuntimeFixedList; use types::{ BlobSidecar, ChainSpec, ColumnIndex, DataColumnIdentifier, DataColumnSidecar, - DataColumnSidecarList, Epoch, EthSpec, Hash256, SignedBeaconBlock, + DataColumnSidecarList, Epoch, EthSpec, Hash256, RuntimeVariableList, SignedBeaconBlock, }; pub type DataColumnsToPublish = Option>; @@ -32,7 +33,7 @@ pub type DataColumnsToPublish = Option>; #[derive(Clone)] pub struct PendingComponents { pub block_root: Hash256, - pub verified_blobs: FixedVector>, E::MaxBlobsPerBlock>, + pub verified_blobs: RuntimeFixedList>>, pub verified_data_columns: Vec>, pub executed_block: Option>, pub reconstruction_started: bool, @@ -50,9 +51,7 @@ impl PendingComponents { } /// Returns an immutable reference to the fixed vector of cached blobs. - pub fn get_cached_blobs( - &self, - ) -> &FixedVector>, E::MaxBlobsPerBlock> { + pub fn get_cached_blobs(&self) -> &RuntimeFixedList>> { &self.verified_blobs } @@ -73,9 +72,7 @@ impl PendingComponents { } /// Returns a mutable reference to the fixed vector of cached blobs. - pub fn get_cached_blobs_mut( - &mut self, - ) -> &mut FixedVector>, E::MaxBlobsPerBlock> { + pub fn get_cached_blobs_mut(&mut self) -> &mut RuntimeFixedList>> { &mut self.verified_blobs } @@ -147,10 +144,7 @@ impl PendingComponents { /// Blobs are only inserted if: /// 1. The blob entry at the index is empty and no block exists. /// 2. The block exists and its commitment matches the blob's commitment. - pub fn merge_blobs( - &mut self, - blobs: FixedVector>, E::MaxBlobsPerBlock>, - ) { + pub fn merge_blobs(&mut self, blobs: RuntimeFixedList>>) { for (index, blob) in blobs.iter().cloned().enumerate() { let Some(blob) = blob else { continue }; self.merge_single_blob(index, blob); @@ -194,7 +188,7 @@ impl PendingComponents { /// Blobs that don't match the new block's commitments are evicted. pub fn merge_block(&mut self, block: DietAvailabilityPendingExecutedBlock) { self.insert_block(block); - let reinsert = std::mem::take(self.get_cached_blobs_mut()); + let reinsert = self.get_cached_blobs_mut().take(); self.merge_blobs(reinsert); } @@ -223,10 +217,11 @@ impl PendingComponents { } /// Returns an empty `PendingComponents` object with the given block root. - pub fn empty(block_root: Hash256) -> Self { + pub fn empty(block_root: Hash256, max_len: usize) -> Self { Self { block_root, - verified_blobs: FixedVector::default(), + // TODO(pawan): just make this a vec potentially + verified_blobs: RuntimeFixedList::new(vec![None; max_len]), verified_data_columns: vec![], executed_block: None, reconstruction_started: false, @@ -280,7 +275,12 @@ impl PendingComponents { else { return Err(AvailabilityCheckError::Unexpected); }; - (Some(VariableList::new(verified_blobs)?), None) + let max_len = + spec.max_blobs_per_block(diet_executed_block.as_block().epoch()) as usize; + ( + Some(RuntimeVariableList::new(verified_blobs, max_len)?), + None, + ) } BlockImportRequirement::CustodyColumns(_) => { let verified_data_columns = verified_data_columns @@ -477,7 +477,8 @@ impl DataAvailabilityCheckerInner { epoch: Epoch, kzg_verified_blobs: I, ) -> Result, AvailabilityCheckError> { - let mut fixed_blobs = FixedVector::default(); + let mut fixed_blobs = + RuntimeFixedList::new(vec![None; self.spec.max_blobs_per_block(epoch) as usize]); for blob in kzg_verified_blobs { if let Some(blob_opt) = fixed_blobs.get_mut(blob.blob_index() as usize) { @@ -491,7 +492,9 @@ impl DataAvailabilityCheckerInner { let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the blobs. pending_components.merge_blobs(fixed_blobs); @@ -527,7 +530,9 @@ impl DataAvailabilityCheckerInner { let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the data columns. pending_components.merge_data_columns(kzg_verified_data_columns)?; @@ -614,7 +619,9 @@ impl DataAvailabilityCheckerInner { let mut pending_components = write_lock .pop_entry(&block_root) .map(|(_, v)| v) - .unwrap_or_else(|| PendingComponents::empty(block_root)); + .unwrap_or_else(|| { + PendingComponents::empty(block_root, self.spec.max_blobs_per_block(epoch) as usize) + }); // Merge in the block. pending_components.merge_block(diet_executed_block); diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index 55c1ee9e980..db0847073ad 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -188,9 +188,10 @@ fn build_data_column_sidecars( spec: &ChainSpec, ) -> Result, String> { let number_of_columns = spec.number_of_columns; - let mut columns = vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; - let mut column_kzg_proofs = - vec![Vec::with_capacity(E::max_blobs_per_block()); number_of_columns]; + let max_blobs_per_block = + spec.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) as usize; + let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; + let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; for (blob_cells, blob_cell_proofs) in blob_cells_and_proofs_vec { // we iterate over each column, and we construct the column from "top to bottom", diff --git a/beacon_node/beacon_chain/src/observed_data_sidecars.rs b/beacon_node/beacon_chain/src/observed_data_sidecars.rs index 601241dd8ad..7cc5f2e92a1 100644 --- a/beacon_node/beacon_chain/src/observed_data_sidecars.rs +++ b/beacon_node/beacon_chain/src/observed_data_sidecars.rs @@ -23,7 +23,7 @@ pub trait ObservableDataSidecar { fn slot(&self) -> Slot; fn block_proposer_index(&self) -> u64; fn index(&self) -> u64; - fn max_num_of_items(spec: &ChainSpec) -> usize; + fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize; } impl ObservableDataSidecar for BlobSidecar { @@ -39,8 +39,8 @@ impl ObservableDataSidecar for BlobSidecar { self.index } - fn max_num_of_items(_spec: &ChainSpec) -> usize { - E::max_blobs_per_block() + fn max_num_of_items(spec: &ChainSpec, slot: Slot) -> usize { + spec.max_blobs_per_block(slot.epoch(E::slots_per_epoch())) as usize } } @@ -57,7 +57,7 @@ impl ObservableDataSidecar for DataColumnSidecar { self.index } - fn max_num_of_items(spec: &ChainSpec) -> usize { + fn max_num_of_items(spec: &ChainSpec, _slot: Slot) -> usize { spec.number_of_columns } } @@ -102,7 +102,9 @@ impl ObservedDataSidecars { slot: data_sidecar.slot(), proposer: data_sidecar.block_proposer_index(), }) - .or_insert_with(|| HashSet::with_capacity(T::max_num_of_items(&self.spec))); + .or_insert_with(|| { + HashSet::with_capacity(T::max_num_of_items(&self.spec, data_sidecar.slot())) + }); let did_not_exist = data_indices.insert(data_sidecar.index()); Ok(!did_not_exist) @@ -122,7 +124,7 @@ impl ObservedDataSidecars { } fn sanitize_data_sidecar(&self, data_sidecar: &T) -> Result<(), Error> { - if data_sidecar.index() >= T::max_num_of_items(&self.spec) as u64 { + if data_sidecar.index() >= T::max_num_of_items(&self.spec, data_sidecar.slot()) as u64 { return Err(Error::InvalidDataIndex(data_sidecar.index())); } let finalized_slot = self.finalized_slot; diff --git a/beacon_node/beacon_chain/src/test_utils.rs b/beacon_node/beacon_chain/src/test_utils.rs index b28d221da7e..cd2e355cafe 100644 --- a/beacon_node/beacon_chain/src/test_utils.rs +++ b/beacon_node/beacon_chain/src/test_utils.rs @@ -1991,7 +1991,7 @@ where let (block, blob_items) = block_contents; let sidecars = blob_items - .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs)) + .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec)) .transpose() .unwrap(); let block_hash: SignedBeaconBlockHash = self @@ -2017,7 +2017,7 @@ where let (block, blob_items) = block_contents; let sidecars = blob_items - .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs)) + .map(|(proofs, blobs)| BlobSidecar::build_sidecars(blobs, &block, proofs, &self.spec)) .transpose() .unwrap(); let block_root = block.canonical_root(); @@ -2636,7 +2636,8 @@ pub fn generate_rand_block_and_blobs( // Get either zero blobs or a random number of blobs between 1 and Max Blobs. let payload: &mut FullPayloadDeneb = &mut message.body.execution_payload; let num_blobs = match num_blobs { - NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()), + // TODO(pawan): thread the chainspec value here + NumBlobs::Random => rng.gen_range(1..=6), NumBlobs::Number(n) => n, NumBlobs::None => 0, }; @@ -2656,7 +2657,8 @@ pub fn generate_rand_block_and_blobs( // Get either zero blobs or a random number of blobs between 1 and Max Blobs. let payload: &mut FullPayloadElectra = &mut message.body.execution_payload; let num_blobs = match num_blobs { - NumBlobs::Random => rng.gen_range(1..=E::max_blobs_per_block()), + // TODO(pawan): thread the chainspec value here + NumBlobs::Random => rng.gen_range(1..=6), NumBlobs::Number(n) => n, NumBlobs::None => 0, }; diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index d299eebec8e..aa50ede84a9 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -352,10 +352,11 @@ where let anchor_block = SignedBeaconBlock::from_ssz_bytes(&anchor_block_bytes, &spec) .map_err(|e| format!("Unable to parse weak subj block SSZ: {:?}", e))?; let anchor_blobs = if anchor_block.message().body().has_blobs() { + let max_blobs_len = spec.max_blobs_per_block(anchor_block.epoch()) as usize; let anchor_blobs_bytes = anchor_blobs_bytes .ok_or("Blobs for checkpoint must be provided using --checkpoint-blobs")?; Some( - BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes) + BlobSidecarList::from_ssz_bytes(&anchor_blobs_bytes, max_blobs_len) .map_err(|e| format!("Unable to parse weak subj blobs SSZ: {e:?}"))?, ) } else { diff --git a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs index 216c3b7844f..a18ac711aa7 100644 --- a/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs +++ b/beacon_node/execution_layer/src/test_utils/execution_block_generator.rs @@ -673,7 +673,8 @@ impl ExecutionBlockGenerator { ForkName::Deneb | ForkName::Electra => { // get random number between 0 and Max Blobs let mut rng = self.rng.lock(); - let num_blobs = rng.gen::() % (E::max_blobs_per_block() + 1); + // TODO(pawan): thread the chainspec value here somehow + let num_blobs = rng.gen::() % 6; let (bundle, transactions) = generate_blobs(num_blobs)?; for tx in Vec::from(transactions) { execution_payload diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index f35df2f5e86..e3ed15ebc22 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -279,13 +279,14 @@ impl BlockId { .get_blobs(&root) .map_err(warp_utils::reject::beacon_chain_error)?; + let max_len = blob_sidecar_list.max_len(); let blob_sidecar_list_filtered = match indices.indices { Some(vec) => { let list = blob_sidecar_list .into_iter() .filter(|blob_sidecar| vec.contains(&blob_sidecar.index)) .collect(); - BlobSidecarList::new(list) + BlobSidecarList::new(list, max_len) .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? } None => blob_sidecar_list, diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index bbdfc31d430..9f1d17677e9 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -22,7 +22,8 @@ use tree_hash::TreeHash; use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecarList, BlockImportSource, DataColumnSidecarList, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, - FullPayloadBellatrix, Hash256, SignedBeaconBlock, SignedBlindedBeaconBlock, VariableList, + FullPayloadBellatrix, Hash256, RuntimeVariableList, SignedBeaconBlock, + SignedBlindedBeaconBlock, VariableList, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; @@ -198,7 +199,10 @@ pub async fn publish_block>(); - VariableList::from(blobs) + RuntimeVariableList::from_vec( + blobs, + chain.spec.max_blobs_per_block(block.epoch()) as usize, + ) }); let data_cols_opt = gossip_verified_data_columns .as_ref() diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index a96b9d1b166..0ff469aafbb 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -24,6 +24,13 @@ use types::{ pub type MaxErrorLen = U256; pub const MAX_ERROR_LEN: u64 = 256; +/// The max number of blobs we expect in the configs to set for compile time params. +/// Note: This value is an estimate that we should use only for rate limiting, +/// bounds checking and other non-consensus critical operations. +/// +/// For exact value, we should always check the chainspec. +pub const MAX_BLOBS_PER_BLOCK_CEILING: u64 = 16; + /// Wrapper over SSZ List to represent error message in rpc responses. #[derive(Debug, Clone)] pub struct ErrorType(pub VariableList); @@ -326,8 +333,13 @@ pub struct BlobsByRangeRequest { } impl BlobsByRangeRequest { + /// This function provides an upper bound on number of blobs expected in + /// a certain slot range. + /// + /// Note: **must not** use for anything consensus critical, only for + /// bounds checking and rate limiting. pub fn max_blobs_requested(&self) -> u64 { - self.count.saturating_mul(E::max_blobs_per_block() as u64) + self.count.saturating_mul(MAX_BLOBS_PER_BLOCK_CEILING as u64) } } diff --git a/beacon_node/lighthouse_network/src/rpc/protocol.rs b/beacon_node/lighthouse_network/src/rpc/protocol.rs index f4bdf6450b8..1b560f1d1e6 100644 --- a/beacon_node/lighthouse_network/src/rpc/protocol.rs +++ b/beacon_node/lighthouse_network/src/rpc/protocol.rs @@ -92,7 +92,7 @@ pub static SIGNED_BEACON_BLOCK_DENEB_MAX: LazyLock = LazyLock::new(|| { *SIGNED_BEACON_BLOCK_CAPELLA_MAX_WITHOUT_PAYLOAD + types::ExecutionPayload::::max_execution_payload_deneb_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional offsets for the `ExecutionPayload` - + (::ssz_fixed_len() * ::max_blobs_per_block()) + + (::ssz_fixed_len() * MAX_BLOBS_PER_BLOCK_CEILING as usize) + ssz::BYTES_PER_LENGTH_OFFSET }); // Length offset for the blob commitments field. // @@ -100,7 +100,7 @@ pub static SIGNED_BEACON_BLOCK_ELECTRA_MAX: LazyLock = LazyLock::new(|| { *SIGNED_BEACON_BLOCK_ELECTRA_MAX_WITHOUT_PAYLOAD + types::ExecutionPayload::::max_execution_payload_electra_size() // adding max size of execution payload (~16gb) + ssz::BYTES_PER_LENGTH_OFFSET // Adding the additional ssz offset for the `ExecutionPayload` field - + (::ssz_fixed_len() * ::max_blobs_per_block()) + + (::ssz_fixed_len() * MAX_BLOBS_PER_BLOCK_CEILING as usize) + ssz::BYTES_PER_LENGTH_OFFSET }); // Length offset for the blob commitments field. @@ -636,7 +636,7 @@ pub fn rpc_blob_limits() -> RpcLimits { pub fn rpc_data_column_limits() -> RpcLimits { RpcLimits::new( DataColumnSidecar::::empty().as_ssz_bytes().len(), - DataColumnSidecar::::max_size(), + DataColumnSidecar::::max_size(MAX_BLOBS_PER_BLOCK_CEILING as usize), ) } diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index 966ce55fabe..d642a2b4dce 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -8,7 +8,8 @@ use std::{ sync::Arc, }; use types::{ - BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, SignedBeaconBlock, + BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, EthSpec, Hash256, RuntimeVariableList, + SignedBeaconBlock, }; #[derive(Debug)] @@ -31,6 +32,7 @@ pub struct RangeBlockComponentsRequest { num_custody_column_requests: Option, /// The peers the request was made to. pub(crate) peer_ids: Vec, + max_blobs_per_block: usize, } impl RangeBlockComponentsRequest { @@ -39,6 +41,7 @@ impl RangeBlockComponentsRequest { expects_custody_columns: Option>, num_custody_column_requests: Option, peer_ids: Vec, + max_blobs_per_block: usize, ) -> Self { Self { blocks: <_>::default(), @@ -51,6 +54,7 @@ impl RangeBlockComponentsRequest { expects_custody_columns, num_custody_column_requests, peer_ids, + max_blobs_per_block, } } @@ -100,7 +104,7 @@ impl RangeBlockComponentsRequest { let mut responses = Vec::with_capacity(blocks.len()); let mut blob_iter = blobs.into_iter().peekable(); for block in blocks.into_iter() { - let mut blob_list = Vec::with_capacity(E::max_blobs_per_block()); + let mut blob_list = Vec::with_capacity(self.max_blobs_per_block); while { let pair_next_blob = blob_iter .peek() @@ -111,7 +115,7 @@ impl RangeBlockComponentsRequest { blob_list.push(blob_iter.next().ok_or("Missing next blob".to_string())?); } - let mut blobs_buffer = vec![None; E::max_blobs_per_block()]; + let mut blobs_buffer = vec![None; self.max_blobs_per_block]; for blob in blob_list { let blob_index = blob.index as usize; let Some(blob_opt) = blobs_buffer.get_mut(blob_index) else { @@ -123,7 +127,11 @@ impl RangeBlockComponentsRequest { *blob_opt = Some(blob); } } - let blobs = VariableList::from(blobs_buffer.into_iter().flatten().collect::>()); + let blobs = RuntimeVariableList::new( + blobs_buffer.into_iter().flatten().collect::>(), + self.max_blobs_per_block, + ) + .map_err(|_| "Blobs returned exceeds max length".to_string())?; responses.push(RpcBlock::new(None, block, Some(blobs)).map_err(|e| format!("{e:?}"))?) } diff --git a/beacon_node/network/src/sync/manager.rs b/beacon_node/network/src/sync/manager.rs index d6ce14adb16..866268b6fd3 100644 --- a/beacon_node/network/src/sync/manager.rs +++ b/beacon_node/network/src/sync/manager.rs @@ -1120,6 +1120,7 @@ impl SyncManager { .network .range_block_and_blob_response(id, block_or_blob) { + let epoch = resp.sender_id.batch_id(); match resp.responses { Ok(blocks) => { match resp.sender_id { @@ -1163,6 +1164,7 @@ impl SyncManager { resp.expects_custody_columns, None, vec![], + self.chain.spec.max_blobs_per_block(epoch) as usize, ), ); // inform range that the request needs to be treated as failed diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 3da6f92cfed..8d954628eb1 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -36,8 +36,8 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, - SignedBeaconBlock, Slot, + chain_spec, BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, + EthSpec, Hash256, SignedBeaconBlock, Slot, }; pub mod custody; @@ -61,6 +61,15 @@ pub enum RangeRequestId { }, } +impl RangeRequestId { + pub fn batch_id(&self) -> BatchId { + match self { + RangeRequestId::RangeSync { batch_id, .. } => *batch_id, + RangeRequestId::BackfillSync { batch_id, .. } => *batch_id, + } + } +} + #[derive(Debug)] pub enum RpcEvent { StreamTermination, @@ -422,11 +431,14 @@ impl SyncNetworkContext { (None, None) }; + // TODO(pawan): this would break if a batch contains multiple epochs + let max_blobs_len = self.chain.spec.max_blobs_per_block(epoch); let info = RangeBlockComponentsRequest::new( expected_blobs, expects_custody_columns, num_of_custody_column_req, requested_peers, + max_blobs_len as usize, ); self.range_block_components_requests .insert(id, (sender_id, info)); @@ -977,9 +989,16 @@ impl SyncNetworkContext { RpcEvent::Response(blob, seen_timestamp) => { let request = request.get_mut(); match request.add_response(blob) { - Ok(Some(blobs)) => to_fixed_blob_sidecar_list(blobs) - .map(|blobs| (blobs, seen_timestamp)) - .map_err(|e| (e.into(), request.resolve())), + Ok(Some(blobs)) => { + let max_len = if let Some(blob) = blobs.first() { + self.chain.spec.max_blobs_per_block(blob.epoch()) as usize + } else { + 6 + }; + to_fixed_blob_sidecar_list(blobs, max_len) + .map(|blobs| (blobs, seen_timestamp)) + .map_err(|e| (e.into(), request.resolve())) + } Ok(None) => return None, Err(e) => Err((e.into(), request.resolve())), } @@ -1218,8 +1237,11 @@ impl SyncNetworkContext { fn to_fixed_blob_sidecar_list( blobs: Vec>>, + max_len: usize, ) -> Result, LookupVerifyError> { - let mut fixed_list = FixedBlobSidecarList::default(); + // TODO(pawan): have a method on fixed runtime vector that just initializes a raw vec with max_len = None + // to signify an empty fixed runtime vector + let mut fixed_list = FixedBlobSidecarList::new(vec![None; max_len]); for blob in blobs.into_iter() { let index = blob.index as usize; *fixed_list diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index fecd8e37442..0addb619377 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -1669,7 +1669,23 @@ impl, Cold: ItemStore> HotColdDB .get_bytes(DBColumn::BeaconBlob.into(), block_root.as_bytes())? { Some(ref blobs_bytes) => { - let blobs = BlobSidecarList::from_ssz_bytes(blobs_bytes)?; + // We insert a VariableList of BlobSidecars into the db, but retrieve + // a plain vec since we don't know the length limit of the list without + // knowing the slot. + // The encoding of a VariableList is same as a regular vec. + let blobs = BlobSidecarVec::from_ssz_bytes(blobs_bytes)?; + let max_blobs_per_block = blobs + .first() + .map(|blob| { + self.spec + .max_blobs_per_block(blob.slot().epoch(E::slots_per_epoch())) + }) + // This is the case where we have no blobs for the slot, doesn't matter what value we keep for max here + // TODO(pawan): double check that this is the case + // we could also potentially deal with just vecs in the db since we only add length validated sidecar + // lists to the db + .unwrap_or(6); + let blobs = BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize); self.block_cache .lock() .put_blobs(*block_root, blobs.clone()); diff --git a/beacon_node/store/src/impls/execution_payload.rs b/beacon_node/store/src/impls/execution_payload.rs index 14fc10ad6de..3f7c4172bc9 100644 --- a/beacon_node/store/src/impls/execution_payload.rs +++ b/beacon_node/store/src/impls/execution_payload.rs @@ -1,7 +1,7 @@ use crate::{DBColumn, Error, StoreItem}; use ssz::{Decode, Encode}; use types::{ - BlobSidecarList, EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, + EthSpec, ExecutionPayload, ExecutionPayloadBellatrix, ExecutionPayloadCapella, ExecutionPayloadDeneb, ExecutionPayloadElectra, }; @@ -26,7 +26,6 @@ impl_store_item!(ExecutionPayloadBellatrix); impl_store_item!(ExecutionPayloadCapella); impl_store_item!(ExecutionPayloadDeneb); impl_store_item!(ExecutionPayloadElectra); -impl_store_item!(BlobSidecarList); /// This fork-agnostic implementation should be only used for writing. /// diff --git a/consensus/state_processing/src/per_block_processing.rs b/consensus/state_processing/src/per_block_processing.rs index e7655b453a8..7b2ca900556 100644 --- a/consensus/state_processing/src/per_block_processing.rs +++ b/consensus/state_processing/src/per_block_processing.rs @@ -391,10 +391,12 @@ pub fn partially_verify_execution_payload = VariableList::MaxBlobCommitmentsPerBlock>; -pub type KzgCommitmentOpts = - FixedVector, ::MaxBlobsPerBlock>; /// The number of leaves (including padding) on the `BeaconBlockBody` Merkle tree. /// diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 0f7dbb2673c..c2981e14cc2 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -1,10 +1,13 @@ use crate::test_utils::TestRandom; -use crate::ForkName; use crate::{ beacon_block_body::BLOB_KZG_COMMITMENTS_INDEX, BeaconBlockHeader, BeaconStateError, Blob, Epoch, EthSpec, FixedVector, Hash256, SignedBeaconBlockHeader, Slot, VariableList, }; -use crate::{ForkVersionDeserialize, KzgProofs, SignedBeaconBlock}; +use crate::{ + runtime_var_list::RuntimeFixedList, ForkVersionDeserialize, KzgProofs, RuntimeVariableList, + SignedBeaconBlock, +}; +use crate::{ChainSpec, ForkName}; use bls::Signature; use derivative::Derivative; use kzg::{Blob as KzgBlob, Kzg, KzgCommitment, KzgProof, BYTES_PER_BLOB, BYTES_PER_FIELD_ELEMENT}; @@ -30,19 +33,6 @@ pub struct BlobIdentifier { pub index: u64, } -impl BlobIdentifier { - pub fn get_all_blob_ids(block_root: Hash256) -> Vec { - let mut blob_ids = Vec::with_capacity(E::max_blobs_per_block()); - for i in 0..E::max_blobs_per_block() { - blob_ids.push(BlobIdentifier { - block_root, - index: i as u64, - }); - } - blob_ids - } -} - impl PartialOrd for BlobIdentifier { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) @@ -260,19 +250,24 @@ impl BlobSidecar { blobs: BlobsList, block: &SignedBeaconBlock, kzg_proofs: KzgProofs, + spec: &ChainSpec, ) -> Result, BlobSidecarError> { let mut blob_sidecars = vec![]; for (i, (kzg_proof, blob)) in kzg_proofs.iter().zip(blobs).enumerate() { let blob_sidecar = BlobSidecar::new(i, blob, block, *kzg_proof)?; blob_sidecars.push(Arc::new(blob_sidecar)); } - Ok(VariableList::from(blob_sidecars)) + Ok(RuntimeVariableList::from_vec( + blob_sidecars, + spec.max_blobs_per_block(block.epoch()) as usize, + )) } } -pub type BlobSidecarList = VariableList>, ::MaxBlobsPerBlock>; -pub type FixedBlobSidecarList = - FixedVector>>, ::MaxBlobsPerBlock>; +pub type BlobSidecarList = RuntimeVariableList>>; +/// Alias for a non length-constrained list of `BlobSidecar`s. +pub type BlobSidecarVec = Vec>>; +pub type FixedBlobSidecarList = RuntimeFixedList>>>; pub type BlobsList = VariableList, ::MaxBlobCommitmentsPerBlock>; impl ForkVersionDeserialize for BlobSidecarList { diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 10b00d5ba1d..5af0f349de3 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -230,6 +230,7 @@ pub struct ChainSpec { pub max_request_data_column_sidecars: u64, pub min_epochs_for_blob_sidecars_requests: u64, pub blob_sidecar_subnet_count: u64, + max_blobs_per_block: u64, /* * Networking Derived @@ -607,6 +608,16 @@ impl ChainSpec { } } + /// Returns the deneb preset value if peerdas epoch hasn't hit. + /// Otherwise, returns the value obtained from the config.yaml. + pub fn max_blobs_per_block(&self, epoch: Epoch) -> u64 { + if self.is_peer_das_enabled_for_epoch(epoch) { + self.max_blobs_per_block + } else { + default_max_blobs_per_block() + } + } + pub fn data_columns_per_subnet(&self) -> usize { self.number_of_columns .safe_div(self.data_column_sidecar_subnet_count as usize) @@ -843,6 +854,7 @@ impl ChainSpec { max_request_data_column_sidecars: default_max_request_data_column_sidecars(), min_epochs_for_blob_sidecars_requests: default_min_epochs_for_blob_sidecars_requests(), blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), + max_blobs_per_block: default_max_blobs_per_block(), /* * Derived Deneb Specific @@ -1164,6 +1176,8 @@ impl ChainSpec { max_request_data_column_sidecars: default_max_request_data_column_sidecars(), min_epochs_for_blob_sidecars_requests: 16384, blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), + // TODO(pawan): check if gnosis preset values match + max_blobs_per_block: default_max_blobs_per_block(), /* * Derived Deneb Specific @@ -1364,6 +1378,9 @@ pub struct Config { #[serde(default = "default_blob_sidecar_subnet_count")] #[serde(with = "serde_utils::quoted_u64")] blob_sidecar_subnet_count: u64, + #[serde(default = "default_max_blobs_per_block")] + #[serde(with = "serde_utils::quoted_u64")] + max_blobs_per_block: u64, #[serde(default = "default_min_per_epoch_churn_limit_electra")] #[serde(with = "serde_utils::quoted_u64")] @@ -1499,6 +1516,12 @@ const fn default_blob_sidecar_subnet_count() -> u64 { 6 } +/// Its important to keep this consistent with the deneb preset value for +/// `MAX_BLOBS_PER_BLOCK` else we might run into consensus issues. +const fn default_max_blobs_per_block() -> u64 { + 6 +} + const fn default_min_per_epoch_churn_limit_electra() -> u64 { 128_000_000_000 } @@ -1718,6 +1741,7 @@ impl Config { max_request_data_column_sidecars: spec.max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests: spec.min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count: spec.blob_sidecar_subnet_count, + max_blobs_per_block: spec.max_blobs_per_block, min_per_epoch_churn_limit_electra: spec.min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit: spec @@ -1795,6 +1819,7 @@ impl Config { max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + max_blobs_per_block, min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit, @@ -1863,6 +1888,7 @@ impl Config { max_request_data_column_sidecars, min_epochs_for_blob_sidecars_requests, blob_sidecar_subnet_count, + max_blobs_per_block, min_per_epoch_churn_limit_electra, max_per_epoch_activation_exit_churn_limit, diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index 90c05aea1f7..ea0fb2b14a7 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,7 +1,7 @@ use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX}; use crate::test_utils::TestRandom; -use crate::BeaconStateError; -use crate::{BeaconBlockHeader, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; +use crate::{BeaconBlockHeader, Epoch, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; +use crate::{BeaconStateError, ChainSpec}; use bls::Signature; use derivative::Derivative; use kzg::Error as KzgError; @@ -110,18 +110,16 @@ impl DataColumnSidecar { .len() } - pub fn max_size() -> usize { + pub fn max_size(max_blobs_per_block: usize) -> usize { Self { index: 0, - column: VariableList::new(vec![Cell::::default(); E::MaxBlobsPerBlock::to_usize()]) - .unwrap(), + column: VariableList::new(vec![Cell::::default(); max_blobs_per_block]).unwrap(), kzg_commitments: VariableList::new(vec![ KzgCommitment::empty_for_testing(); - E::MaxBlobsPerBlock::to_usize() + max_blobs_per_block ]) .unwrap(), - kzg_proofs: VariableList::new(vec![KzgProof::empty(); E::MaxBlobsPerBlock::to_usize()]) - .unwrap(), + kzg_proofs: VariableList::new(vec![KzgProof::empty(); max_blobs_per_block]).unwrap(), signed_block_header: SignedBeaconBlockHeader { message: BeaconBlockHeader::empty(), signature: Signature::empty(), diff --git a/consensus/types/src/eth_spec.rs b/consensus/types/src/eth_spec.rs index 09ef8e3c1a7..3d496d214eb 100644 --- a/consensus/types/src/eth_spec.rs +++ b/consensus/types/src/eth_spec.rs @@ -4,8 +4,7 @@ use safe_arith::SafeArith; use serde::{Deserialize, Serialize}; use ssz_types::typenum::{ bit::B0, UInt, U0, U1, U1024, U1048576, U1073741824, U1099511627776, U128, U131072, U134217728, - U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U6, U625, U64, U65536, U8, - U8192, + U16, U16777216, U2, U2048, U256, U262144, U32, U4, U4096, U512, U625, U64, U65536, U8, U8192, }; use ssz_types::typenum::{U17, U9}; use std::fmt::{self, Debug}; @@ -109,7 +108,6 @@ pub trait EthSpec: /* * New in Deneb */ - type MaxBlobsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type MaxBlobCommitmentsPerBlock: Unsigned + Clone + Sync + Send + Debug + PartialEq + Unpin; type FieldElementsPerBlob: Unsigned + Clone + Sync + Send + Debug + PartialEq; type BytesPerFieldElement: Unsigned + Clone + Sync + Send + Debug + PartialEq; @@ -280,11 +278,6 @@ pub trait EthSpec: Self::MaxWithdrawalsPerPayload::to_usize() } - /// Returns the `MAX_BLOBS_PER_BLOCK` constant for this specification. - fn max_blobs_per_block() -> usize { - Self::MaxBlobsPerBlock::to_usize() - } - /// Returns the `MAX_BLOB_COMMITMENTS_PER_BLOCK` constant for this specification. fn max_blob_commitments_per_block() -> usize { Self::MaxBlobCommitmentsPerBlock::to_usize() @@ -415,7 +408,6 @@ impl EthSpec for MainnetEthSpec { type GasLimitDenominator = U1024; type MinGasLimit = U5000; type MaxExtraDataBytes = U32; - type MaxBlobsPerBlock = U6; type MaxBlobCommitmentsPerBlock = U4096; type BytesPerFieldElement = U32; type FieldElementsPerBlob = U4096; @@ -498,7 +490,6 @@ impl EthSpec for MinimalEthSpec { MinGasLimit, MaxExtraDataBytes, MaxBlsToExecutionChanges, - MaxBlobsPerBlock, BytesPerFieldElement, PendingBalanceDepositsLimit, MaxConsolidationRequestsPerPayload, @@ -551,7 +542,6 @@ impl EthSpec for GnosisEthSpec { type SlotsPerEth1VotingPeriod = U1024; // 64 epochs * 16 slots per epoch type MaxBlsToExecutionChanges = U16; type MaxWithdrawalsPerPayload = U8; - type MaxBlobsPerBlock = U6; type MaxBlobCommitmentsPerBlock = U4096; type FieldElementsPerBlob = U4096; type BytesPerFieldElement = U32; diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 68d48ec7c8b..4db50c30e13 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -138,7 +138,9 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{Error as BeaconStateError, *}; -pub use crate::blob_sidecar::{BlobIdentifier, BlobSidecar, BlobSidecarList, BlobsList}; +pub use crate::blob_sidecar::{ + BlobIdentifier, BlobSidecar, BlobSidecarList, BlobSidecarVec, BlobsList, +}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; diff --git a/consensus/types/src/preset.rs b/consensus/types/src/preset.rs index 2c576ed332c..0d067d3fe95 100644 --- a/consensus/types/src/preset.rs +++ b/consensus/types/src/preset.rs @@ -208,8 +208,6 @@ impl CapellaPreset { #[derive(Debug, PartialEq, Clone, Serialize, Deserialize)] #[serde(rename_all = "UPPERCASE")] pub struct DenebPreset { - #[serde(with = "serde_utils::quoted_u64")] - pub max_blobs_per_block: u64, #[serde(with = "serde_utils::quoted_u64")] pub max_blob_commitments_per_block: u64, #[serde(with = "serde_utils::quoted_u64")] @@ -219,7 +217,6 @@ pub struct DenebPreset { impl DenebPreset { pub fn from_chain_spec(_spec: &ChainSpec) -> Self { Self { - max_blobs_per_block: E::max_blobs_per_block() as u64, max_blob_commitments_per_block: E::max_blob_commitments_per_block() as u64, field_elements_per_blob: E::field_elements_per_blob() as u64, } diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index af4ee87c158..4c6d6e2d927 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -214,6 +214,67 @@ where } } +#[derive(Clone, Debug)] +pub struct RuntimeFixedList { + vec: Vec, + len: usize, +} + +impl RuntimeFixedList { + // TODO(pawan): no need to take len + pub fn new(vec: Vec) -> Self { + let len = vec.len(); + Self { vec, len } + } + + pub fn to_vec(&self) -> Vec { + self.vec.clone() + } + + pub fn as_slice(&self) -> &[T] { + self.vec.as_slice() + } + + pub fn len(&self) -> usize { + self.len + } + + pub fn into_vec(self) -> Vec { + self.vec + } + + pub fn take(&mut self) -> Self { + let new = std::mem::take(&mut self.vec); + Self { + vec: new, + len: self.len, + } + } +} + +impl std::ops::Deref for RuntimeFixedList { + type Target = [T]; + + fn deref(&self) -> &[T] { + &self.vec[..] + } +} + +impl std::ops::DerefMut for RuntimeFixedList { + fn deref_mut(&mut self) -> &mut [T] { + &mut self.vec[..] + } +} + +impl<'a, T> IntoIterator for &'a RuntimeFixedList { + type Item = &'a T; + type IntoIter = std::slice::Iter<'a, T>; + + fn into_iter(self) -> Self::IntoIter { + self.vec.iter() + } +} + #[cfg(test)] mod test { use super::*; From 4dc6e6515ecf75cefa4de840edc7b57e76a8fc9e Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 30 Aug 2024 15:53:18 -0700 Subject: [PATCH 2/9] Add restrictions to RuntimeVariableList api --- consensus/types/src/blob_sidecar.rs | 1 - consensus/types/src/data_column_sidecar.rs | 5 +- consensus/types/src/runtime_var_list.rs | 73 +++++++++++++++++----- 3 files changed, 59 insertions(+), 20 deletions(-) diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index c2981e14cc2..108c807b475 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -266,7 +266,6 @@ impl BlobSidecar { pub type BlobSidecarList = RuntimeVariableList>>; /// Alias for a non length-constrained list of `BlobSidecar`s. -pub type BlobSidecarVec = Vec>>; pub type FixedBlobSidecarList = RuntimeFixedList>>>; pub type BlobsList = VariableList, ::MaxBlobCommitmentsPerBlock>; diff --git a/consensus/types/src/data_column_sidecar.rs b/consensus/types/src/data_column_sidecar.rs index ea0fb2b14a7..34bdc33886a 100644 --- a/consensus/types/src/data_column_sidecar.rs +++ b/consensus/types/src/data_column_sidecar.rs @@ -1,7 +1,7 @@ use crate::beacon_block_body::{KzgCommitments, BLOB_KZG_COMMITMENTS_INDEX}; use crate::test_utils::TestRandom; -use crate::{BeaconBlockHeader, Epoch, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; -use crate::{BeaconStateError, ChainSpec}; +use crate::BeaconStateError; +use crate::{BeaconBlockHeader, EthSpec, Hash256, KzgProofs, SignedBeaconBlockHeader, Slot}; use bls::Signature; use derivative::Derivative; use kzg::Error as KzgError; @@ -11,7 +11,6 @@ use safe_arith::ArithError; use serde::{Deserialize, Serialize}; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use ssz_types::typenum::Unsigned; use ssz_types::Error as SszError; use ssz_types::{FixedVector, VariableList}; use std::hash::Hash; diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index 4c6d6e2d927..aa43e3fe759 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -2,7 +2,7 @@ use derivative::Derivative; use serde::{Deserialize, Serialize}; use ssz::Decode; use ssz_types::Error; -use std::ops::{Deref, DerefMut, Index, IndexMut}; +use std::ops::{Deref, Index, IndexMut}; use std::slice::SliceIndex; /// Emulates a SSZ `List`. @@ -41,8 +41,10 @@ use std::slice::SliceIndex; #[serde(transparent)] pub struct RuntimeVariableList { vec: Vec, + /// A `None` here indicates an uninitialized `Self`. + /// No mutating operation will be allowed until `max_len` is Some #[serde(skip)] - max_len: usize, + max_len: Option, } impl RuntimeVariableList { @@ -50,7 +52,10 @@ impl RuntimeVariableList { /// `Err(OutOfBounds { .. })`. pub fn new(vec: Vec, max_len: usize) -> Result { if vec.len() <= max_len { - Ok(Self { vec, max_len }) + Ok(Self { + vec, + max_len: Some(max_len), + }) } else { Err(Error::OutOfBounds { i: vec.len(), @@ -62,14 +67,17 @@ impl RuntimeVariableList { pub fn from_vec(mut vec: Vec, max_len: usize) -> Self { vec.truncate(max_len); - Self { vec, max_len } + Self { + vec, + max_len: Some(max_len), + } } /// Create an empty list. pub fn empty(max_len: usize) -> Self { Self { vec: vec![], - max_len, + max_len: Some(max_len), } } @@ -77,6 +85,28 @@ impl RuntimeVariableList { self.vec.as_slice() } + pub fn as_mut_slice(&mut self) -> Option<&mut [T]> { + if self.max_len.is_none() { + return None; + }; + Some(self.vec.as_mut_slice()) + } + + /// Returns an instance of `Self` with max_len = None. + /// + /// No mutating operation can be performed on an uninitialized instance + /// without first setting max_len. + pub fn empty_uninitialized() -> Self { + Self { + vec: vec![], + max_len: None, + } + } + + pub fn set_max_len(&mut self, max_len: usize) { + self.max_len = Some(max_len); + } + /// Returns the number of values presently in `self`. pub fn len(&self) -> usize { self.vec.len() @@ -88,7 +118,9 @@ impl RuntimeVariableList { } /// Returns the type-level maximum length. - pub fn max_len(&self) -> usize { + /// + /// Returns `None` if self is uninitialized with a max_len. + pub fn max_len(&self) -> Option { self.max_len } @@ -96,13 +128,17 @@ impl RuntimeVariableList { /// /// Returns `Err(())` when appending `value` would exceed the maximum length. pub fn push(&mut self, value: T) -> Result<(), Error> { - if self.vec.len() < self.max_len { + let Some(max_len) = self.max_len else { + // TODO(pawan): set a better error + return Err(Error::MissingLengthInformation); + }; + if self.vec.len() < max_len { self.vec.push(value); Ok(()) } else { Err(Error::OutOfBounds { i: self.vec.len().saturating_add(1), - len: self.max_len, + len: max_len, }) } } @@ -135,7 +171,10 @@ impl RuntimeVariableList { } else { ssz::decode_list_of_variable_length_items(bytes, Some(max_len))? }; - Ok(Self { vec, max_len }) + Ok(Self { + vec, + max_len: Some(max_len), + }) } } @@ -169,12 +208,6 @@ impl Deref for RuntimeVariableList { } } -impl DerefMut for RuntimeVariableList { - fn deref_mut(&mut self) -> &mut [T] { - &mut self.vec[..] - } -} - impl<'a, T> IntoIterator for &'a RuntimeVariableList { type Item = &'a T; type IntoIter = std::slice::Iter<'a, T>; @@ -221,7 +254,6 @@ pub struct RuntimeFixedList { } impl RuntimeFixedList { - // TODO(pawan): no need to take len pub fn new(vec: Vec) -> Self { let len = vec.len(); Self { vec, len } @@ -280,6 +312,7 @@ mod test { use super::*; use ssz::*; use std::fmt::Debug; + use tree_hash::TreeHash; #[test] fn new() { @@ -358,4 +391,12 @@ mod test { round_trip::(RuntimeVariableList::from_vec(vec![42; 8], 8)); round_trip::(RuntimeVariableList::from_vec(vec![0; 8], 8)); } + + #[test] + fn test_empty_list_encoding() { + use ssz_types::{typenum::U16, VariableList}; + + let a: RuntimeVariableList = RuntimeVariableList::from_vec(vec![], 16); + dbg!(a.tree_hash_root()); + } } From a9cb329a221a809f7dd818984753826f91c2e26b Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 30 Aug 2024 15:54:00 -0700 Subject: [PATCH 3/9] Use empty_uninitialized and fix warnings --- .../beacon_chain/src/blob_verification.rs | 1 - .../beacon_chain/src/block_verification.rs | 1 - .../src/block_verification_types.rs | 3 +-- .../overflow_lru_cache.rs | 1 - beacon_node/client/src/builder.rs | 1 - beacon_node/http_api/src/block_id.rs | 14 +++++++++---- beacon_node/http_api/src/publish_blocks.rs | 2 +- .../src/sync/block_sidecar_coupling.rs | 1 - .../network/src/sync/network_context.rs | 4 ++-- beacon_node/store/src/hot_cold_store.rs | 21 ++++++++----------- consensus/types/src/lib.rs | 4 +--- 11 files changed, 24 insertions(+), 29 deletions(-) diff --git a/beacon_node/beacon_chain/src/blob_verification.rs b/beacon_node/beacon_chain/src/blob_verification.rs index 4a39a9d2531..ceeb32427e6 100644 --- a/beacon_node/beacon_chain/src/blob_verification.rs +++ b/beacon_node/beacon_chain/src/blob_verification.rs @@ -12,7 +12,6 @@ use crate::{metrics, BeaconChainError}; use kzg::{Error as KzgError, Kzg, KzgCommitment}; use slog::debug; use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; use std::time::Duration; use tree_hash::TreeHash; use types::blob_sidecar::BlobIdentifier; diff --git a/beacon_node/beacon_chain/src/block_verification.rs b/beacon_node/beacon_chain/src/block_verification.rs index 6582096befb..ef3e3363364 100644 --- a/beacon_node/beacon_chain/src/block_verification.rs +++ b/beacon_node/beacon_chain/src/block_verification.rs @@ -81,7 +81,6 @@ use slog::{debug, error, warn, Logger}; use slot_clock::SlotClock; use ssz::Encode; use ssz_derive::{Decode, Encode}; -use ssz_types::VariableList; use state_processing::per_block_processing::{errors::IntoWithIndex, is_merge_transition_block}; use state_processing::{ block_signature_verifier::{BlockSignatureVerifier, Error as BlockSignatureVerifierError}, diff --git a/beacon_node/beacon_chain/src/block_verification_types.rs b/beacon_node/beacon_chain/src/block_verification_types.rs index 67643318f2c..00442ff9668 100644 --- a/beacon_node/beacon_chain/src/block_verification_types.rs +++ b/beacon_node/beacon_chain/src/block_verification_types.rs @@ -8,11 +8,10 @@ use crate::data_column_verification::{ use crate::eth1_finalization_cache::Eth1FinalizationData; use crate::{get_block_root, GossipVerifiedBlock, PayloadVerificationOutcome}; use derivative::Derivative; -use ssz_types::VariableList; use state_processing::ConsensusContext; use std::fmt::{Debug, Formatter}; use std::sync::Arc; -use types::blob_sidecar::{self, BlobIdentifier, FixedBlobSidecarList}; +use types::blob_sidecar::{self, BlobIdentifier}; use types::data_column_sidecar::{self}; use types::{ BeaconBlockRef, BeaconState, BlindedPayload, BlobSidecarList, ChainSpec, Epoch, EthSpec, diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index 9c33051ae02..2ff64dac97d 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -11,7 +11,6 @@ use crate::BeaconChainTypes; use kzg::Kzg; use lru::LruCache; use parking_lot::RwLock; -use ssz_types::{FixedVector, VariableList}; use std::collections::HashSet; use std::num::NonZeroUsize; use std::sync::Arc; diff --git a/beacon_node/client/src/builder.rs b/beacon_node/client/src/builder.rs index aa50ede84a9..0d1aba8b4d1 100644 --- a/beacon_node/client/src/builder.rs +++ b/beacon_node/client/src/builder.rs @@ -37,7 +37,6 @@ use network::{NetworkConfig, NetworkSenders, NetworkService}; use slasher::Slasher; use slasher_service::SlasherService; use slog::{debug, info, warn, Logger}; -use ssz::Decode; use std::net::TcpListener; use std::path::{Path, PathBuf}; use std::sync::Arc; diff --git a/beacon_node/http_api/src/block_id.rs b/beacon_node/http_api/src/block_id.rs index e3ed15ebc22..63d00edbff6 100644 --- a/beacon_node/http_api/src/block_id.rs +++ b/beacon_node/http_api/src/block_id.rs @@ -279,15 +279,21 @@ impl BlockId { .get_blobs(&root) .map_err(warp_utils::reject::beacon_chain_error)?; - let max_len = blob_sidecar_list.max_len(); let blob_sidecar_list_filtered = match indices.indices { Some(vec) => { - let list = blob_sidecar_list + let list: Vec<_> = blob_sidecar_list .into_iter() .filter(|blob_sidecar| vec.contains(&blob_sidecar.index)) .collect(); - BlobSidecarList::new(list, max_len) - .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? + if let Some(max_len) = list + .first() + .map(|sidecar| chain.spec.max_blobs_per_block(sidecar.epoch())) + { + BlobSidecarList::new(list, max_len as usize) + .map_err(|e| warp_utils::reject::custom_server_error(format!("{:?}", e)))? + } else { + BlobSidecarList::empty_uninitialized() + } } None => blob_sidecar_list, }; diff --git a/beacon_node/http_api/src/publish_blocks.rs b/beacon_node/http_api/src/publish_blocks.rs index 9f1d17677e9..a53bb7340df 100644 --- a/beacon_node/http_api/src/publish_blocks.rs +++ b/beacon_node/http_api/src/publish_blocks.rs @@ -23,7 +23,7 @@ use types::{ AbstractExecPayload, BeaconBlockRef, BlobSidecarList, BlockImportSource, DataColumnSidecarList, DataColumnSubnetId, EthSpec, ExecPayload, ExecutionBlockHash, ForkName, FullPayload, FullPayloadBellatrix, Hash256, RuntimeVariableList, SignedBeaconBlock, - SignedBlindedBeaconBlock, VariableList, + SignedBlindedBeaconBlock, }; use warp::http::StatusCode; use warp::{reply::Response, Rejection, Reply}; diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index d642a2b4dce..f241a52c9c8 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -2,7 +2,6 @@ use beacon_chain::{ block_verification_types::RpcBlock, data_column_verification::CustodyDataColumn, get_block_root, }; use lighthouse_network::PeerId; -use ssz_types::VariableList; use std::{ collections::{HashMap, VecDeque}, sync::Arc, diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 8d954628eb1..bcfc9a7da22 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -36,8 +36,8 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - chain_spec, BlobSidecar, ChainSpec, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, - EthSpec, Hash256, SignedBeaconBlock, Slot, + BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, + Hash256, SignedBeaconBlock, Slot, }; pub mod custody; diff --git a/beacon_node/store/src/hot_cold_store.rs b/beacon_node/store/src/hot_cold_store.rs index 0addb619377..13f672d2144 100644 --- a/beacon_node/store/src/hot_cold_store.rs +++ b/beacon_node/store/src/hot_cold_store.rs @@ -1673,19 +1673,16 @@ impl, Cold: ItemStore> HotColdDB // a plain vec since we don't know the length limit of the list without // knowing the slot. // The encoding of a VariableList is same as a regular vec. - let blobs = BlobSidecarVec::from_ssz_bytes(blobs_bytes)?; - let max_blobs_per_block = blobs + let blobs: Vec>> = Vec::<_>::from_ssz_bytes(blobs_bytes)?; + let blobs = if let Some(max_blobs_per_block) = blobs .first() - .map(|blob| { - self.spec - .max_blobs_per_block(blob.slot().epoch(E::slots_per_epoch())) - }) - // This is the case where we have no blobs for the slot, doesn't matter what value we keep for max here - // TODO(pawan): double check that this is the case - // we could also potentially deal with just vecs in the db since we only add length validated sidecar - // lists to the db - .unwrap_or(6); - let blobs = BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize); + .map(|blob| self.spec.max_blobs_per_block(blob.epoch())) + { + BlobSidecarList::from_vec(blobs, max_blobs_per_block as usize) + } else { + // This always implies that there were no blobs for this block_root + BlobSidecarList::empty_uninitialized() + }; self.block_cache .lock() .put_blobs(*block_root, blobs.clone()); diff --git a/consensus/types/src/lib.rs b/consensus/types/src/lib.rs index 4db50c30e13..68d48ec7c8b 100644 --- a/consensus/types/src/lib.rs +++ b/consensus/types/src/lib.rs @@ -138,9 +138,7 @@ pub use crate::beacon_block_body::{ pub use crate::beacon_block_header::BeaconBlockHeader; pub use crate::beacon_committee::{BeaconCommittee, OwnedBeaconCommittee}; pub use crate::beacon_state::{Error as BeaconStateError, *}; -pub use crate::blob_sidecar::{ - BlobIdentifier, BlobSidecar, BlobSidecarList, BlobSidecarVec, BlobsList, -}; +pub use crate::blob_sidecar::{BlobIdentifier, BlobSidecar, BlobSidecarList, BlobsList}; pub use crate::bls_to_execution_change::BlsToExecutionChange; pub use crate::chain_spec::{ChainSpec, Config, Domain}; pub use crate::checkpoint::Checkpoint; From 60100fc6be72792ff33913d7e5a53434c792aacf Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 30 Aug 2024 16:04:11 -0700 Subject: [PATCH 4/9] Fix some todos --- beacon_node/beacon_chain/src/beacon_chain.rs | 5 +---- beacon_node/network/src/sync/network_context.rs | 2 -- consensus/types/src/chain_spec.rs | 1 - 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/beacon_node/beacon_chain/src/beacon_chain.rs b/beacon_node/beacon_chain/src/beacon_chain.rs index e1cf40f65f6..2785eac0a44 100644 --- a/beacon_node/beacon_chain/src/beacon_chain.rs +++ b/beacon_node/beacon_chain/src/beacon_chain.rs @@ -1249,10 +1249,7 @@ impl BeaconChain { pub fn get_blobs(&self, block_root: &Hash256) -> Result, Error> { match self.store.get_blobs(block_root)? { Some(blobs) => Ok(blobs), - None => Ok(BlobSidecarList::empty( - // TODO(pawan): fix this - self.spec.max_blobs_per_block(Epoch::new(0)) as usize, - )), + None => Ok(BlobSidecarList::empty_uninitialized()), } } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index bcfc9a7da22..84e968a5951 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -1239,8 +1239,6 @@ fn to_fixed_blob_sidecar_list( blobs: Vec>>, max_len: usize, ) -> Result, LookupVerifyError> { - // TODO(pawan): have a method on fixed runtime vector that just initializes a raw vec with max_len = None - // to signify an empty fixed runtime vector let mut fixed_list = FixedBlobSidecarList::new(vec![None; max_len]); for blob in blobs.into_iter() { let index = blob.index as usize; diff --git a/consensus/types/src/chain_spec.rs b/consensus/types/src/chain_spec.rs index 5af0f349de3..2c28c3d31d7 100644 --- a/consensus/types/src/chain_spec.rs +++ b/consensus/types/src/chain_spec.rs @@ -1176,7 +1176,6 @@ impl ChainSpec { max_request_data_column_sidecars: default_max_request_data_column_sidecars(), min_epochs_for_blob_sidecars_requests: 16384, blob_sidecar_subnet_count: default_blob_sidecar_subnet_count(), - // TODO(pawan): check if gnosis preset values match max_blobs_per_block: default_max_blobs_per_block(), /* From e71020e3e613910e0315f558ead661b490a0ff20 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 3 Sep 2024 17:16:10 -0700 Subject: [PATCH 5/9] Fix take impl on RuntimeFixedList --- consensus/types/src/runtime_var_list.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index 2bca5df2d9e..b24be185da8 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -253,7 +253,7 @@ pub struct RuntimeFixedList { len: usize, } -impl RuntimeFixedList { +impl RuntimeFixedList { pub fn new(vec: Vec) -> Self { let len = vec.len(); Self { vec, len } @@ -277,6 +277,7 @@ impl RuntimeFixedList { pub fn take(&mut self) -> Self { let new = std::mem::take(&mut self.vec); + *self = Self::new(vec![T::default(); self.len]); Self { vec: new, len: self.len, From 52bb581e071d5f474d519366e860a4b3a0b52f78 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Tue, 3 Sep 2024 18:38:19 -0700 Subject: [PATCH 6/9] cleanup --- beacon_node/beacon_chain/src/kzg_utils.rs | 5 ++-- .../lighthouse_network/src/rpc/methods.rs | 8 +++---- .../network/src/sync/network_context.rs | 4 ++-- consensus/types/src/blob_sidecar.rs | 2 +- consensus/types/src/runtime_var_list.rs | 23 ++++++++++++++++--- 5 files changed, 30 insertions(+), 12 deletions(-) diff --git a/beacon_node/beacon_chain/src/kzg_utils.rs b/beacon_node/beacon_chain/src/kzg_utils.rs index db0847073ad..64e960f0f3a 100644 --- a/beacon_node/beacon_chain/src/kzg_utils.rs +++ b/beacon_node/beacon_chain/src/kzg_utils.rs @@ -188,8 +188,9 @@ fn build_data_column_sidecars( spec: &ChainSpec, ) -> Result, String> { let number_of_columns = spec.number_of_columns; - let max_blobs_per_block = - spec.max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) as usize; + let max_blobs_per_block = spec + .max_blobs_per_block(signed_block_header.message.slot.epoch(E::slots_per_epoch())) + as usize; let mut columns = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; let mut column_kzg_proofs = vec![Vec::with_capacity(max_blobs_per_block); number_of_columns]; diff --git a/beacon_node/lighthouse_network/src/rpc/methods.rs b/beacon_node/lighthouse_network/src/rpc/methods.rs index 0ff469aafbb..8fe57ae23aa 100644 --- a/beacon_node/lighthouse_network/src/rpc/methods.rs +++ b/beacon_node/lighthouse_network/src/rpc/methods.rs @@ -27,7 +27,7 @@ pub const MAX_ERROR_LEN: u64 = 256; /// The max number of blobs we expect in the configs to set for compile time params. /// Note: This value is an estimate that we should use only for rate limiting, /// bounds checking and other non-consensus critical operations. -/// +/// /// For exact value, we should always check the chainspec. pub const MAX_BLOBS_PER_BLOCK_CEILING: u64 = 16; @@ -335,11 +335,11 @@ pub struct BlobsByRangeRequest { impl BlobsByRangeRequest { /// This function provides an upper bound on number of blobs expected in /// a certain slot range. - /// + /// /// Note: **must not** use for anything consensus critical, only for - /// bounds checking and rate limiting. + /// bounds checking and rate limiting. pub fn max_blobs_requested(&self) -> u64 { - self.count.saturating_mul(MAX_BLOBS_PER_BLOCK_CEILING as u64) + self.count.saturating_mul(MAX_BLOBS_PER_BLOCK_CEILING) } } diff --git a/beacon_node/network/src/sync/network_context.rs b/beacon_node/network/src/sync/network_context.rs index 4b255fee4d4..a489ed70575 100644 --- a/beacon_node/network/src/sync/network_context.rs +++ b/beacon_node/network/src/sync/network_context.rs @@ -35,8 +35,8 @@ use std::time::Duration; use tokio::sync::mpsc; use types::blob_sidecar::FixedBlobSidecarList; use types::{ - BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, - Hash256, SignedBeaconBlock, Slot, + BlobSidecar, ColumnIndex, DataColumnSidecar, DataColumnSidecarList, EthSpec, Hash256, + SignedBeaconBlock, Slot, }; pub mod custody; diff --git a/consensus/types/src/blob_sidecar.rs b/consensus/types/src/blob_sidecar.rs index 108c807b475..9acbd8d95c2 100644 --- a/consensus/types/src/blob_sidecar.rs +++ b/consensus/types/src/blob_sidecar.rs @@ -265,7 +265,7 @@ impl BlobSidecar { } pub type BlobSidecarList = RuntimeVariableList>>; -/// Alias for a non length-constrained list of `BlobSidecar`s. +/// Alias for a non length-constrained list of `BlobSidecar`s. pub type FixedBlobSidecarList = RuntimeFixedList>>>; pub type BlobsList = VariableList, ::MaxBlobCommitmentsPerBlock>; diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index b24be185da8..8d03535f34d 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -10,6 +10,11 @@ use std::slice::SliceIndex; /// An ordered, heap-allocated, variable-length, homogeneous collection of `T`, with no more than /// `max_len` values. /// +/// In cases where the `max_length` of the container is unknown at time of initialization, we provide +/// a `Self::empty_uninitialized` constructor that initializes a runtime list without setting the max_len. +/// +/// To ensure there are no inconsistent states, we do not allow any mutating operation if `max_len` is not set. +/// /// ## Example /// /// ``` @@ -35,6 +40,16 @@ use std::slice::SliceIndex; /// /// // Push a value to if it _does_ exceed the maximum. /// assert!(long.push(6).is_err()); +/// +/// let mut uninit = RuntimeVariableList::empty_unitialized(); +/// assert!(uninit.push(5).is_err()); +/// +/// // Set max_len to allow mutation. +/// uninit.set_max_len(5usize); +/// +/// uninit.push(5).unwrap(); +/// assert_eq!(&uninit[..], &[5]); +/// /// ``` #[derive(Debug, Clone, Serialize, Deserialize, Derivative)] #[derivative(PartialEq, Eq, Hash(bound = "T: std::hash::Hash"))] @@ -73,7 +88,7 @@ impl RuntimeVariableList { } } - /// Create an empty list. + /// Create an empty list with the given `max_len`. pub fn empty(max_len: usize) -> Self { Self { vec: vec![], @@ -95,7 +110,7 @@ impl RuntimeVariableList { /// Returns an instance of `Self` with max_len = None. /// /// No mutating operation can be performed on an uninitialized instance - /// without first setting max_len. + /// without first setting `max_len`. pub fn empty_uninitialized() -> Self { Self { vec: vec![], @@ -129,7 +144,7 @@ impl RuntimeVariableList { /// Returns `Err(())` when appending `value` would exceed the maximum length. pub fn push(&mut self, value: T) -> Result<(), Error> { let Some(max_len) = self.max_len else { - // TODO(pawan): set a better error + // TODO(pawan): set a better error? return Err(Error::MissingLengthInformation); }; if self.vec.len() < max_len { @@ -247,6 +262,7 @@ where } } +/// Emulates a SSZ `Vector`. #[derive(Clone, Debug)] pub struct RuntimeFixedList { vec: Vec, @@ -267,6 +283,7 @@ impl RuntimeFixedList { self.vec.as_slice() } + #[allow(clippy::len_without_is_empty)] pub fn len(&self) -> usize { self.len } From d37733b846ce58e318e976d6503ca394b4901141 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 4 Sep 2024 12:47:36 -0700 Subject: [PATCH 7/9] Fix test compilations --- .../overflow_lru_cache.rs | 73 ++++++++++--------- .../src/observed_data_sidecars.rs | 6 +- beacon_node/beacon_chain/tests/events.rs | 2 +- .../network/src/sync/block_lookups/tests.rs | 13 +++- .../src/sync/block_sidecar_coupling.rs | 42 +++++++---- consensus/types/src/runtime_var_list.rs | 18 ++--- 6 files changed, 86 insertions(+), 68 deletions(-) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index ad027a45dd0..ebbb1c5a6fa 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -837,7 +837,8 @@ mod test { info!(log, "done printing kzg commitments"); let gossip_verified_blobs = if let Some((kzg_proofs, blobs)) = maybe_blobs { - let sidecars = BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap(); + let sidecars = + BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap(); Vec::from(sidecars) .into_iter() .map(|sidecar| { @@ -1152,7 +1153,7 @@ mod pending_components_tests { use super::*; use crate::block_verification_types::BlockImportData; use crate::eth1_finalization_cache::Eth1FinalizationData; - use crate::test_utils::{generate_rand_block_and_blobs, NumBlobs}; + use crate::test_utils::{generate_rand_block_and_blobs, test_spec, NumBlobs}; use crate::PayloadVerificationOutcome; use fork_choice::PayloadVerificationStatus; use kzg::KzgCommitment; @@ -1168,15 +1169,19 @@ mod pending_components_tests { type Setup = ( SignedBeaconBlock, - FixedVector>>, ::MaxBlobsPerBlock>, - FixedVector>>, ::MaxBlobsPerBlock>, + RuntimeFixedList>>>, + RuntimeFixedList>>>, + usize, ); pub fn pre_setup() -> Setup { let mut rng = StdRng::seed_from_u64(0xDEADBEEF0BAD5EEDu64); + let spec = test_spec::(); let (block, blobs_vec) = generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Random, &mut rng); - let mut blobs: FixedVector<_, ::MaxBlobsPerBlock> = FixedVector::default(); + let max_len = spec.max_blobs_per_block(block.epoch()) as usize; + let mut blobs: RuntimeFixedList>>> = + RuntimeFixedList::default(max_len); for blob in blobs_vec { if let Some(b) = blobs.get_mut(blob.index as usize) { @@ -1184,10 +1189,8 @@ mod pending_components_tests { } } - let mut invalid_blobs: FixedVector< - Option>>, - ::MaxBlobsPerBlock, - > = FixedVector::default(); + let mut invalid_blobs: RuntimeFixedList>>> = + RuntimeFixedList::default(max_len); for (index, blob) in blobs.iter().enumerate() { if let Some(invalid_blob) = blob { let mut blob_copy = invalid_blob.as_ref().clone(); @@ -1196,21 +1199,21 @@ mod pending_components_tests { } } - (block, blobs, invalid_blobs) + (block, blobs, invalid_blobs, max_len) } type PendingComponentsSetup = ( DietAvailabilityPendingExecutedBlock, - FixedVector>, ::MaxBlobsPerBlock>, - FixedVector>, ::MaxBlobsPerBlock>, + RuntimeFixedList>>, + RuntimeFixedList>>, ); pub fn setup_pending_components( block: SignedBeaconBlock, - valid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, - invalid_blobs: FixedVector>>, ::MaxBlobsPerBlock>, + valid_blobs: RuntimeFixedList>>>, + invalid_blobs: RuntimeFixedList>>>, ) -> PendingComponentsSetup { - let blobs = FixedVector::from( + let blobs = RuntimeFixedList::new( valid_blobs .iter() .map(|blob_opt| { @@ -1220,7 +1223,7 @@ mod pending_components_tests { }) .collect::>(), ); - let invalid_blobs = FixedVector::from( + let invalid_blobs = RuntimeFixedList::new( invalid_blobs .iter() .map(|blob_opt| { @@ -1252,10 +1255,10 @@ mod pending_components_tests { (block.into(), blobs, invalid_blobs) } - pub fn assert_cache_consistent(cache: PendingComponents) { + pub fn assert_cache_consistent(cache: PendingComponents, max_len: usize) { if let Some(cached_block) = cache.get_cached_block() { let cached_block_commitments = cached_block.get_commitments(); - for index in 0..E::max_blobs_per_block() { + for index in 0..max_len { let block_commitment = cached_block_commitments.get(index).copied(); let blob_commitment_opt = cache.get_cached_blobs().get(index).unwrap(); let blob_commitment = blob_commitment_opt.as_ref().map(|b| *b.get_commitment()); @@ -1274,40 +1277,40 @@ mod pending_components_tests { #[test] fn valid_block_invalid_blobs_valid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn invalid_blobs_block_valid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_block(block_commitments); cache.merge_blobs(blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn invalid_blobs_valid_blobs_block() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(random_blobs); cache.merge_blobs(blobs); cache.merge_block(block_commitments); @@ -1317,46 +1320,46 @@ mod pending_components_tests { #[test] fn block_valid_blobs_invalid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_block(block_commitments); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn valid_blobs_block_invalid_blobs() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_block(block_commitments); cache.merge_blobs(random_blobs); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } #[test] fn valid_blobs_invalid_blobs_block() { - let (block_commitments, blobs, random_blobs) = pre_setup(); + let (block_commitments, blobs, random_blobs, max_len) = pre_setup(); let (block_commitments, blobs, random_blobs) = setup_pending_components(block_commitments, blobs, random_blobs); let block_root = Hash256::zero(); - let mut cache = >::empty(block_root); + let mut cache = >::empty(block_root, max_len); cache.merge_blobs(blobs); cache.merge_blobs(random_blobs); cache.merge_block(block_commitments); - assert_cache_consistent(cache); + assert_cache_consistent(cache, max_len); } } diff --git a/beacon_node/beacon_chain/src/observed_data_sidecars.rs b/beacon_node/beacon_chain/src/observed_data_sidecars.rs index 7cc5f2e92a1..318f9f252d5 100644 --- a/beacon_node/beacon_chain/src/observed_data_sidecars.rs +++ b/beacon_node/beacon_chain/src/observed_data_sidecars.rs @@ -155,7 +155,7 @@ mod tests { use crate::test_utils::test_spec; use bls::Hash256; use std::sync::Arc; - use types::MainnetEthSpec; + use types::{Epoch, MainnetEthSpec}; type E = MainnetEthSpec; @@ -309,7 +309,7 @@ mod tests { #[test] fn simple_observations() { let spec = test_spec::(); - let mut cache = ObservedDataSidecars::>::new(spec); + let mut cache = ObservedDataSidecars::>::new(spec.clone()); // Slot 0, index 0 let proposer_index_a = 420; @@ -465,7 +465,7 @@ mod tests { ); // Try adding an out of bounds index - let invalid_index = E::max_blobs_per_block() as u64; + let invalid_index = spec.max_blobs_per_block(Epoch::new(0)); let sidecar_d = get_blob_sidecar(0, proposer_index_a, invalid_index); assert_eq!( cache.observe_sidecar(&sidecar_d), diff --git a/beacon_node/beacon_chain/tests/events.rs b/beacon_node/beacon_chain/tests/events.rs index d54543e4f6f..463453f0aec 100644 --- a/beacon_node/beacon_chain/tests/events.rs +++ b/beacon_node/beacon_chain/tests/events.rs @@ -69,7 +69,7 @@ async fn blob_sidecar_event_on_process_rpc_blobs() { index: 1, ..BlobSidecar::random_valid(&mut rng, kzg).unwrap() }); - let blobs = FixedBlobSidecarList::from(vec![Some(blob_1.clone()), Some(blob_2.clone())]); + let blobs = FixedBlobSidecarList::new(vec![Some(blob_1.clone()), Some(blob_2.clone())]); let expected_sse_blobs = vec![ SseBlobSidecar::from_blob_sidecar(blob_1.as_ref()), SseBlobSidecar::from_blob_sidecar(blob_2.as_ref()), diff --git a/beacon_node/network/src/sync/block_lookups/tests.rs b/beacon_node/network/src/sync/block_lookups/tests.rs index 6d852b2572d..afd2fe0c861 100644 --- a/beacon_node/network/src/sync/block_lookups/tests.rs +++ b/beacon_node/network/src/sync/block_lookups/tests.rs @@ -39,7 +39,7 @@ use types::{ test_utils::{SeedableRng, XorShiftRng}, BlobSidecar, ForkName, MinimalEthSpec as E, SignedBeaconBlock, Slot, }; -use types::{BeaconState, BeaconStateBase}; +use types::{BeaconState, BeaconStateBase, ChainSpec}; use types::{DataColumnSidecar, Epoch}; type T = Witness, E, MemoryStore, MemoryStore>; @@ -86,6 +86,7 @@ struct TestRig { /// `rng` for generating test blocks and blobs. rng: XorShiftRng, fork_name: ForkName, + spec: ChainSpec, log: Logger, } @@ -153,6 +154,8 @@ impl TestRig { .network_globals .set_sync_state(SyncState::Synced); + let spec = chain.spec.clone(); + let rng = XorShiftRng::from_seed([42; 16]); TestRig { beacon_processor_rx, @@ -174,6 +177,7 @@ impl TestRig { harness, fork_name, log, + spec, } } @@ -2048,8 +2052,8 @@ mod deneb_only { use beacon_chain::{ block_verification_types::RpcBlock, data_availability_checker::AvailabilityCheckError, }; - use ssz_types::VariableList; use std::collections::VecDeque; + use types::RuntimeVariableList; struct DenebTester { rig: TestRig, @@ -2407,12 +2411,15 @@ mod deneb_only { fn parent_block_unknown_parent(mut self) -> Self { self.rig.log("parent_block_unknown_parent"); let block = self.unknown_parent_block.take().unwrap(); + let max_len = self.rig.spec.max_blobs_per_block(block.epoch()) as usize; // Now this block is the one we expect requests from self.block = block.clone(); let block = RpcBlock::new( Some(block.canonical_root()), block, - self.unknown_parent_blobs.take().map(VariableList::from), + self.unknown_parent_blobs + .take() + .map(|vec| RuntimeVariableList::from_vec(vec, max_len)), ) .unwrap(); self.rig.parent_block_processed( diff --git a/beacon_node/network/src/sync/block_sidecar_coupling.rs b/beacon_node/network/src/sync/block_sidecar_coupling.rs index f241a52c9c8..0d0a5b80815 100644 --- a/beacon_node/network/src/sync/block_sidecar_coupling.rs +++ b/beacon_node/network/src/sync/block_sidecar_coupling.rs @@ -252,12 +252,15 @@ mod tests { #[test] fn no_blobs_into_responses() { + let spec = test_spec::(); let peer_id = PeerId::random(); - let mut info = RangeBlockComponentsRequest::::new(false, None, None, vec![peer_id]); let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| generate_rand_block_and_blobs::(ForkName::Base, NumBlobs::None, &mut rng).0) .collect::>(); + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().epoch()) as usize; + let mut info = + RangeBlockComponentsRequest::::new(false, None, None, vec![peer_id], max_len); // Send blocks and complete terminate response for block in blocks { @@ -272,8 +275,8 @@ mod tests { #[test] fn empty_blobs_into_responses() { + let spec = test_spec::(); let peer_id = PeerId::random(); - let mut info = RangeBlockComponentsRequest::::new(true, None, None, vec![peer_id]); let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { @@ -281,6 +284,9 @@ mod tests { generate_rand_block_and_blobs::(ForkName::Deneb, NumBlobs::Number(3), &mut rng).0 }) .collect::>(); + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().epoch()) as usize; + let mut info = + RangeBlockComponentsRequest::::new(true, None, None, vec![peer_id], max_len); // Send blocks and complete terminate response for block in blocks { @@ -301,12 +307,7 @@ mod tests { fn rpc_block_with_custody_columns() { let spec = test_spec::(); let expects_custody_columns = vec![1, 2, 3, 4]; - let mut info = RangeBlockComponentsRequest::::new( - false, - Some(expects_custody_columns.clone()), - Some(expects_custody_columns.len()), - vec![PeerId::random()], - ); + let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { @@ -318,7 +319,14 @@ mod tests { ) }) .collect::>(); - + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().0.epoch()) as usize; + let mut info = RangeBlockComponentsRequest::::new( + false, + Some(expects_custody_columns.clone()), + Some(expects_custody_columns.len()), + vec![PeerId::random()], + max_len, + ); // Send blocks and complete terminate response for block in &blocks { info.add_block_response(Some(block.0.clone().into())); @@ -362,12 +370,7 @@ mod tests { let spec = test_spec::(); let expects_custody_columns = vec![1, 2, 3, 4]; let num_of_data_column_requests = 2; - let mut info = RangeBlockComponentsRequest::::new( - false, - Some(expects_custody_columns.clone()), - Some(num_of_data_column_requests), - vec![PeerId::random()], - ); + let mut rng = XorShiftRng::from_seed([42; 16]); let blocks = (0..4) .map(|_| { @@ -379,7 +382,14 @@ mod tests { ) }) .collect::>(); - + let max_len = spec.max_blobs_per_block(blocks.first().unwrap().0.epoch()) as usize; + let mut info = RangeBlockComponentsRequest::::new( + false, + Some(expects_custody_columns.clone()), + Some(num_of_data_column_requests), + vec![PeerId::random()], + max_len, + ); // Send blocks and complete terminate response for block in &blocks { info.add_block_response(Some(block.0.clone().into())); diff --git a/consensus/types/src/runtime_var_list.rs b/consensus/types/src/runtime_var_list.rs index 8d03535f34d..e93f07981ec 100644 --- a/consensus/types/src/runtime_var_list.rs +++ b/consensus/types/src/runtime_var_list.rs @@ -292,6 +292,13 @@ impl RuntimeFixedList { self.vec } + pub fn default(max_len: usize) -> Self { + Self { + vec: vec![T::default(); max_len], + len: max_len, + } + } + pub fn take(&mut self) -> Self { let new = std::mem::take(&mut self.vec); *self = Self::new(vec![T::default(); self.len]); @@ -339,7 +346,6 @@ mod test { use super::*; use ssz::*; use std::fmt::Debug; - use tree_hash::TreeHash; #[test] fn new() { @@ -404,7 +410,7 @@ mod test { } fn round_trip(item: RuntimeVariableList) { - let max_len = item.max_len(); + let max_len = item.max_len().unwrap(); let encoded = &item.as_ssz_bytes(); assert_eq!(item.ssz_bytes_len(), encoded.len()); assert_eq!( @@ -418,12 +424,4 @@ mod test { round_trip::(RuntimeVariableList::from_vec(vec![42; 8], 8)); round_trip::(RuntimeVariableList::from_vec(vec![0; 8], 8)); } - - #[test] - fn test_empty_list_encoding() { - use ssz_types::{typenum::U16, VariableList}; - - let a: RuntimeVariableList = RuntimeVariableList::from_vec(vec![], 16); - dbg!(a.tree_hash_root()); - } } From 12c6ef118a1a6d910c48d9d4b23004f3609264c7 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Wed, 4 Sep 2024 16:16:36 -0700 Subject: [PATCH 8/9] Fix some more tests --- beacon_node/beacon_chain/tests/block_verification.rs | 6 +++--- .../network/src/network_beacon_processor/tests.rs | 11 ++++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/beacon_node/beacon_chain/tests/block_verification.rs b/beacon_node/beacon_chain/tests/block_verification.rs index faa4d74a182..174d1a68e5f 100644 --- a/beacon_node/beacon_chain/tests/block_verification.rs +++ b/beacon_node/beacon_chain/tests/block_verification.rs @@ -206,7 +206,7 @@ fn update_blob_signed_header( signed_block: &SignedBeaconBlock, blobs: &mut BlobSidecarList, ) { - for old_blob_sidecar in blobs.iter_mut() { + for old_blob_sidecar in blobs.as_mut_slice().unwrap() { let new_blob = Arc::new(BlobSidecar:: { index: old_blob_sidecar.index, blob: old_blob_sidecar.blob.clone(), @@ -1214,7 +1214,7 @@ async fn verify_block_for_gossip_slashing_detection() { let slasher = Arc::new( Slasher::open( SlasherConfig::new(slasher_dir.path().into()), - spec, + spec.clone(), test_logger(), ) .unwrap(), @@ -1238,7 +1238,7 @@ async fn verify_block_for_gossip_slashing_detection() { if let Some((kzg_proofs, blobs)) = blobs1 { let sidecars = - BlobSidecar::build_sidecars(blobs, verified_block.block(), kzg_proofs).unwrap(); + BlobSidecar::build_sidecars(blobs, verified_block.block(), kzg_proofs, &spec).unwrap(); for sidecar in sidecars { let blob_index = sidecar.index; let verified_blob = harness diff --git a/beacon_node/network/src/network_beacon_processor/tests.rs b/beacon_node/network/src/network_beacon_processor/tests.rs index 40c69a0baa5..9207f6a2dd6 100644 --- a/beacon_node/network/src/network_beacon_processor/tests.rs +++ b/beacon_node/network/src/network_beacon_processor/tests.rs @@ -256,7 +256,7 @@ impl TestRig { assert!(beacon_processor.is_ok()); let block = next_block_tuple.0; let blob_sidecars = if let Some((kzg_proofs, blobs)) = next_block_tuple.1 { - Some(BlobSidecar::build_sidecars(blobs, &block, kzg_proofs).unwrap()) + Some(BlobSidecar::build_sidecars(blobs, &block, kzg_proofs, &chain.spec).unwrap()) } else { None }; @@ -341,7 +341,7 @@ impl TestRig { } pub fn enqueue_single_lookup_rpc_blobs(&self) { if let Some(blobs) = self.next_blobs.clone() { - let blobs = FixedBlobSidecarList::from(blobs.into_iter().map(Some).collect::>()); + let blobs = FixedBlobSidecarList::new(blobs.into_iter().map(Some).collect::>()); self.network_beacon_processor .send_rpc_blobs( self.next_block.canonical_root(), @@ -1100,7 +1100,12 @@ async fn test_blobs_by_range() { .block_root_at_slot(Slot::new(slot), WhenSlotSkipped::None) .unwrap(); blob_count += root - .map(|root| rig.chain.get_blobs(&root).unwrap_or_default().len()) + .map(|root| { + rig.chain + .get_blobs(&root) + .map(|list| list.len()) + .unwrap_or(0) + }) .unwrap_or(0); } let mut actual_count = 0; From 2fcb2935ec7ef4cd18bbdd8aedb7de61fac69e61 Mon Sep 17 00:00:00 2001 From: Pawan Dhananjay Date: Fri, 6 Sep 2024 18:28:31 -0700 Subject: [PATCH 9/9] Fix test from unstable --- .../src/data_availability_checker/overflow_lru_cache.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs index ebbb1c5a6fa..1735e782573 100644 --- a/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs +++ b/beacon_node/beacon_chain/src/data_availability_checker/overflow_lru_cache.rs @@ -972,6 +972,8 @@ mod test { assert_eq!(cache.critical.read().len(), 1); } } + // remove the blob to simulate successful import + cache.remove_pending_components(root); assert!( cache.critical.read().is_empty(), "cache should be empty now that all components available"