From 6e730c26ac59c67b03604b710f9dab4fc02233f5 Mon Sep 17 00:00:00 2001 From: Anders Konring Date: Fri, 23 Aug 2024 13:41:26 +0200 Subject: [PATCH 01/40] compute actual multiplicity from max_multiplicity This commit defines a function `min_multiplicity` which can compute the actual multiplicity that will be used from `max_multiplicity` and `payload_len`. The original argument `multiplicity` has been renamed to `max_multiplicity` to indicate that this is an upper bound. --- vid/src/advz.rs | 130 ++++++++++++++++++++------------- vid/src/advz/payload_prover.rs | 8 +- vid/src/advz/precomputable.rs | 26 ++++--- 3 files changed, 100 insertions(+), 64 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 2951879a9..0f43e7113 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -83,7 +83,7 @@ where { recovery_threshold: u32, num_storage_nodes: u32, - multiplicity: u32, + max_multiplicity: u32, ck: KzgProverParam, vk: KzgVerifierParam, multi_open_domain: Radix2EvaluationDomain>, @@ -131,15 +131,20 @@ where ) -> VidResult { // TODO intelligent choice of multiplicity // https://github.com/EspressoSystems/jellyfish/issues/534 - let multiplicity = 1; + let max_multiplicity = 1; - Self::with_multiplicity_internal(num_storage_nodes, recovery_threshold, multiplicity, srs) + Self::with_multiplicity_internal( + num_storage_nodes, + recovery_threshold, + max_multiplicity, + srs, + ) } pub(crate) fn with_multiplicity_internal( num_storage_nodes: u32, // n (code rate: r = k/n) recovery_threshold: u32, // k - multiplicity: u32, // batch m chunks, keep the rate r = (m*k)/(m*n) + max_multiplicity: u32, // batch m chunks, keep the rate r = (m*k)/(m*n) srs: impl Borrow>, ) -> VidResult { if num_storage_nodes < recovery_threshold { @@ -149,15 +154,15 @@ where ))); } - if !multiplicity.is_power_of_two() { + if !max_multiplicity.is_power_of_two() { return Err(VidError::Argument(format!( - "multiplicity {multiplicity} should be a power of two" + "max multiplicity {max_multiplicity} should be a power of two" ))); } // erasure code params - let chunk_size = multiplicity * recovery_threshold; // message length m - let code_word_size = multiplicity * num_storage_nodes; // code word length n + let chunk_size = max_multiplicity * recovery_threshold; // message length m + let code_word_size = max_multiplicity * num_storage_nodes; // code word length n let poly_degree = chunk_size - 1; let (ck, vk) = UnivariateKzgPCS::trim_fft_size(srs, poly_degree as usize).map_err(vid)?; @@ -187,7 +192,7 @@ where Ok(Self { recovery_threshold, num_storage_nodes, - multiplicity, + max_multiplicity, ck, vk, multi_open_domain, @@ -224,21 +229,28 @@ where Self::new_internal(num_storage_nodes, recovery_threshold, srs) } - /// Like [`Advz::new`] except with a `multiplicity` arg. + /// Like [`Advz::new`] except with a `max_multiplicity` arg. /// - /// `multiplicity` is an implementation-specific optimization arg. - /// Each storage node gets `multiplicity` evaluations per polynomial. + /// `max_multiplicity` is an implementation-specific optimization arg. + /// Each storage node gets up to `max_multiplicity` evaluations per + /// polynomial. The actual multiplicity used will be the smallest value m + /// such that payload can fit (payload_len <= m * recovery_threshold). /// /// # Errors /// In addition to [`Advz::new`], return [`VidError::Argument`] if - /// - TEMPORARY `multiplicity` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/339) + /// - TEMPORARY `max_multiplicity` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/339) pub fn with_multiplicity( num_storage_nodes: u32, recovery_threshold: u32, - multiplicity: u32, + max_multiplicity: u32, srs: impl Borrow>, ) -> VidResult { - Self::with_multiplicity_internal(num_storage_nodes, recovery_threshold, multiplicity, srs) + Self::with_multiplicity_internal( + num_storage_nodes, + recovery_threshold, + max_multiplicity, + srs, + ) } } @@ -262,13 +274,13 @@ where pub fn with_multiplicity( num_storage_nodes: u32, recovery_threshold: u32, - multiplicity: u32, + max_multiplicity: u32, srs: impl Borrow>, ) -> VidResult { let mut advz = Self::with_multiplicity_internal( num_storage_nodes, recovery_threshold, - multiplicity, + max_multiplicity, srs, )?; advz.init_gpu_srs()?; @@ -307,7 +319,7 @@ where evals: Vec>, #[serde(with = "canonical")] - // aggretate_proofs.len() equals self.multiplicity + // aggretate_proofs.len() equals multiplicity // TODO further aggregate into a single KZG proof. aggregate_proofs: Vec>, @@ -407,8 +419,11 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); + let payload_byte_len = payload.len().try_into().map_err(vid)?; + let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let chunk_size = multiplicity * self.recovery_threshold; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload, chunk_size as usize); end_timer!(bytes_to_polys_time); let poly_commits_time = start_timer!(|| "batch poly commit"); @@ -428,12 +443,13 @@ where "VID disperse {} payload bytes to {} nodes", payload_byte_len, self.num_storage_nodes )); - let _chunk_size = self.multiplicity * self.recovery_threshold; - let code_word_size = self.multiplicity * self.num_storage_nodes; + let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let chunk_size = multiplicity * self.recovery_threshold; + let code_word_size = multiplicity * self.num_storage_nodes; // partition payload into polynomial coefficients let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload, chunk_size as usize); end_timer!(bytes_to_polys_time); // evaluate polynomials @@ -442,7 +458,7 @@ where polys.len(), _chunk_size )); - let all_storage_node_evals = self.evaluate_polys(&polys)?; + let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; end_timer!(all_storage_node_evals_timer); // vector commitment to polynomial evaluations @@ -458,7 +474,7 @@ where all_evals_digest: all_evals_commit.commitment().digest(), payload_byte_len, num_storage_nodes: self.num_storage_nodes, - multiplicity: self.multiplicity, + multiplicity, }; end_timer!(common_timer); @@ -489,8 +505,12 @@ where end_timer!(agg_proofs_timer); let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); - let shares = - self.assemble_shares(all_storage_node_evals, aggregate_proofs, all_evals_commit)?; + let shares = self.assemble_shares( + all_storage_node_evals, + aggregate_proofs, + all_evals_commit, + multiplicity, + )?; end_timer!(assemblage_timer); end_timer!(disperse_time); @@ -514,21 +534,21 @@ where common.num_storage_nodes, self.num_storage_nodes ))); } - if common.multiplicity != self.multiplicity { + let multiplicity = self.min_multiplicity(common.payload_byte_len, self.max_multiplicity); + if common.multiplicity != multiplicity { return Err(VidError::Argument(format!( - "common multiplicity {} differs from self {}", - common.multiplicity, self.multiplicity + "common multiplicity {} differs from derived min {}", + common.multiplicity, multiplicity ))); } - let multiplicity: usize = common.multiplicity.try_into().map_err(vid)?; - if share.evals.len() / multiplicity != common.poly_commits.len() { + if share.evals.len() / multiplicity as usize != common.poly_commits.len() { return Err(VidError::Argument(format!( "number of share evals / multiplicity {}/{} differs from number of common polynomial commitments {}", share.evals.len(), multiplicity, common.poly_commits.len() ))); } - if share.eval_proofs.len() != multiplicity { + if share.eval_proofs.len() != multiplicity as usize { return Err(VidError::Argument(format!( "number of eval_proofs {} differs from common multiplicity {}", share.eval_proofs.len(), @@ -543,10 +563,10 @@ where } // verify eval proofs - for i in 0..self.multiplicity { + for i in 0..multiplicity { if KzgEvalsMerkleTree::::verify( common.all_evals_digest, - &KzgEvalsMerkleTreeIndex::::from((share.index * self.multiplicity) + i), + &KzgEvalsMerkleTreeIndex::::from((share.index * multiplicity) + i), &share.eval_proofs[i as usize], ) .map_err(vid)? @@ -577,7 +597,7 @@ where // // some boilerplate needed to accommodate builds without `parallel` // feature. - let multiplicities = Vec::from_iter((0..self.multiplicity as usize)); + let multiplicities = Vec::from_iter((0..multiplicity as usize)); let polys_len = common.poly_commits.len(); let verification_iter = parallelizable_slice_iter(&multiplicities).map(|i| { let range = i * polys_len..(i + 1) * polys_len; @@ -601,7 +621,7 @@ where &aggregate_poly_commit, &self .multi_open_domain - .element((share.index as usize * multiplicity) + i), + .element((share.index * multiplicity) as usize + i), &aggregate_eval, &share.aggregate_proofs[*i], ) @@ -645,6 +665,7 @@ where .ok_or_else(|| VidError::Argument("shares is empty".into()))? .evals .len(); + let multiplicity = common.multiplicity; if let Some((index, share)) = shares .iter() .enumerate() @@ -658,15 +679,15 @@ where share.evals.len() ))); } - if num_evals != self.multiplicity as usize * common.poly_commits.len() { + if num_evals != multiplicity as usize * common.poly_commits.len() { return Err(VidError::Argument(format!( "num_evals should be (multiplicity * poly_commits): {} but is instead: {}", - self.multiplicity as usize * common.poly_commits.len(), + multiplicity as usize * common.poly_commits.len(), num_evals, ))); } - let chunk_size = self.multiplicity * self.recovery_threshold; - let num_polys = num_evals / self.multiplicity as usize; + let chunk_size = multiplicity * self.recovery_threshold; + let num_polys = num_evals / multiplicity as usize; let elems_capacity = num_polys * chunk_size as usize; let mut elems = Vec::with_capacity(elems_capacity); @@ -675,9 +696,9 @@ where for p in 0..num_polys { for share in shares { // extract all evaluations for polynomial p from the share - for m in 0..self.multiplicity as usize { + for m in 0..multiplicity as usize { evals.push(( - (share.index * self.multiplicity) as usize + m, + (share.index * multiplicity) as usize + m, share.evals[(m * num_polys) + p], )) } @@ -741,12 +762,12 @@ where fn evaluate_polys( &self, polys: &[DensePolynomial<::ScalarField>], + code_word_size: usize, ) -> Result::ScalarField>>, VidError> where E: Pairing, H: HasherDigest, { - let code_word_size = (self.num_storage_nodes * self.multiplicity) as usize; let mut all_storage_node_evals = vec![Vec::with_capacity(polys.len()); code_word_size]; // this is to avoid `SrsRef` not implementing `Sync` problem, // instead of sending entire `self` cross thread, we only send a ref which is @@ -809,11 +830,14 @@ where Ok(PrimeField::from_le_bytes_mod_order(&hasher.finalize())) } - fn bytes_to_polys(&self, payload: &[u8]) -> Vec::ScalarField>> + fn bytes_to_polys( + &self, + payload: &[u8], + chunk_size: usize, + ) -> Vec::ScalarField>> where E: Pairing, { - let chunk_size = (self.recovery_threshold * self.multiplicity) as usize; let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); let eval_domain_ref = &self.eval_domain; @@ -858,16 +882,17 @@ where DenseUVPolynomial::from_coefficients_vec(coeffs_vec) } - fn polynomial(&self, coeffs: I) -> KzgPolynomial + fn polynomial(&self, coeffs: I, chunk_size: usize) -> KzgPolynomial where I: Iterator, I::Item: Borrow>, { - Self::polynomial_internal( - &self.eval_domain, - (self.recovery_threshold * self.multiplicity) as usize, - coeffs, - ) + Self::polynomial_internal(&self.eval_domain, chunk_size, coeffs) + } + + fn min_multiplicity(&self, payload_byte_len: u32, multiplicity: u32) -> u32 { + let _elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); + multiplicity } /// Derive a commitment from whatever data is needed. @@ -914,6 +939,7 @@ where all_storage_node_evals: Vec::ScalarField>>, aggregate_proofs: Vec>, all_evals_commit: KzgEvalsMerkleTree, + multiplicity: u32, ) -> Result>, VidError> where E: Pairing, @@ -937,7 +963,7 @@ where // split share data into chunks of size multiplicity Ok(share_data .into_iter() - .chunks(self.multiplicity as usize) + .chunks(multiplicity as usize) .into_iter() .enumerate() .map(|(index, chunk)| { diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 93d1cacb8..f40a52432 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -95,12 +95,14 @@ where let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); + let chunk_size = + self.min_multiplicity(payload.len() as u32, self.recovery_threshold) as usize; for (i, evals_iter) in elems_iter .chunks(self.recovery_threshold as usize) .into_iter() .enumerate() { - let poly = self.polynomial(evals_iter); + let poly = self.polynomial(evals_iter, chunk_size); let points_range = Range { // first polynomial? skip to the start of the proof range start: if i == 0 { offset_elem } else { 0 }, @@ -260,14 +262,14 @@ where .chain(proof.suffix_bytes.iter()), )) .chain(proof.suffix_elems.iter().cloned()); - + let chunk_size = (stmt.common.multiplicity * self.recovery_threshold) as usize; // rebuild the poly commits, check against `common` for (commit_index, evals_iter) in range_poly.into_iter().zip( elems_iter .chunks(self.recovery_threshold as usize) .into_iter(), ) { - let poly = self.polynomial(evals_iter); + let poly = self.polynomial(evals_iter, chunk_size); let poly_commit = UnivariateKzgPCS::commit(&self.ck, &poly).map_err(vid)?; if poly_commit != stmt.common.poly_commits[commit_index] { return Ok(Err(())); diff --git a/vid/src/advz/precomputable.rs b/vid/src/advz/precomputable.rs index 046367408..9823c453d 100644 --- a/vid/src/advz/precomputable.rs +++ b/vid/src/advz/precomputable.rs @@ -39,7 +39,10 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let polys = self.bytes_to_polys(payload); + let payload_byte_len = payload.len().try_into().map_err(vid)?; + let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let chunk_size = (multiplicity * self.recovery_threshold) as usize; + let polys = self.bytes_to_polys(payload, chunk_size); let poly_commits: Vec> = UnivariateKzgPCS::batch_commit(&self.ck, &polys).map_err(vid)?; Ok(( @@ -63,22 +66,23 @@ where payload_byte_len, self.num_storage_nodes )); - let _chunk_size = self.multiplicity * self.recovery_threshold; - let code_word_size = self.multiplicity * self.num_storage_nodes; + let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let chunk_size = multiplicity * self.recovery_threshold; + let code_word_size = multiplicity * self.num_storage_nodes; // partition payload into polynomial coefficients // and count `elems_len` for later let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload, chunk_size as usize); end_timer!(bytes_to_polys_time); // evaluate polynomials let all_storage_node_evals_timer = start_timer!(|| ark_std::format!( "compute all storage node evals for {} polynomials with {} coefficients", polys.len(), - _chunk_size + chunk_size )); - let all_storage_node_evals = self.evaluate_polys(&polys)?; + let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; end_timer!(all_storage_node_evals_timer); // vector commitment to polynomial evaluations @@ -98,7 +102,7 @@ where all_evals_digest: all_evals_commit.commitment().digest(), payload_byte_len, num_storage_nodes: self.num_storage_nodes, - multiplicity: self.multiplicity, + multiplicity, }; end_timer!(common_timer); @@ -129,8 +133,12 @@ where end_timer!(agg_proofs_timer); let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); - let shares = - self.assemble_shares(all_storage_node_evals, aggregate_proofs, all_evals_commit)?; + let shares = self.assemble_shares( + all_storage_node_evals, + aggregate_proofs, + all_evals_commit, + multiplicity, + )?; end_timer!(assemblage_timer); end_timer!(disperse_time); From 656d096403eab3f30b03bda72c7057f585ee3adf Mon Sep 17 00:00:00 2001 From: Anders Konring Date: Fri, 23 Aug 2024 14:04:09 +0200 Subject: [PATCH 02/40] compute multiplicity --- vid/src/advz.rs | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 0f43e7113..5b3a449c5 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -891,8 +891,26 @@ where } fn min_multiplicity(&self, payload_byte_len: u32, multiplicity: u32) -> u32 { - let _elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); - multiplicity + let elem_bytes_len = + bytes_to_field::elem_byte_capacity::<::ScalarField>() as u32; + let elems = payload_byte_len / elem_bytes_len; + let recovery_threshold = self.recovery_threshold; + if recovery_threshold * multiplicity < elems { + // payload is large. no change in multiplicity needed. + return multiplicity; + } + + // payload is small: choose m such that 0 < m < multiplicity + // and m is the power of two that fits the payload best. + let mut m = multiplicity; + loop { + if elems <= recovery_threshold * m { + break; + } else { + m >> 1; + } + } + m } /// Derive a commitment from whatever data is needed. From bde475770f471e58a12734f8a6efe54184dfa457 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 23 Aug 2024 14:19:27 -0400 Subject: [PATCH 03/40] rename AdvzParams::multiplicity -> max_multiplicity --- vid/src/advz/test.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index d2aab1843..55e0b89b2 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -248,7 +248,7 @@ fn verify_share_with_multiplicity() { let advz_params = AdvzParams { recovery_threshold: 16, num_storage_nodes: 20, - multiplicity: 4, + max_multiplicity: 4, payload_len: 4000, }; let (mut advz, payload) = advz_init_with(advz_params); @@ -267,7 +267,7 @@ fn sad_path_verify_share_with_multiplicity() { let advz_params = AdvzParams { recovery_threshold: 16, num_storage_nodes: 20, - multiplicity: 32, // payload fitting into a single polynomial + max_multiplicity: 32, // payload fitting into a single polynomial payload_len: 8200, }; let (mut advz, payload) = advz_init_with(advz_params); @@ -362,7 +362,7 @@ fn verify_share_with_different_multiplicity_helper( struct AdvzParams { recovery_threshold: u32, num_storage_nodes: u32, - multiplicity: u32, + max_multiplicity: u32, payload_len: usize, } @@ -375,7 +375,7 @@ pub(super) fn advz_init() -> (Advz, Vec) { let advz_params = AdvzParams { recovery_threshold: 16, num_storage_nodes: 20, - multiplicity: 1, + max_multiplicity: 1, payload_len: 4000, }; advz_init_with(advz_params) @@ -383,17 +383,17 @@ pub(super) fn advz_init() -> (Advz, Vec) { fn advz_init_with(advz_params: AdvzParams) -> (Advz, Vec) { let mut rng = jf_utils::test_rng(); - let poly_len = advz_params.recovery_threshold * advz_params.multiplicity; + let poly_len = advz_params.recovery_threshold * advz_params.max_multiplicity; let srs = init_srs(poly_len as usize, &mut rng); assert_ne!( - advz_params.multiplicity, 0, + advz_params.max_multiplicity, 0, "multiplicity should not be zero" ); - let advz = if advz_params.multiplicity > 1 { + let advz = if advz_params.max_multiplicity > 1 { Advz::with_multiplicity( advz_params.num_storage_nodes, advz_params.recovery_threshold, - advz_params.multiplicity, + advz_params.max_multiplicity, srs, ) .unwrap() From c50fe2ab08963701dade3ff07a669be77e603bb2 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 26 Aug 2024 12:16:45 -0400 Subject: [PATCH 04/40] new test max_multiplicity --- vid/src/advz/test.rs | 85 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 82 insertions(+), 3 deletions(-) diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index 55e0b89b2..4dfcd7d6a 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -251,7 +251,7 @@ fn verify_share_with_multiplicity() { max_multiplicity: 4, payload_len: 4000, }; - let (mut advz, payload) = advz_init_with(advz_params); + let (mut advz, payload) = advz_init_with::(advz_params); let disperse = advz.disperse(payload).unwrap(); let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); @@ -270,7 +270,7 @@ fn sad_path_verify_share_with_multiplicity() { max_multiplicity: 32, // payload fitting into a single polynomial payload_len: 8200, }; - let (mut advz, payload) = advz_init_with(advz_params); + let (mut advz, payload) = advz_init_with::(advz_params); let disperse = advz.disperse(payload).unwrap(); let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); @@ -359,6 +359,85 @@ fn verify_share_with_different_multiplicity_helper( } } +#[test] +fn max_multiplicity() { + // regression test for https://github.com/EspressoSystems/jellyfish/issues/663 + + // play with these items + let num_storage_nodes = 6; + let recovery_threshold = 4; + let max_multiplicity = 1 << 10; // intentionally large so as to fit many payload sizes into a single polynomial + + // TODO add payload byte len 0 to this test + let payload_byte_lens = [1, 100, 1_000, 10_000, 100_000, 1_000_000]; + type E = Bls12_381; + + // more items as a function of the above + let (mut advz, payload_bytes) = advz_init_with::(AdvzParams { + recovery_threshold, + num_storage_nodes, + max_multiplicity, + payload_len: *payload_byte_lens.iter().max().unwrap(), + }); + let elem_byte_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); + let (mut found_small_payload, mut found_large_payload) = (false, false); + + for payload_byte_len in payload_byte_lens { + let payload = &payload_bytes[..payload_byte_len]; + let num_payload_elems = payload_byte_len.div_ceil(elem_byte_len) as u32; + + let disperse = advz.disperse(payload).unwrap(); + let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); + + // test: multiplicity set correctly + assert!( + common.multiplicity <= max_multiplicity, + "derived multiplicity should never exceed max_multiplicity" + ); + if num_payload_elems < max_multiplicity * recovery_threshold { + // small payload + found_small_payload = true; + assert!( + num_payload_elems <= common.multiplicity * advz.recovery_threshold, + "derived multiplicity too small" + ); + + // TODO TEMPORARY: multiplicity, recovery_threshold must be a power + // of 2 https://github.com/EspressoSystems/jellyfish/issues/668 + // + // After this issue is fixed the following test should use + // `common.multiplicity - 1` instead of `common.multiplicity / 2`. + assert!( + num_payload_elems > common.multiplicity / 2 * advz.recovery_threshold, + "derived multiplicity too large: payload_byte_len {}, common.multiplicity {}", + payload_byte_len, + common.multiplicity + ); + + assert_eq!( + common.poly_commits.len(), + 1, + "small payload should fit into a single polynomial" + ); + } else { + // large payload + found_large_payload = true; + assert_eq!( + common.multiplicity, max_multiplicity, + "derived multiplicity should equal max_multiplicity for large payload" + ); + } + assert!(found_large_payload && found_small_payload, "missing tests"); + + // sanity: verify shares + for share in shares { + advz.verify_share(&share, &common, &commit) + .unwrap() + .unwrap(); + } + } +} + struct AdvzParams { recovery_threshold: u32, num_storage_nodes: u32, @@ -381,7 +460,7 @@ pub(super) fn advz_init() -> (Advz, Vec) { advz_init_with(advz_params) } -fn advz_init_with(advz_params: AdvzParams) -> (Advz, Vec) { +fn advz_init_with(advz_params: AdvzParams) -> (Advz, Vec) { let mut rng = jf_utils::test_rng(); let poly_len = advz_params.recovery_threshold * advz_params.max_multiplicity; let srs = init_srs(poly_len as usize, &mut rng); From 17d55fba72b2cb4739fb417525c57ac41e9523e0 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 26 Aug 2024 13:06:56 -0400 Subject: [PATCH 05/40] fix min_multiplicity --- vid/src/advz.rs | 31 +++++++++++++++++++------------ vid/src/advz/test.rs | 5 +++-- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 5b3a449c5..6dc490a0b 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -183,6 +183,8 @@ where // https://github.com/EspressoSystems/jellyfish/issues/339 if chunk_size as usize != eval_domain.size() { return Err(VidError::Argument(format!( + // TODO GUS FIX THIS ERROR CHECK & MESSAGE + // TODO GUS EXPLAIN WHY WE ENFORCE POWER OF TWO "recovery_threshold {} currently unsupported, round to {} instead", chunk_size, eval_domain.size() @@ -858,7 +860,7 @@ where fn polynomial_internal( domain_ref: &Radix2EvaluationDomain>, chunk_size: usize, - coeffs: I, + coeffs: I, // TODO GUS rename to evals, they're not coefficients!! ) -> KzgPolynomial where I: Iterator, @@ -876,6 +878,7 @@ where // then we were not given the correct number of coeffs. In that case // coeffs.len() could be anything, so there's nothing to sanity check. if pre_fft_len == chunk_size { + // TODO GUS don't panic, return internal error instead assert_eq!(coeffs_vec.len(), pre_fft_len); } @@ -893,24 +896,28 @@ where fn min_multiplicity(&self, payload_byte_len: u32, multiplicity: u32) -> u32 { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>() as u32; - let elems = payload_byte_len / elem_bytes_len; + let elems = payload_byte_len.div_ceil(elem_bytes_len); let recovery_threshold = self.recovery_threshold; if recovery_threshold * multiplicity < elems { // payload is large. no change in multiplicity needed. return multiplicity; } - // payload is small: choose m such that 0 < m < multiplicity - // and m is the power of two that fits the payload best. - let mut m = multiplicity; - loop { - if elems <= recovery_threshold * m { - break; - } else { - m >> 1; - } + // payload is small: choose the smallest `m` such that `0 < m < + // multiplicity` and the entire payload fits into `m * + // recovery_threshold` elements. + let m = elems.div_ceil(recovery_threshold).max(1); + + // TODO TEMPORARY: multiplicity, recovery_threshold must be a power of 2 + // https://github.com/EspressoSystems/jellyfish/issues/668 + // + // Round up to the nearest power of 2. Delete this line after this issue + // is fixed. + if m <= 1 { + 1 + } else { + 1 << ((m - 1).ilog2() + 1) } - m } /// Derive a commitment from whatever data is needed. diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index 4dfcd7d6a..a397aef77 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -427,8 +427,6 @@ fn max_multiplicity() { "derived multiplicity should equal max_multiplicity for large payload" ); } - assert!(found_large_payload && found_small_payload, "missing tests"); - // sanity: verify shares for share in shares { advz.verify_share(&share, &common, &commit) @@ -436,6 +434,9 @@ fn max_multiplicity() { .unwrap(); } } + + assert!(found_large_payload, "missing test for large payload"); + assert!(found_small_payload, "missing test for small payload"); } struct AdvzParams { From 35e1510923a0a4457017be915f5124a821f4542c Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Mon, 26 Aug 2024 17:37:09 -0400 Subject: [PATCH 06/40] WIP polynomial_internal construct its own eval domain --- vid/src/advz.rs | 62 +++++++++++++++++++++++++++----------------- vid/src/advz/test.rs | 11 +++++--- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 6dc490a0b..36120cc01 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -845,23 +845,19 @@ where parallelizable_chunks(payload, chunk_size * elem_bytes_len) .map(|chunk| { - Self::polynomial_internal( - eval_domain_ref, - chunk_size, - bytes_to_field::<_, KzgEval>(chunk), - ) + Self::polynomial_internal(bytes_to_field::<_, KzgEval>(chunk), chunk_size) }) .collect() } + /// Return a polynomial that interpolates `evals` on a evaluation domain of + /// size `domain_size`. + /// + /// TODO: explain what happens when number of evals is not a power of 2. // This is an associated function, not a method, doesn't take in `self`, thus // more friendly to cross-thread `Sync`, especially when on of the generic // param of `Self` didn't implement `Sync` - fn polynomial_internal( - domain_ref: &Radix2EvaluationDomain>, - chunk_size: usize, - coeffs: I, // TODO GUS rename to evals, they're not coefficients!! - ) -> KzgPolynomial + fn polynomial_internal(evals: I, domain_size: usize) -> KzgPolynomial where I: Iterator, I::Item: Borrow>, @@ -869,28 +865,46 @@ where // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - let mut coeffs_vec: Vec<_> = coeffs.map(|c| *c.borrow()).collect(); - let pre_fft_len = coeffs_vec.len(); - EvaluationDomain::ifft_in_place(domain_ref, &mut coeffs_vec); - - // sanity check: the fft did not resize coeffs. - // If pre_fft_len != self.recovery_threshold * self.multiplicity - // then we were not given the correct number of coeffs. In that case - // coeffs.len() could be anything, so there's nothing to sanity check. - if pre_fft_len == chunk_size { - // TODO GUS don't panic, return internal error instead - assert_eq!(coeffs_vec.len(), pre_fft_len); + let mut evals_vec: Vec<_> = evals.map(|c| *c.borrow()).collect(); + let pre_fft_len = evals_vec.len(); + + // Check for too many evals + // TODO GUS don't panic, return internal error instead + assert!(pre_fft_len <= domain_size); + + let domain = Radix2EvaluationDomain::>::new(domain_size) + .expect("TODO return error instead"); + // .ok_or_else(|| { + // VidError::Internal(anyhow::anyhow!( + // "fail to construct domain of size {}", + // chunk_size + // )) + // })?; + + domain.ifft_in_place(&mut evals_vec); + + // sanity: the fft did not resize evals. If pre_fft_len < domain_size + // then we were given too few evals, in which caseso there's nothing to + // sanity check. + // + // TODO GUS don't panic, return internal error instead + if pre_fft_len == domain_size { + assert_eq!(evals_vec.len(), pre_fft_len); } - DenseUVPolynomial::from_coefficients_vec(coeffs_vec) + DenseUVPolynomial::from_coefficients_vec(evals_vec) } - fn polynomial(&self, coeffs: I, chunk_size: usize) -> KzgPolynomial + // TODO delete this method? + fn polynomial(&self, evals: I, multiplicity: usize) -> KzgPolynomial where I: Iterator, I::Item: Borrow>, { - Self::polynomial_internal(&self.eval_domain, chunk_size, coeffs) + Self::polynomial_internal( + evals, + usize::try_from(self.recovery_threshold).unwrap() * multiplicity, + ) } fn min_multiplicity(&self, payload_byte_len: u32, multiplicity: u32) -> u32 { diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index a397aef77..e71241ddc 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -368,8 +368,8 @@ fn max_multiplicity() { let recovery_threshold = 4; let max_multiplicity = 1 << 10; // intentionally large so as to fit many payload sizes into a single polynomial - // TODO add payload byte len 0 to this test - let payload_byte_lens = [1, 100, 1_000, 10_000, 100_000, 1_000_000]; + // let payload_byte_lens = [0, 1, 100, 1_000, 10_000, 100_000, 1_000_000]; + let payload_byte_lens = [1, 100, 1_000]; type E = Bls12_381; // more items as a function of the above @@ -427,6 +427,11 @@ fn max_multiplicity() { "derived multiplicity should equal max_multiplicity for large payload" ); } + + // sanity: recover payload + // let bytes_recovered = advz.recover_payload(&shares, &common).unwrap(); + // assert_eq!(bytes_recovered, payload); + // sanity: verify shares for share in shares { advz.verify_share(&share, &common, &commit) @@ -435,7 +440,7 @@ fn max_multiplicity() { } } - assert!(found_large_payload, "missing test for large payload"); + // assert!(found_large_payload, "missing test for large payload"); assert!(found_small_payload, "missing test for small payload"); } From f79e7c652509a8bfe5d64ccde447be2e4b0dd53c Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 27 Aug 2024 14:59:37 -0400 Subject: [PATCH 07/40] WIP --- vid/src/advz/payload_prover.rs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index f40a52432..3f030714a 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -25,7 +25,7 @@ use crate::{ }; use anyhow::anyhow; use ark_ec::pairing::Pairing; -use ark_poly::EvaluationDomain; +use ark_poly::{EvaluationDomain, Radix2EvaluationDomain}; use ark_serialize::{CanonicalDeserialize, CanonicalSerialize}; use ark_std::{format, ops::Range}; use itertools::Itertools; @@ -90,19 +90,21 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - // perf: we might not need all these points - let points: Vec<_> = self.eval_domain.elements().collect(); + let chunk_size = self.min_multiplicity(payload.len() as u32, self.max_multiplicity) + * self.recovery_threshold; // TODO tidy + let points: Vec<_> = Radix2EvaluationDomain::new(chunk_size as usize) + .expect("TODO return error instead") + .elements() + .collect(); // perf: we might not need all these points let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); - let chunk_size = - self.min_multiplicity(payload.len() as u32, self.recovery_threshold) as usize; for (i, evals_iter) in elems_iter .chunks(self.recovery_threshold as usize) .into_iter() .enumerate() { - let poly = self.polynomial(evals_iter, chunk_size); + let poly = self.polynomial(evals_iter, chunk_size as usize); let points_range = Range { // first polynomial? skip to the start of the proof range start: if i == 0 { offset_elem } else { 0 }, @@ -113,9 +115,11 @@ where points.len() }, }; + ark_std::println!("points.len {}", points.len()); proofs.extend( UnivariateKzgPCS::multi_open(&self.ck, &poly, &points[points_range]) - .map_err(vid)? + .expect("GUS WTF") + // .map_err(vid)? .0, ); } @@ -478,7 +482,7 @@ mod tests { }; let small_range_proof: SmallRangeProof<_> = - advz.payload_proof(&payload, range.clone()).unwrap(); + advz.payload_proof(&payload, range.clone()).unwrap(); // TODO this fails! advz.payload_verify(stmt.clone(), &small_range_proof) .unwrap() .unwrap(); From 2b6bc7d4ac4445461b273b15b964ffc8528ab412 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 27 Aug 2024 16:29:39 -0400 Subject: [PATCH 08/40] WIP --- vid/src/advz/payload_prover.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 3f030714a..e08d34af1 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -90,9 +90,8 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - let chunk_size = self.min_multiplicity(payload.len() as u32, self.max_multiplicity) - * self.recovery_threshold; // TODO tidy - let points: Vec<_> = Radix2EvaluationDomain::new(chunk_size as usize) + let multiplicity = self.min_multiplicity(payload.len() as u32, self.max_multiplicity); // TODO tidy + let points: Vec<_> = Radix2EvaluationDomain::new(multiplicity as usize) .expect("TODO return error instead") .elements() .collect(); // perf: we might not need all these points @@ -104,7 +103,7 @@ where .into_iter() .enumerate() { - let poly = self.polynomial(evals_iter, chunk_size as usize); + let poly = self.polynomial(evals_iter, multiplicity as usize); let points_range = Range { // first polynomial? skip to the start of the proof range start: if i == 0 { offset_elem } else { 0 }, @@ -164,8 +163,10 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - // perf: we might not need all these points - let points: Vec<_> = self.eval_domain.elements().collect(); + let points: Vec<_> = Radix2EvaluationDomain::new(stmt.common.multiplicity as usize) + .expect("TODO return error instead") + .elements() + .collect(); // perf: we might not need all these points // verify proof let mut cur_proof_index = 0; @@ -266,14 +267,13 @@ where .chain(proof.suffix_bytes.iter()), )) .chain(proof.suffix_elems.iter().cloned()); - let chunk_size = (stmt.common.multiplicity * self.recovery_threshold) as usize; // rebuild the poly commits, check against `common` for (commit_index, evals_iter) in range_poly.into_iter().zip( elems_iter .chunks(self.recovery_threshold as usize) .into_iter(), ) { - let poly = self.polynomial(evals_iter, chunk_size); + let poly = self.polynomial(evals_iter, stmt.common.multiplicity as usize); let poly_commit = UnivariateKzgPCS::commit(&self.ck, &poly).map_err(vid)?; if poly_commit != stmt.common.poly_commits[commit_index] { return Ok(Err(())); From 974a5613053aa5ee743a3ecf4f33aa79ba4134b0 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 27 Aug 2024 16:42:18 -0400 Subject: [PATCH 09/40] fix: correctness test now passes --- vid/src/advz/payload_prover.rs | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index e08d34af1..a76887b5a 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -91,10 +91,11 @@ where // prepare list of input points let multiplicity = self.min_multiplicity(payload.len() as u32, self.max_multiplicity); // TODO tidy - let points: Vec<_> = Radix2EvaluationDomain::new(multiplicity as usize) - .expect("TODO return error instead") - .elements() - .collect(); // perf: we might not need all these points + let points: Vec<_> = + Radix2EvaluationDomain::new((self.recovery_threshold * multiplicity) as usize) + .expect("TODO return error instead") + .elements() + .collect(); // perf: we might not need all these points let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); @@ -163,10 +164,12 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - let points: Vec<_> = Radix2EvaluationDomain::new(stmt.common.multiplicity as usize) - .expect("TODO return error instead") - .elements() - .collect(); // perf: we might not need all these points + let points: Vec<_> = Radix2EvaluationDomain::new( + (self.recovery_threshold * stmt.common.multiplicity) as usize, + ) + .expect("TODO return error instead") + .elements() + .collect(); // perf: we might not need all these points // verify proof let mut cur_proof_index = 0; From 972f5bd576d5118ded7f0274beeeceb603835cfd Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Tue, 27 Aug 2024 16:52:28 -0400 Subject: [PATCH 10/40] remove println --- vid/src/advz/payload_prover.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index a76887b5a..8288f8f8c 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -115,7 +115,6 @@ where points.len() }, }; - ark_std::println!("points.len {}", points.len()); proofs.extend( UnivariateKzgPCS::multi_open(&self.ck, &poly, &points[points_range]) .expect("GUS WTF") From 0a05da07c161e17028ec96c585bce0fa8dd763f4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 09:59:03 -0400 Subject: [PATCH 11/40] fix test --- vid/tests/vid/mod.rs | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/vid/tests/vid/mod.rs b/vid/tests/vid/mod.rs index cf8c3a48a..fb3fe7c55 100644 --- a/vid/tests/vid/mod.rs +++ b/vid/tests/vid/mod.rs @@ -8,28 +8,32 @@ use jf_vid::{VidError, VidResult, VidScheme}; /// Correctness test generic over anything that impls [`VidScheme`] /// -/// `pub` visibility, but it's not part of this crate's public API -/// because it's in an integration test. +/// TODO this test should not have a `max_multiplicities` arg. It is intended to +/// be generic over the [`VidScheme`] and a generic VID scheme does not have a +/// multiplicity arg. +/// +/// `pub` visibility, but it's not part of this crate's public +/// API because it's in an integration test. /// pub fn round_trip( vid_factory: impl Fn(u32, u32, u32) -> V, vid_sizes: &[(u32, u32)], - multiplicities: &[u32], + max_multiplicities: &[u32], payload_byte_lens: &[u32], rng: &mut R, ) where V: VidScheme, R: RngCore + CryptoRng, { - for (&mult, &(recovery_threshold, num_storage_nodes)) in - zip(multiplicities.iter().cycle(), vid_sizes) + for (&max_multiplicity, &(recovery_threshold, num_storage_nodes)) in + zip(max_multiplicities.iter().cycle(), vid_sizes) { - let mut vid = vid_factory(recovery_threshold, num_storage_nodes, mult); + let mut vid = vid_factory(recovery_threshold, num_storage_nodes, max_multiplicity); for &len in payload_byte_lens { println!( - "m: {} n: {} mult: {} byte_len: {}", - recovery_threshold, num_storage_nodes, mult, len + "m: {} n: {} byte_len: {} max_mult: {}", + recovery_threshold, num_storage_nodes, len, max_multiplicity ); let bytes_random = { @@ -43,7 +47,7 @@ pub fn round_trip( assert_eq!(shares.len(), num_storage_nodes as usize); assert_eq!(commit, vid.commit_only(&bytes_random).unwrap()); assert_eq!(len, V::get_payload_byte_len(&common)); - assert_eq!(mult, V::get_multiplicity(&common)); + assert!(V::get_multiplicity(&common) <= max_multiplicity); assert_eq!(num_storage_nodes, V::get_num_storage_nodes(&common)); for share in shares.iter() { From 96ec40af808ca4642444c8b0edd85a31fb09020f Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 10:27:45 -0400 Subject: [PATCH 12/40] fix recover_payload --- vid/src/advz.rs | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 36120cc01..72a1092d9 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -647,6 +647,7 @@ where } fn recover_payload(&self, shares: &[Self::Share], common: &Self::Common) -> VidResult> { + // check args if shares.len() < self.recovery_threshold as usize { return Err(VidError::Argument(format!( "not enough shares {}, expected at least {}", @@ -661,13 +662,12 @@ where ))); } - // all shares must have equal evals len + // check args: all shares must have equal evals len let num_evals = shares .first() .ok_or_else(|| VidError::Argument("shares is empty".into()))? .evals .len(); - let multiplicity = common.multiplicity; if let Some((index, share)) = shares .iter() .enumerate() @@ -681,26 +681,29 @@ where share.evals.len() ))); } - if num_evals != multiplicity as usize * common.poly_commits.len() { + if num_evals != common.multiplicity as usize * common.poly_commits.len() { return Err(VidError::Argument(format!( "num_evals should be (multiplicity * poly_commits): {} but is instead: {}", - multiplicity as usize * common.poly_commits.len(), + common.multiplicity as usize * common.poly_commits.len(), num_evals, ))); } - let chunk_size = multiplicity * self.recovery_threshold; - let num_polys = num_evals / multiplicity as usize; + // convenience quantities + let chunk_size = common.multiplicity * self.recovery_threshold; + let num_polys = common.poly_commits.len() as usize; let elems_capacity = num_polys * chunk_size as usize; - let mut elems = Vec::with_capacity(elems_capacity); + let fft_domain = Radix2EvaluationDomain::>::new(chunk_size as usize) + .expect("TODO return error instead"); + let mut elems = Vec::with_capacity(elems_capacity); let mut evals = Vec::with_capacity(num_evals); for p in 0..num_polys { for share in shares { // extract all evaluations for polynomial p from the share - for m in 0..multiplicity as usize { + for m in 0..common.multiplicity as usize { evals.push(( - (share.index * multiplicity) as usize + m, + (share.index * common.multiplicity) as usize + m, share.evals[(m * num_polys) + p], )) } @@ -715,11 +718,12 @@ where // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - self.eval_domain.fft_in_place(&mut coeffs); + // TODO GUS fix this comment + fft_domain.fft_in_place(&mut coeffs); elems.append(&mut coeffs); } - assert_eq!(elems.len(), elems_capacity); + assert_eq!(elems.len(), elems_capacity); // 4 vs 2 let mut payload: Vec<_> = field_to_bytes(elems).collect(); payload.truncate(common.payload_byte_len.try_into().map_err(vid)?); From ccf81b25fd0e8cb457afab46f0784c0e28e25fdb Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 10:43:26 -0400 Subject: [PATCH 13/40] delete field Advz::eval_domain (yay) --- vid/src/advz.rs | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 72a1092d9..f1a2f644f 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -88,11 +88,6 @@ where vk: KzgVerifierParam, multi_open_domain: Radix2EvaluationDomain>, - // TODO might be able to eliminate this field and instead use - // `EvaluationDomain::reindex_by_subdomain()` on `multi_open_domain` - // but that method consumes `other` and its doc is unclear. - eval_domain: Radix2EvaluationDomain>, - // tuple of // - reference to the SRS/ProverParam loaded to GPU // - cuda stream handle @@ -154,9 +149,16 @@ where ))); } + // TODO TEMPORARY: enforce power-of-2 + // https://github.com/EspressoSystems/jellyfish/issues/668 + if !recovery_threshold.is_power_of_two() { + return Err(VidError::Argument(format!( + "recovery_threshold {recovery_threshold} should be a power of two" + ))); + } if !max_multiplicity.is_power_of_two() { return Err(VidError::Argument(format!( - "max multiplicity {max_multiplicity} should be a power of two" + "max_multiplicity {max_multiplicity} should be a power of two" ))); } @@ -171,25 +173,6 @@ where code_word_size as usize, ) .map_err(vid)?; - let eval_domain = Radix2EvaluationDomain::new(chunk_size as usize).ok_or_else(|| { - VidError::Internal(anyhow::anyhow!( - "fail to construct domain of size {}", - chunk_size - )) - })?; - - // TODO TEMPORARY: enforce power-of-2 chunk size - // Remove this restriction after we get KZG in eval form - // https://github.com/EspressoSystems/jellyfish/issues/339 - if chunk_size as usize != eval_domain.size() { - return Err(VidError::Argument(format!( - // TODO GUS FIX THIS ERROR CHECK & MESSAGE - // TODO GUS EXPLAIN WHY WE ENFORCE POWER OF TWO - "recovery_threshold {} currently unsupported, round to {} instead", - chunk_size, - eval_domain.size() - ))); - } Ok(Self { recovery_threshold, @@ -198,7 +181,6 @@ where ck, vk, multi_open_domain, - eval_domain, srs_on_gpu_and_cuda_stream: None, _pd: Default::default(), }) @@ -695,6 +677,12 @@ where let elems_capacity = num_polys * chunk_size as usize; let fft_domain = Radix2EvaluationDomain::>::new(chunk_size as usize) .expect("TODO return error instead"); + // let eval_domain = Radix2EvaluationDomain::new(chunk_size as + // usize).ok_or_else(|| { VidError::Internal(anyhow::anyhow!( + // "fail to construct domain of size {}", + // chunk_size + // )) + // })?; let mut elems = Vec::with_capacity(elems_capacity); let mut evals = Vec::with_capacity(num_evals); @@ -845,7 +833,6 @@ where E: Pairing, { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); - let eval_domain_ref = &self.eval_domain; parallelizable_chunks(payload, chunk_size * elem_bytes_len) .map(|chunk| { From 959f044b0a19b1fd96f60aa92a4dd3817f8077bd Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 12:30:01 -0400 Subject: [PATCH 14/40] fix test max_multiplicity --- vid/src/advz/test.rs | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index e71241ddc..dee9acd8b 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -366,10 +366,9 @@ fn max_multiplicity() { // play with these items let num_storage_nodes = 6; let recovery_threshold = 4; - let max_multiplicity = 1 << 10; // intentionally large so as to fit many payload sizes into a single polynomial + let max_multiplicity = 1 << 5; // intentionally large so as to fit many payload sizes into a single polynomial - // let payload_byte_lens = [0, 1, 100, 1_000, 10_000, 100_000, 1_000_000]; - let payload_byte_lens = [1, 100, 1_000]; + let payload_byte_lens = [0, 1, 100, 10_000]; type E = Bls12_381; // more items as a function of the above @@ -402,21 +401,28 @@ fn max_multiplicity() { "derived multiplicity too small" ); - // TODO TEMPORARY: multiplicity, recovery_threshold must be a power - // of 2 https://github.com/EspressoSystems/jellyfish/issues/668 - // - // After this issue is fixed the following test should use - // `common.multiplicity - 1` instead of `common.multiplicity / 2`. - assert!( - num_payload_elems > common.multiplicity / 2 * advz.recovery_threshold, - "derived multiplicity too large: payload_byte_len {}, common.multiplicity {}", - payload_byte_len, - common.multiplicity - ); + if num_payload_elems > 0 { + // TODO TEMPORARY: enforce power-of-2 + // https://github.com/EspressoSystems/jellyfish/issues/668 + // + // After this issue is fixed the following test should use + // `common.multiplicity - 1` instead of `common.multiplicity / 2`. + assert!( + num_payload_elems > common.multiplicity / 2 * advz.recovery_threshold, + "derived multiplicity too large: payload_byte_len {}, common.multiplicity {}", + payload_byte_len, + common.multiplicity + ); + } else { + assert_eq!( + common.multiplicity, 1, + "zero-length payload should have multiplicity 1, found {}", + common.multiplicity + ); + } - assert_eq!( - common.poly_commits.len(), - 1, + assert!( + common.poly_commits.len() <= 1, "small payload should fit into a single polynomial" ); } else { @@ -440,7 +446,7 @@ fn max_multiplicity() { } } - // assert!(found_large_payload, "missing test for large payload"); + assert!(found_large_payload, "missing test for large payload"); assert!(found_small_payload, "missing test for small payload"); } From deaf009e4e1f18b2795d07a821685b62e0b8b1f4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 13:49:04 -0400 Subject: [PATCH 15/40] remove unneeded arg from min_multiplicity --- vid/src/advz.rs | 31 +++++++++++++++++-------------- vid/src/advz/payload_prover.rs | 2 +- vid/src/advz/precomputable.rs | 7 +++---- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index f1a2f644f..ba26723d9 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -404,7 +404,7 @@ where { let payload = payload.as_ref(); let payload_byte_len = payload.len().try_into().map_err(vid)?; - let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let multiplicity = self.min_multiplicity(payload_byte_len); let chunk_size = multiplicity * self.recovery_threshold; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let polys = self.bytes_to_polys(payload, chunk_size as usize); @@ -427,7 +427,7 @@ where "VID disperse {} payload bytes to {} nodes", payload_byte_len, self.num_storage_nodes )); - let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let multiplicity = self.min_multiplicity(payload.len()); let chunk_size = multiplicity * self.recovery_threshold; let code_word_size = multiplicity * self.num_storage_nodes; @@ -518,7 +518,7 @@ where common.num_storage_nodes, self.num_storage_nodes ))); } - let multiplicity = self.min_multiplicity(common.payload_byte_len, self.max_multiplicity); + let multiplicity = self.min_multiplicity(common.payload_byte_len.try_into().map_err(vid)?); if common.multiplicity != multiplicity { return Err(VidError::Argument(format!( "common multiplicity {} differs from derived min {}", @@ -898,26 +898,29 @@ where ) } - fn min_multiplicity(&self, payload_byte_len: u32, multiplicity: u32) -> u32 { - let elem_bytes_len = - bytes_to_field::elem_byte_capacity::<::ScalarField>() as u32; - let elems = payload_byte_len.div_ceil(elem_bytes_len); - let recovery_threshold = self.recovery_threshold; - if recovery_threshold * multiplicity < elems { + fn min_multiplicity(&self, payload_byte_len: usize) -> u32 { + let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); + let elems: u32 = payload_byte_len + .div_ceil(elem_bytes_len) + .try_into() + .unwrap(); + if self.recovery_threshold * self.max_multiplicity < elems { // payload is large. no change in multiplicity needed. - return multiplicity; + return self.max_multiplicity; } // payload is small: choose the smallest `m` such that `0 < m < // multiplicity` and the entire payload fits into `m * // recovery_threshold` elements. - let m = elems.div_ceil(recovery_threshold).max(1); + let m = elems.div_ceil(self.recovery_threshold).max(1); - // TODO TEMPORARY: multiplicity, recovery_threshold must be a power of 2 + // TODO TEMPORARY: enforce power-of-2 // https://github.com/EspressoSystems/jellyfish/issues/668 // - // Round up to the nearest power of 2. Delete this line after this issue - // is fixed. + // Round up to the nearest power of 2. + // + // After the above issue is fixed: delete the following code and return + // `m` from above.ß if m <= 1 { 1 } else { diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 8288f8f8c..d83661e6c 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -90,7 +90,7 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - let multiplicity = self.min_multiplicity(payload.len() as u32, self.max_multiplicity); // TODO tidy + let multiplicity = self.min_multiplicity(payload.len()); let points: Vec<_> = Radix2EvaluationDomain::new((self.recovery_threshold * multiplicity) as usize) .expect("TODO return error instead") diff --git a/vid/src/advz/precomputable.rs b/vid/src/advz/precomputable.rs index 9823c453d..df6b9ad15 100644 --- a/vid/src/advz/precomputable.rs +++ b/vid/src/advz/precomputable.rs @@ -39,8 +39,7 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let payload_byte_len = payload.len().try_into().map_err(vid)?; - let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let multiplicity = self.min_multiplicity(payload.len()); let chunk_size = (multiplicity * self.recovery_threshold) as usize; let polys = self.bytes_to_polys(payload, chunk_size); let poly_commits: Vec> = @@ -60,13 +59,12 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let payload_byte_len = payload.len().try_into().map_err(vid)?; let disperse_time = start_timer!(|| ark_std::format!( "(PRECOMPUTE): VID disperse {} payload bytes to {} nodes", payload_byte_len, self.num_storage_nodes )); - let multiplicity = self.min_multiplicity(payload_byte_len, self.max_multiplicity); + let multiplicity = self.min_multiplicity(payload.len()); let chunk_size = multiplicity * self.recovery_threshold; let code_word_size = multiplicity * self.num_storage_nodes; @@ -97,6 +95,7 @@ where "(PRECOMPUTE): compute {} KZG commitments", polys.len() )); + let payload_byte_len = payload.len().try_into().map_err(vid)?; let common = Common { poly_commits: data.poly_commits.clone(), all_evals_digest: all_evals_commit.commitment().digest(), From 8d7039e834cac32f0c51685df174bd44cc086517 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 13:58:03 -0400 Subject: [PATCH 16/40] remove unneeded arg from bytes_to_polys --- vid/src/advz.rs | 15 +++++++-------- vid/src/advz/precomputable.rs | 5 ++--- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index ba26723d9..ed6024666 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -407,7 +407,7 @@ where let multiplicity = self.min_multiplicity(payload_byte_len); let chunk_size = multiplicity * self.recovery_threshold; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload, chunk_size as usize); + let polys = self.bytes_to_polys(payload); end_timer!(bytes_to_polys_time); let poly_commits_time = start_timer!(|| "batch poly commit"); @@ -433,7 +433,7 @@ where // partition payload into polynomial coefficients let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload, chunk_size as usize); + let polys = self.bytes_to_polys(payload); end_timer!(bytes_to_polys_time); // evaluate polynomials @@ -824,15 +824,14 @@ where Ok(PrimeField::from_le_bytes_mod_order(&hasher.finalize())) } - fn bytes_to_polys( - &self, - payload: &[u8], - chunk_size: usize, - ) -> Vec::ScalarField>> + fn bytes_to_polys(&self, payload: &[u8]) -> Vec::ScalarField>> where E: Pairing, { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); + let chunk_size = + usize::try_from(self.min_multiplicity(payload.len()) * self.recovery_threshold) + .unwrap(); parallelizable_chunks(payload, chunk_size * elem_bytes_len) .map(|chunk| { @@ -920,7 +919,7 @@ where // Round up to the nearest power of 2. // // After the above issue is fixed: delete the following code and return - // `m` from above.ß + // `m` from above. if m <= 1 { 1 } else { diff --git a/vid/src/advz/precomputable.rs b/vid/src/advz/precomputable.rs index df6b9ad15..67306c921 100644 --- a/vid/src/advz/precomputable.rs +++ b/vid/src/advz/precomputable.rs @@ -40,8 +40,7 @@ where { let payload = payload.as_ref(); let multiplicity = self.min_multiplicity(payload.len()); - let chunk_size = (multiplicity * self.recovery_threshold) as usize; - let polys = self.bytes_to_polys(payload, chunk_size); + let polys = self.bytes_to_polys(payload); let poly_commits: Vec> = UnivariateKzgPCS::batch_commit(&self.ck, &polys).map_err(vid)?; Ok(( @@ -71,7 +70,7 @@ where // partition payload into polynomial coefficients // and count `elems_len` for later let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload, chunk_size as usize); + let polys = self.bytes_to_polys(payload); end_timer!(bytes_to_polys_time); // evaluate polynomials From 56d25747791a5f65b7fd0648ab3a80762642ca89 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 14:13:10 -0400 Subject: [PATCH 17/40] tidy bytes_to_polys --- vid/src/advz.rs | 6 +++--- vid/src/advz/payload_prover.rs | 2 ++ 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index ed6024666..bbf8ca82c 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -829,13 +829,13 @@ where E: Pairing, { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); - let chunk_size = + let domain_size = usize::try_from(self.min_multiplicity(payload.len()) * self.recovery_threshold) .unwrap(); - parallelizable_chunks(payload, chunk_size * elem_bytes_len) + parallelizable_chunks(payload, domain_size * elem_bytes_len) .map(|chunk| { - Self::polynomial_internal(bytes_to_field::<_, KzgEval>(chunk), chunk_size) + Self::polynomial_internal(bytes_to_field::<_, KzgEval>(chunk), domain_size) }) .collect() } diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index d83661e6c..565d5f8eb 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -91,6 +91,8 @@ where // prepare list of input points let multiplicity = self.min_multiplicity(payload.len()); + + // TODO GUS delete `points`!!! let points: Vec<_> = Radix2EvaluationDomain::new((self.recovery_threshold * multiplicity) as usize) .expect("TODO return error instead") From 7eb66a980c488c9ee445c7150d175aa3855a9ca0 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 14:44:30 -0400 Subject: [PATCH 18/40] refactor disperse, disperse_precompute into disperse_with_polys_and_commits --- vid/src/advz.rs | 173 ++++++++++++++++++---------------- vid/src/advz/precomputable.rs | 87 +---------------- 2 files changed, 94 insertions(+), 166 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index bbf8ca82c..c4344e884 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -422,87 +422,10 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let payload_byte_len = payload.len().try_into().map_err(vid)?; - let disperse_time = start_timer!(|| format!( - "VID disperse {} payload bytes to {} nodes", - payload_byte_len, self.num_storage_nodes - )); - let multiplicity = self.min_multiplicity(payload.len()); - let chunk_size = multiplicity * self.recovery_threshold; - let code_word_size = multiplicity * self.num_storage_nodes; - - // partition payload into polynomial coefficients - let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let polys = self.bytes_to_polys(payload); - end_timer!(bytes_to_polys_time); - - // evaluate polynomials - let all_storage_node_evals_timer = start_timer!(|| format!( - "compute all storage node evals for {} polynomials with {} coefficients", - polys.len(), - _chunk_size - )); - let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; - end_timer!(all_storage_node_evals_timer); - - // vector commitment to polynomial evaluations - let all_evals_commit_timer = - start_timer!(|| "compute merkle root of all storage node evals"); - let all_evals_commit = - KzgEvalsMerkleTree::::from_elems(None, &all_storage_node_evals).map_err(vid)?; - end_timer!(all_evals_commit_timer); - - let common_timer = start_timer!(|| format!("compute {} KZG commitments", polys.len())); - let common = Common { - poly_commits: >::kzg_batch_commit(self, &polys)?, - all_evals_digest: all_evals_commit.commitment().digest(), - payload_byte_len, - num_storage_nodes: self.num_storage_nodes, - multiplicity, - }; - end_timer!(common_timer); - - let commit = Self::derive_commit( - &common.poly_commits, - payload_byte_len, - self.num_storage_nodes, - )?; - let pseudorandom_scalar = Self::pseudorandom_scalar(&common, &commit)?; - - // Compute aggregate polynomial as a pseudorandom linear combo of polynomial via - // evaluation of the polynomial whose coefficients are polynomials and whose - // input point is the pseudorandom scalar. - let aggregate_poly = - polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); - - let agg_proofs_timer = start_timer!(|| format!( - "compute aggregate proofs for {} storage nodes", - self.num_storage_nodes - )); - let aggregate_proofs = UnivariateKzgPCS::multi_open_rou_proofs( - &self.ck, - &aggregate_poly, - code_word_size as usize, - &self.multi_open_domain, - ) - .map_err(vid)?; - end_timer!(agg_proofs_timer); - - let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); - let shares = self.assemble_shares( - all_storage_node_evals, - aggregate_proofs, - all_evals_commit, - multiplicity, - )?; - end_timer!(assemblage_timer); - end_timer!(disperse_time); + let poly_commits = >::kzg_batch_commit(self, &polys)?; - Ok(VidDisperse { - shares, - common, - commit, - }) + self.disperse_with_polys_and_commits(payload, polys, poly_commits) } fn verify_share( @@ -753,6 +676,90 @@ where SrsRef: Sync, AdvzInternal: MaybeGPU, { + fn disperse_with_polys_and_commits( + &self, + payload: &[u8], + polys: Vec::ScalarField>>, + poly_commits: Vec>, + ) -> VidResult> { + let payload_byte_len = payload.len().try_into().map_err(vid)?; + let disperse_time = start_timer!(|| format!( + "VID disperse {} payload bytes to {} nodes", + payload_byte_len, self.num_storage_nodes + )); + let multiplicity = self.min_multiplicity(payload.len()); + let chunk_size = multiplicity * self.recovery_threshold; + let code_word_size = multiplicity * self.num_storage_nodes; + + // evaluate polynomials + let all_storage_node_evals_timer = start_timer!(|| format!( + "compute all storage node evals for {} polynomials with {} coefficients", + polys.len(), + _chunk_size + )); + let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; + end_timer!(all_storage_node_evals_timer); + + // vector commitment to polynomial evaluations + let all_evals_commit_timer = + start_timer!(|| "compute merkle root of all storage node evals"); + let all_evals_commit = + KzgEvalsMerkleTree::::from_elems(None, &all_storage_node_evals).map_err(vid)?; + end_timer!(all_evals_commit_timer); + + let common_timer = start_timer!(|| format!("compute {} KZG commitments", polys.len())); + let common = Common { + poly_commits, + all_evals_digest: all_evals_commit.commitment().digest(), + payload_byte_len, + num_storage_nodes: self.num_storage_nodes, + multiplicity, + }; + end_timer!(common_timer); + + let commit = Self::derive_commit( + &common.poly_commits, + payload_byte_len, + self.num_storage_nodes, + )?; + let pseudorandom_scalar = Self::pseudorandom_scalar(&common, &commit)?; + + // Compute aggregate polynomial as a pseudorandom linear combo of polynomial via + // evaluation of the polynomial whose coefficients are polynomials and whose + // input point is the pseudorandom scalar. + let aggregate_poly = + polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); + + let agg_proofs_timer = start_timer!(|| format!( + "compute aggregate proofs for {} storage nodes", + self.num_storage_nodes + )); + let aggregate_proofs = UnivariateKzgPCS::multi_open_rou_proofs( + &self.ck, + &aggregate_poly, + code_word_size as usize, + &self.multi_open_domain, + ) + .map_err(vid)?; + end_timer!(agg_proofs_timer); + + let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); + let shares = self.assemble_shares( + all_storage_node_evals, + aggregate_proofs, + all_evals_commit, + multiplicity, + )?; + end_timer!(assemblage_timer); + end_timer!(disperse_time); + + Ok(VidDisperse { + shares, + common, + commit, + }) + } + fn evaluate_polys( &self, polys: &[DensePolynomial<::ScalarField>], @@ -824,6 +831,7 @@ where Ok(PrimeField::from_le_bytes_mod_order(&hasher.finalize())) } + /// Partition payload into polynomial coefficients fn bytes_to_polys(&self, payload: &[u8]) -> Vec::ScalarField>> where E: Pairing, @@ -833,11 +841,14 @@ where usize::try_from(self.min_multiplicity(payload.len()) * self.recovery_threshold) .unwrap(); - parallelizable_chunks(payload, domain_size * elem_bytes_len) + let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); + let result = parallelizable_chunks(payload, domain_size * elem_bytes_len) .map(|chunk| { Self::polynomial_internal(bytes_to_field::<_, KzgEval>(chunk), domain_size) }) - .collect() + .collect(); + end_timer!(bytes_to_polys_time); + result } /// Return a polynomial that interpolates `evals` on a evaluation domain of diff --git a/vid/src/advz/precomputable.rs b/vid/src/advz/precomputable.rs index 67306c921..4941d5774 100644 --- a/vid/src/advz/precomputable.rs +++ b/vid/src/advz/precomputable.rs @@ -58,93 +58,10 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let disperse_time = start_timer!(|| ark_std::format!( - "(PRECOMPUTE): VID disperse {} payload bytes to {} nodes", - payload_byte_len, - self.num_storage_nodes - )); - let multiplicity = self.min_multiplicity(payload.len()); - let chunk_size = multiplicity * self.recovery_threshold; - let code_word_size = multiplicity * self.num_storage_nodes; - - // partition payload into polynomial coefficients - // and count `elems_len` for later - let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let polys = self.bytes_to_polys(payload); - end_timer!(bytes_to_polys_time); - - // evaluate polynomials - let all_storage_node_evals_timer = start_timer!(|| ark_std::format!( - "compute all storage node evals for {} polynomials with {} coefficients", - polys.len(), - chunk_size - )); - let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; - end_timer!(all_storage_node_evals_timer); - - // vector commitment to polynomial evaluations - // TODO why do I need to compute the height of the merkle tree? - let all_evals_commit_timer = - start_timer!(|| "compute merkle root of all storage node evals"); - let all_evals_commit = - KzgEvalsMerkleTree::::from_elems(None, &all_storage_node_evals).map_err(vid)?; - end_timer!(all_evals_commit_timer); - - let common_timer = start_timer!(|| ark_std::format!( - "(PRECOMPUTE): compute {} KZG commitments", - polys.len() - )); - let payload_byte_len = payload.len().try_into().map_err(vid)?; - let common = Common { - poly_commits: data.poly_commits.clone(), - all_evals_digest: all_evals_commit.commitment().digest(), - payload_byte_len, - num_storage_nodes: self.num_storage_nodes, - multiplicity, - }; - end_timer!(common_timer); - - let commit = Self::derive_commit( - &common.poly_commits, - payload_byte_len, - self.num_storage_nodes, - )?; - let pseudorandom_scalar = Self::pseudorandom_scalar(&common, &commit)?; - - // Compute aggregate polynomial as a pseudorandom linear combo of polynomial via - // evaluation of the polynomial whose coefficients are polynomials and whose - // input point is the pseudorandom scalar. - let aggregate_poly = - polynomial_eval(polys.iter().map(PolynomialMultiplier), pseudorandom_scalar); - - let agg_proofs_timer = start_timer!(|| ark_std::format!( - "compute aggregate proofs for {} storage nodes", - self.num_storage_nodes - )); - let aggregate_proofs = UnivariateKzgPCS::multi_open_rou_proofs( - &self.ck, - &aggregate_poly, - code_word_size as usize, - &self.multi_open_domain, - ) - .map_err(vid)?; - end_timer!(agg_proofs_timer); - - let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); - let shares = self.assemble_shares( - all_storage_node_evals, - aggregate_proofs, - all_evals_commit, - multiplicity, - )?; - end_timer!(assemblage_timer); - end_timer!(disperse_time); + let poly_commits = data.poly_commits.clone(); - Ok(VidDisperse { - shares, - common, - commit, - }) + self.disperse_with_polys_and_commits(payload, polys, poly_commits) } } From fc5349d7c6b56cd40ff4fe0fb8a6ec3d77e4fae4 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 15:01:35 -0400 Subject: [PATCH 19/40] move code from evaluate_polys, assemble_shares into disperse_with_polys_and_commits --- vid/src/advz.rs | 174 +++++++++++++++++++----------------------------- 1 file changed, 70 insertions(+), 104 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index c4344e884..610a5410e 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -688,16 +688,49 @@ where payload_byte_len, self.num_storage_nodes )); let multiplicity = self.min_multiplicity(payload.len()); - let chunk_size = multiplicity * self.recovery_threshold; - let code_word_size = multiplicity * self.num_storage_nodes; + let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).unwrap(); // evaluate polynomials let all_storage_node_evals_timer = start_timer!(|| format!( "compute all storage node evals for {} polynomials with {} coefficients", polys.len(), - _chunk_size + multiplicity * self.recovery_threshold )); - let all_storage_node_evals = self.evaluate_polys(&polys, code_word_size as usize)?; + let all_storage_node_evals = { + let mut all_storage_node_evals = vec![Vec::with_capacity(polys.len()); code_word_size]; + // this is to avoid `SrsRef` not implementing `Sync` problem, + // instead of sending entire `self` cross thread, we only send a ref which is + // Sync + let multi_open_domain_ref = &self.multi_open_domain; + + let all_poly_evals = parallelizable_slice_iter(&polys) + .map(|poly| { + UnivariateKzgPCS::::multi_open_rou_evals( + poly, + code_word_size, + multi_open_domain_ref, + ) + .map_err(vid) + }) + .collect::, VidError>>()?; + + for poly_evals in all_poly_evals { + for (storage_node_evals, poly_eval) in all_storage_node_evals + .iter_mut() + .zip(poly_evals.into_iter()) + { + storage_node_evals.push(poly_eval); + } + } + + // sanity checks + assert_eq!(all_storage_node_evals.len(), code_word_size); + for storage_node_evals in all_storage_node_evals.iter() { + assert_eq!(storage_node_evals.len(), polys.len()); + } + + all_storage_node_evals + }; end_timer!(all_storage_node_evals_timer); // vector commitment to polynomial evaluations @@ -744,12 +777,39 @@ where end_timer!(agg_proofs_timer); let assemblage_timer = start_timer!(|| "assemble shares for dispersal"); - let shares = self.assemble_shares( - all_storage_node_evals, - aggregate_proofs, - all_evals_commit, - multiplicity, - )?; + let shares: Vec<_> = { + // compute share data + let share_data = all_storage_node_evals + .into_iter() + .zip(aggregate_proofs) + .enumerate() + .map(|(i, (eval, proof))| { + let eval_proof = all_evals_commit + .lookup(KzgEvalsMerkleTreeIndex::::from(i as u64)) + .expect_ok() + .map_err(vid)? + .1; + Ok((eval, proof, eval_proof)) + }) + .collect::, VidError>>()?; + + // split share data into chunks of size multiplicity + share_data + .into_iter() + .chunks(multiplicity as usize) + .into_iter() + .enumerate() + .map(|(index, chunk)| { + let (evals, proofs, eval_proofs): (Vec<_>, _, _) = chunk.multiunzip(); + Share { + index: index as u32, + evals: evals.into_iter().flatten().collect::>(), + aggregate_proofs: proofs, + eval_proofs, + } + }) + .collect() + }; end_timer!(assemblage_timer); end_timer!(disperse_time); @@ -760,50 +820,6 @@ where }) } - fn evaluate_polys( - &self, - polys: &[DensePolynomial<::ScalarField>], - code_word_size: usize, - ) -> Result::ScalarField>>, VidError> - where - E: Pairing, - H: HasherDigest, - { - let mut all_storage_node_evals = vec![Vec::with_capacity(polys.len()); code_word_size]; - // this is to avoid `SrsRef` not implementing `Sync` problem, - // instead of sending entire `self` cross thread, we only send a ref which is - // Sync - let multi_open_domain_ref = &self.multi_open_domain; - - let all_poly_evals = parallelizable_slice_iter(polys) - .map(|poly| { - UnivariateKzgPCS::::multi_open_rou_evals( - poly, - code_word_size, - multi_open_domain_ref, - ) - .map_err(vid) - }) - .collect::, VidError>>()?; - - for poly_evals in all_poly_evals { - for (storage_node_evals, poly_eval) in all_storage_node_evals - .iter_mut() - .zip(poly_evals.into_iter()) - { - storage_node_evals.push(poly_eval); - } - } - - // sanity checks - assert_eq!(all_storage_node_evals.len(), code_word_size); - for storage_node_evals in all_storage_node_evals.iter() { - assert_eq!(storage_node_evals.len(), polys.len()); - } - - Ok(all_storage_node_evals) - } - fn pseudorandom_scalar( common: &::Common, commit: &::Commit, @@ -970,56 +986,6 @@ where } Ok(hasher.finalize().into()) } - - /// Assemble shares from evaluations and proofs. - /// - /// Each share contains (for multiplicity m): - /// 1. (m * num_poly) evaluations. - /// 2. a collection of m KZG proofs. TODO KZG aggregation https://github.com/EspressoSystems/jellyfish/issues/356 - /// 3. m merkle tree membership proofs. - fn assemble_shares( - &self, - all_storage_node_evals: Vec::ScalarField>>, - aggregate_proofs: Vec>, - all_evals_commit: KzgEvalsMerkleTree, - multiplicity: u32, - ) -> Result>, VidError> - where - E: Pairing, - H: HasherDigest, - { - // compute share data - let share_data = all_storage_node_evals - .into_iter() - .zip(aggregate_proofs) - .enumerate() - .map(|(i, (eval, proof))| { - let eval_proof = all_evals_commit - .lookup(KzgEvalsMerkleTreeIndex::::from(i as u64)) - .expect_ok() - .map_err(vid)? - .1; - Ok((eval, proof, eval_proof)) - }) - .collect::, VidError>>()?; - - // split share data into chunks of size multiplicity - Ok(share_data - .into_iter() - .chunks(multiplicity as usize) - .into_iter() - .enumerate() - .map(|(index, chunk)| { - let (evals, proofs, eval_proofs): (Vec<_>, _, _) = chunk.multiunzip(); - Share { - index: index as u32, - evals: evals.into_iter().flatten().collect::>(), - aggregate_proofs: proofs, - eval_proofs, - } - }) - .collect()) - } } /// Evaluate a generalized polynomial at a given point using Horner's method. From a2459bcafc78338f515fdde0e26eff19f95f2370 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 15:08:47 -0400 Subject: [PATCH 20/40] delete method Advz::polynomial --- vid/src/advz.rs | 12 ------------ vid/src/advz/payload_prover.rs | 10 ++++++++-- 2 files changed, 8 insertions(+), 14 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 610a5410e..f8336c960 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -912,18 +912,6 @@ where DenseUVPolynomial::from_coefficients_vec(evals_vec) } - // TODO delete this method? - fn polynomial(&self, evals: I, multiplicity: usize) -> KzgPolynomial - where - I: Iterator, - I::Item: Borrow>, - { - Self::polynomial_internal( - evals, - usize::try_from(self.recovery_threshold).unwrap() * multiplicity, - ) - } - fn min_multiplicity(&self, payload_byte_len: usize) -> u32 { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); let elems: u32 = payload_byte_len diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 565d5f8eb..5471a0041 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -106,7 +106,10 @@ where .into_iter() .enumerate() { - let poly = self.polynomial(evals_iter, multiplicity as usize); + let poly = Self::polynomial_internal( + evals_iter, + (self.recovery_threshold * multiplicity) as usize, + ); let points_range = Range { // first polynomial? skip to the start of the proof range start: if i == 0 { offset_elem } else { 0 }, @@ -277,7 +280,10 @@ where .chunks(self.recovery_threshold as usize) .into_iter(), ) { - let poly = self.polynomial(evals_iter, stmt.common.multiplicity as usize); + let poly = Self::polynomial_internal( + evals_iter, + (stmt.common.multiplicity * self.recovery_threshold) as usize, + ); let poly_commit = UnivariateKzgPCS::commit(&self.ck, &poly).map_err(vid)?; if poly_commit != stmt.common.poly_commits[commit_index] { return Ok(Err(())); From 19ee6ef801004f5fa64477a1ff671a8a1a50bd06 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 15:35:48 -0400 Subject: [PATCH 21/40] rename polynomial_internal -> interpolate_polynomial, return VidResult, tidy --- vid/src/advz.rs | 74 ++++++++++++++++++++-------------- vid/src/advz/payload_prover.rs | 8 ++-- vid/src/advz/precomputable.rs | 4 +- 3 files changed, 49 insertions(+), 37 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index f8336c960..adc54055a 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -407,7 +407,7 @@ where let multiplicity = self.min_multiplicity(payload_byte_len); let chunk_size = multiplicity * self.recovery_threshold; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload)?; end_timer!(bytes_to_polys_time); let poly_commits_time = start_timer!(|| "batch poly commit"); @@ -422,7 +422,7 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload)?; let poly_commits = >::kzg_batch_commit(self, &polys)?; self.disperse_with_polys_and_commits(payload, polys, poly_commits) @@ -848,7 +848,10 @@ where } /// Partition payload into polynomial coefficients - fn bytes_to_polys(&self, payload: &[u8]) -> Vec::ScalarField>> + fn bytes_to_polys( + &self, + payload: &[u8], + ) -> VidResult::ScalarField>>> where E: Pairing, { @@ -860,21 +863,30 @@ where let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let result = parallelizable_chunks(payload, domain_size * elem_bytes_len) .map(|chunk| { - Self::polynomial_internal(bytes_to_field::<_, KzgEval>(chunk), domain_size) + Self::interpolate_polynomial(bytes_to_field::<_, KzgEval>(chunk), domain_size) }) - .collect(); + .collect::>>(); end_timer!(bytes_to_polys_time); result } - /// Return a polynomial that interpolates `evals` on a evaluation domain of - /// size `domain_size`. + /// Consume `evals` and return a polynomial that interpolates `evals` on a + /// evaluation domain of size `domain_size`. + /// + /// Return an error if the length of `evals` exceeds `domain_size`. + /// + /// The degree-plus-1 of the returned polynomial is always a power of two + /// because: + /// + /// - We use FFT to interpolate, so `domain_size` is rounded up to the next + /// power of two. + /// - [`KzgPolynomial`] implementation is stored in coefficient form. /// - /// TODO: explain what happens when number of evals is not a power of 2. - // This is an associated function, not a method, doesn't take in `self`, thus - // more friendly to cross-thread `Sync`, especially when on of the generic - // param of `Self` didn't implement `Sync` - fn polynomial_internal(evals: I, domain_size: usize) -> KzgPolynomial + /// See https://github.com/EspressoSystems/jellyfish/issues/339 + /// + /// Why is this method an associated function of `Self`? Because we want to + /// use a generic parameter of `Self`. + fn interpolate_polynomial(evals: I, domain_size: usize) -> VidResult> where I: Iterator, I::Item: Borrow>, @@ -884,32 +896,32 @@ where // https://github.com/EspressoSystems/jellyfish/issues/339 let mut evals_vec: Vec<_> = evals.map(|c| *c.borrow()).collect(); let pre_fft_len = evals_vec.len(); - - // Check for too many evals - // TODO GUS don't panic, return internal error instead - assert!(pre_fft_len <= domain_size); - - let domain = Radix2EvaluationDomain::>::new(domain_size) - .expect("TODO return error instead"); - // .ok_or_else(|| { - // VidError::Internal(anyhow::anyhow!( - // "fail to construct domain of size {}", - // chunk_size - // )) - // })?; + if pre_fft_len > domain_size { + return Err(VidError::Internal(anyhow::anyhow!( + "number of evals {} exceeds domain_size {}", + pre_fft_len, + domain_size + ))); + } + let domain = Radix2EvaluationDomain::>::new(domain_size).ok_or_else(|| { + VidError::Internal(anyhow::anyhow!( + "fail to construct domain of size {domain_size}", + )) + })?; domain.ifft_in_place(&mut evals_vec); // sanity: the fft did not resize evals. If pre_fft_len < domain_size - // then we were given too few evals, in which caseso there's nothing to + // then we were given too few evals, in which case there's nothing to // sanity check. - // - // TODO GUS don't panic, return internal error instead - if pre_fft_len == domain_size { - assert_eq!(evals_vec.len(), pre_fft_len); + if pre_fft_len == domain_size && pre_fft_len != evals_vec.len() { + return Err(VidError::Internal(anyhow::anyhow!( + "unexpected output resize from {pre_fft_len} to {}", + evals_vec.len() + ))); } - DenseUVPolynomial::from_coefficients_vec(evals_vec) + Ok(DenseUVPolynomial::from_coefficients_vec(evals_vec)) } fn min_multiplicity(&self, payload_byte_len: usize) -> u32 { diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 5471a0041..59275a154 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -106,10 +106,10 @@ where .into_iter() .enumerate() { - let poly = Self::polynomial_internal( + let poly = Self::interpolate_polynomial( evals_iter, (self.recovery_threshold * multiplicity) as usize, - ); + )?; let points_range = Range { // first polynomial? skip to the start of the proof range start: if i == 0 { offset_elem } else { 0 }, @@ -280,10 +280,10 @@ where .chunks(self.recovery_threshold as usize) .into_iter(), ) { - let poly = Self::polynomial_internal( + let poly = Self::interpolate_polynomial( evals_iter, (stmt.common.multiplicity * self.recovery_threshold) as usize, - ); + )?; let poly_commit = UnivariateKzgPCS::commit(&self.ck, &poly).map_err(vid)?; if poly_commit != stmt.common.poly_commits[commit_index] { return Ok(Err(())); diff --git a/vid/src/advz/precomputable.rs b/vid/src/advz/precomputable.rs index 4941d5774..777bcf50a 100644 --- a/vid/src/advz/precomputable.rs +++ b/vid/src/advz/precomputable.rs @@ -40,7 +40,7 @@ where { let payload = payload.as_ref(); let multiplicity = self.min_multiplicity(payload.len()); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload)?; let poly_commits: Vec> = UnivariateKzgPCS::batch_commit(&self.ck, &polys).map_err(vid)?; Ok(( @@ -58,7 +58,7 @@ where B: AsRef<[u8]>, { let payload = payload.as_ref(); - let polys = self.bytes_to_polys(payload); + let polys = self.bytes_to_polys(payload)?; let poly_commits = data.poly_commits.clone(); self.disperse_with_polys_and_commits(payload, polys, poly_commits) From ac78bd170246f9f25ccd0f8c96fdbb65a4905459 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 16:09:15 -0400 Subject: [PATCH 22/40] delete field Advz::multi_open_domain, derive it on-the-fly from common.multiplicity (yay) --- vid/src/advz.rs | 62 +++++++++++++++++++++----------------------- vid/src/advz/test.rs | 3 ++- 2 files changed, 31 insertions(+), 34 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index adc54055a..43847003a 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -86,7 +86,6 @@ where max_multiplicity: u32, ck: KzgProverParam, vk: KzgVerifierParam, - multi_open_domain: Radix2EvaluationDomain>, // tuple of // - reference to the SRS/ProverParam loaded to GPU @@ -162,15 +161,9 @@ where ))); } - // erasure code params - let chunk_size = max_multiplicity * recovery_threshold; // message length m - let code_word_size = max_multiplicity * num_storage_nodes; // code word length n - let poly_degree = chunk_size - 1; - - let (ck, vk) = UnivariateKzgPCS::trim_fft_size(srs, poly_degree as usize).map_err(vid)?; - let multi_open_domain = UnivariateKzgPCS::::multi_open_rou_eval_domain( - poly_degree as usize, - code_word_size as usize, + let (ck, vk) = UnivariateKzgPCS::trim_fft_size( + srs, + usize::try_from(max_multiplicity * recovery_threshold - 1).unwrap(), ) .map_err(vid)?; @@ -180,7 +173,6 @@ where max_multiplicity, ck, vk, - multi_open_domain, srs_on_gpu_and_cuda_stream: None, _pd: Default::default(), }) @@ -506,6 +498,7 @@ where // feature. let multiplicities = Vec::from_iter((0..multiplicity as usize)); let polys_len = common.poly_commits.len(); + let multi_open_domain = self.multi_open_domain(multiplicity)?; let verification_iter = parallelizable_slice_iter(&multiplicities).map(|i| { let range = i * polys_len..(i + 1) * polys_len; let aggregate_eval = polynomial_eval( @@ -526,9 +519,7 @@ where Ok(UnivariateKzgPCS::verify( &self.vk, &aggregate_poly_commit, - &self - .multi_open_domain - .element((share.index * multiplicity) as usize + i), + &multi_open_domain.element((share.index * multiplicity) as usize + i), &aggregate_eval, &share.aggregate_proofs[*i], ) @@ -595,17 +586,16 @@ where } // convenience quantities - let chunk_size = common.multiplicity * self.recovery_threshold; - let num_polys = common.poly_commits.len() as usize; - let elems_capacity = num_polys * chunk_size as usize; - let fft_domain = Radix2EvaluationDomain::>::new(chunk_size as usize) - .expect("TODO return error instead"); - // let eval_domain = Radix2EvaluationDomain::new(chunk_size as - // usize).ok_or_else(|| { VidError::Internal(anyhow::anyhow!( - // "fail to construct domain of size {}", - // chunk_size - // )) - // })?; + let chunk_size = usize::try_from(common.multiplicity * self.recovery_threshold).unwrap(); + let num_polys = common.poly_commits.len(); + let elems_capacity = num_polys * chunk_size; + let fft_domain = + Radix2EvaluationDomain::>::new(chunk_size).ok_or_else(|| { + VidError::Internal(anyhow::anyhow!( + "fail to construct domain of size {}", + chunk_size + )) + })?; let mut elems = Vec::with_capacity(elems_capacity); let mut evals = Vec::with_capacity(num_evals); @@ -622,7 +612,7 @@ where let mut coeffs = reed_solomon_erasure_decode_rou( mem::take(&mut evals), chunk_size as usize, - &self.multi_open_domain, + &self.multi_open_domain(common.multiplicity)?, ) .map_err(vid)?; @@ -689,6 +679,7 @@ where )); let multiplicity = self.min_multiplicity(payload.len()); let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).unwrap(); + let multi_open_domain = self.multi_open_domain(multiplicity)?; // evaluate polynomials let all_storage_node_evals_timer = start_timer!(|| format!( @@ -698,17 +689,12 @@ where )); let all_storage_node_evals = { let mut all_storage_node_evals = vec![Vec::with_capacity(polys.len()); code_word_size]; - // this is to avoid `SrsRef` not implementing `Sync` problem, - // instead of sending entire `self` cross thread, we only send a ref which is - // Sync - let multi_open_domain_ref = &self.multi_open_domain; - let all_poly_evals = parallelizable_slice_iter(&polys) .map(|poly| { UnivariateKzgPCS::::multi_open_rou_evals( poly, code_word_size, - multi_open_domain_ref, + &multi_open_domain, ) .map_err(vid) }) @@ -771,7 +757,7 @@ where &self.ck, &aggregate_poly, code_word_size as usize, - &self.multi_open_domain, + &multi_open_domain, ) .map_err(vid)?; end_timer!(agg_proofs_timer); @@ -986,6 +972,16 @@ where } Ok(hasher.finalize().into()) } + + fn multi_open_domain( + &self, + multiplicity: u32, + ) -> VidResult::ScalarField>> { + let chunk_size = usize::try_from(multiplicity * self.recovery_threshold).unwrap(); + let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).unwrap(); + UnivariateKzgPCS::::multi_open_rou_eval_domain(chunk_size - 1, code_word_size) + .map_err(vid) + } } /// Evaluate a generalized polynomial at a given point using Horner's method. diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index dee9acd8b..08d9f862b 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -235,8 +235,9 @@ fn sad_path_recover_payload_corrupt_shares() { // corrupted index, out of bounds { let mut shares_bad_indices = shares.clone(); + let multi_open_domain_size = advz.multi_open_domain(common.multiplicity).unwrap().size(); for i in 0..shares_bad_indices.len() { - shares_bad_indices[i].index += u32::try_from(advz.multi_open_domain.size()).unwrap(); + shares_bad_indices[i].index += u32::try_from(multi_open_domain_size).unwrap(); advz.recover_payload(&shares_bad_indices, &common) .expect_err("recover_payload should fail when indices are out of bounds"); } From b7fab02456b0e5943fa61a14a3633e4ad35bb8cc Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 17:47:28 -0400 Subject: [PATCH 23/40] min_multiplicity return VidResult, replace panic with error --- vid/src/advz.rs | 38 +++++++++++++++++----------------- vid/src/advz/payload_prover.rs | 2 +- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 43847003a..2f8dc5f3f 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -163,7 +163,7 @@ where let (ck, vk) = UnivariateKzgPCS::trim_fft_size( srs, - usize::try_from(max_multiplicity * recovery_threshold - 1).unwrap(), + usize::try_from(max_multiplicity * recovery_threshold - 1).map_err(vid)?, ) .map_err(vid)?; @@ -269,7 +269,7 @@ where self.ck.powers_of_g.len() - 1, ) .map_err(vid)?; - self.srs_on_gpu_and_cuda_stream = Some((srs_on_gpu, warmup_new_stream().unwrap())); + self.srs_on_gpu_and_cuda_stream = Some((srs_on_gpu, warmup_new_stream().map_err(vid)?)); Ok(()) } } @@ -364,12 +364,10 @@ where &mut self, polys: &[DensePolynomial], ) -> VidResult>> { - // let mut srs_on_gpu = self.srs_on_gpu_and_cuda_stream.as_mut().unwrap().0; - // let stream = &self.srs_on_gpu_and_cuda_stream.as_ref().unwrap().1; if polys.is_empty() { return Ok(vec![]); } - let (srs_on_gpu, stream) = self.srs_on_gpu_and_cuda_stream.as_mut().unwrap(); // safe by construction + let (srs_on_gpu, stream) = self.srs_on_gpu_and_cuda_stream.as_mut().map_err(vid)?; // safe by construction as GPUCommittable>::gpu_batch_commit_with_loaded_prover_param( srs_on_gpu, polys, stream, ) @@ -396,7 +394,7 @@ where { let payload = payload.as_ref(); let payload_byte_len = payload.len().try_into().map_err(vid)?; - let multiplicity = self.min_multiplicity(payload_byte_len); + let multiplicity = self.min_multiplicity(payload_byte_len)?; let chunk_size = multiplicity * self.recovery_threshold; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let polys = self.bytes_to_polys(payload)?; @@ -433,7 +431,8 @@ where common.num_storage_nodes, self.num_storage_nodes ))); } - let multiplicity = self.min_multiplicity(common.payload_byte_len.try_into().map_err(vid)?); + let multiplicity = + self.min_multiplicity(common.payload_byte_len.try_into().map_err(vid)?)?; if common.multiplicity != multiplicity { return Err(VidError::Argument(format!( "common multiplicity {} differs from derived min {}", @@ -586,7 +585,8 @@ where } // convenience quantities - let chunk_size = usize::try_from(common.multiplicity * self.recovery_threshold).unwrap(); + let chunk_size = + usize::try_from(common.multiplicity * self.recovery_threshold).map_err(vid)?; let num_polys = common.poly_commits.len(); let elems_capacity = num_polys * chunk_size; let fft_domain = @@ -677,8 +677,8 @@ where "VID disperse {} payload bytes to {} nodes", payload_byte_len, self.num_storage_nodes )); - let multiplicity = self.min_multiplicity(payload.len()); - let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).unwrap(); + let multiplicity = self.min_multiplicity(payload.len())?; + let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).map_err(vid)?; let multi_open_domain = self.multi_open_domain(multiplicity)?; // evaluate polynomials @@ -843,8 +843,8 @@ where { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); let domain_size = - usize::try_from(self.min_multiplicity(payload.len()) * self.recovery_threshold) - .unwrap(); + usize::try_from(self.min_multiplicity(payload.len())? * self.recovery_threshold) + .map_err(vid)?; let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let result = parallelizable_chunks(payload, domain_size * elem_bytes_len) @@ -910,15 +910,15 @@ where Ok(DenseUVPolynomial::from_coefficients_vec(evals_vec)) } - fn min_multiplicity(&self, payload_byte_len: usize) -> u32 { + fn min_multiplicity(&self, payload_byte_len: usize) -> VidResult { let elem_bytes_len = bytes_to_field::elem_byte_capacity::<::ScalarField>(); let elems: u32 = payload_byte_len .div_ceil(elem_bytes_len) .try_into() - .unwrap(); + .map_err(vid)?; if self.recovery_threshold * self.max_multiplicity < elems { // payload is large. no change in multiplicity needed. - return self.max_multiplicity; + return Ok(self.max_multiplicity); } // payload is small: choose the smallest `m` such that `0 < m < @@ -934,9 +934,9 @@ where // After the above issue is fixed: delete the following code and return // `m` from above. if m <= 1 { - 1 + Ok(1) } else { - 1 << ((m - 1).ilog2() + 1) + Ok(1 << ((m - 1).ilog2() + 1)) } } @@ -977,8 +977,8 @@ where &self, multiplicity: u32, ) -> VidResult::ScalarField>> { - let chunk_size = usize::try_from(multiplicity * self.recovery_threshold).unwrap(); - let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).unwrap(); + let chunk_size = usize::try_from(multiplicity * self.recovery_threshold).map_err(vid)?; + let code_word_size = usize::try_from(multiplicity * self.num_storage_nodes).map_err(vid)?; UnivariateKzgPCS::::multi_open_rou_eval_domain(chunk_size - 1, code_word_size) .map_err(vid) } diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 59275a154..f9d67da52 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -90,7 +90,7 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - let multiplicity = self.min_multiplicity(payload.len()); + let multiplicity = self.min_multiplicity(payload.len())?; // TODO GUS delete `points`!!! let points: Vec<_> = From 2d72d727ea99fc58e3431f457a395734186a8e7a Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 18:06:49 -0400 Subject: [PATCH 24/40] refactor eval_domain --- vid/src/advz.rs | 24 ++++++++++++------------ vid/src/advz/payload_prover.rs | 19 ++++++++----------- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 2f8dc5f3f..dbadbc26b 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -589,13 +589,7 @@ where usize::try_from(common.multiplicity * self.recovery_threshold).map_err(vid)?; let num_polys = common.poly_commits.len(); let elems_capacity = num_polys * chunk_size; - let fft_domain = - Radix2EvaluationDomain::>::new(chunk_size).ok_or_else(|| { - VidError::Internal(anyhow::anyhow!( - "fail to construct domain of size {}", - chunk_size - )) - })?; + let fft_domain = Self::eval_domain(chunk_size)?; let mut elems = Vec::with_capacity(elems_capacity); let mut evals = Vec::with_capacity(num_evals); @@ -889,11 +883,7 @@ where domain_size ))); } - let domain = Radix2EvaluationDomain::>::new(domain_size).ok_or_else(|| { - VidError::Internal(anyhow::anyhow!( - "fail to construct domain of size {domain_size}", - )) - })?; + let domain = Self::eval_domain(domain_size)?; domain.ifft_in_place(&mut evals_vec); @@ -982,6 +972,16 @@ where UnivariateKzgPCS::::multi_open_rou_eval_domain(chunk_size - 1, code_word_size) .map_err(vid) } + + fn eval_domain( + domain_size: usize, + ) -> VidResult::ScalarField>> { + Radix2EvaluationDomain::>::new(domain_size).ok_or_else(|| { + VidError::Internal(anyhow::anyhow!( + "fail to construct domain of size {domain_size}" + )) + }) + } } /// Evaluate a generalized polynomial at a given point using Horner's method. diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index f9d67da52..a492f5c55 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -91,13 +91,11 @@ where // prepare list of input points let multiplicity = self.min_multiplicity(payload.len())?; - - // TODO GUS delete `points`!!! - let points: Vec<_> = - Radix2EvaluationDomain::new((self.recovery_threshold * multiplicity) as usize) - .expect("TODO return error instead") - .elements() - .collect(); // perf: we might not need all these points + let points: Vec<_> = Self::eval_domain( + usize::try_from(self.recovery_threshold * multiplicity).map_err(vid)?, + )? + .elements() + .collect(); // perf: we might not need all these points let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); @@ -168,10 +166,9 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem); // prepare list of input points - let points: Vec<_> = Radix2EvaluationDomain::new( - (self.recovery_threshold * stmt.common.multiplicity) as usize, - ) - .expect("TODO return error instead") + let points: Vec<_> = Self::eval_domain( + usize::try_from(self.recovery_threshold * stmt.common.multiplicity).map_err(vid)?, + )? .elements() .collect(); // perf: we might not need all these points From 7ee986b4f60c96ef4c98d6b74e2d47fd90f10682 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 18:11:25 -0400 Subject: [PATCH 25/40] remove stupid comment --- vid/src/advz.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index dbadbc26b..33e053f51 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -613,7 +613,6 @@ where // TODO TEMPORARY: use FFT to encode polynomials in eval form // Remove these FFTs after we get KZG in eval form // https://github.com/EspressoSystems/jellyfish/issues/339 - // TODO GUS fix this comment fft_domain.fft_in_place(&mut coeffs); elems.append(&mut coeffs); From 5ca5f55e1377e31d9456f8fba495e645b8efda00 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Wed, 28 Aug 2024 18:17:29 -0400 Subject: [PATCH 26/40] remove more stupid things --- vid/src/advz/payload_prover.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index a492f5c55..e2e860c2f 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -120,8 +120,7 @@ where }; proofs.extend( UnivariateKzgPCS::multi_open(&self.ck, &poly, &points[points_range]) - .expect("GUS WTF") - // .map_err(vid)? + .map_err(vid)? .0, ); } @@ -489,7 +488,7 @@ mod tests { }; let small_range_proof: SmallRangeProof<_> = - advz.payload_proof(&payload, range.clone()).unwrap(); // TODO this fails! + advz.payload_proof(&payload, range.clone()).unwrap(); advz.payload_verify(stmt.clone(), &small_range_proof) .unwrap() .unwrap(); From 71d334d1ab2a2d2c494014b762601b7ec0cc132d Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 14:37:47 -0400 Subject: [PATCH 27/40] modify test to allow nontrivial multiplicity --- vid/src/advz/payload_prover.rs | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index e2e860c2f..ee38734d8 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -415,17 +415,24 @@ mod tests { H: HasherDigest, { // play with these items - let (recovery_threshold, num_storage_nodes) = (4, 6); + let (recovery_threshold, num_storage_nodes, max_multiplicity) = (4, 6, 1); let num_polys = 3; let num_random_cases = 20; // more items as a function of the above - let payload_elems_len = num_polys * recovery_threshold as usize; + let poly_elems_len = recovery_threshold as usize * max_multiplicity as usize; + let payload_elems_len = num_polys * poly_elems_len; + let poly_bytes_len = poly_elems_len * elem_byte_capacity::(); let payload_bytes_base_len = payload_elems_len * elem_byte_capacity::(); - let poly_bytes_len = recovery_threshold as usize * elem_byte_capacity::(); let mut rng = jf_utils::test_rng(); let srs = init_srs(payload_elems_len, &mut rng); - let mut advz = Advz::::new(num_storage_nodes, recovery_threshold, srs).unwrap(); + let mut advz = Advz::::with_multiplicity( + num_storage_nodes, + recovery_threshold, + max_multiplicity, + srs, + ) + .unwrap(); // TEST: different payload byte lengths let payload_byte_len_noise_cases = vec![0, poly_bytes_len / 2, poly_bytes_len - 1]; @@ -456,9 +463,15 @@ mod tests { }; let all_cases = [(edge_cases, "edge"), (random_cases, "rand")]; + // at least one test case should have nontrivial multiplicity + let mut nontrivial_multiplicity = false; + for payload_len_case in payload_len_cases { let payload = init_random_payload(payload_len_case, &mut rng); let d = advz.disperse(&payload).unwrap(); + if d.common.multiplicity > 1 { + nontrivial_multiplicity = true; + } println!("payload byte len case: {}", payload.len()); for cases in all_cases.iter() { @@ -525,6 +538,12 @@ mod tests { } } + // assert!( + // nontrivial_multiplicity, + // "at least one payload size should use multiplicity > 1" + // ); + println!("nontrivial multiplicity? {nontrivial_multiplicity}"); + fn make_edge_cases(min: usize, max: usize) -> Vec> { vec![ Range { From 9c099418d909b986caf6be87cff1c810d597e7d9 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 14:39:29 -0400 Subject: [PATCH 28/40] test use nontrivial multiplicity, fails --- vid/src/advz/payload_prover.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index ee38734d8..ea5686ba2 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -415,7 +415,7 @@ mod tests { H: HasherDigest, { // play with these items - let (recovery_threshold, num_storage_nodes, max_multiplicity) = (4, 6, 1); + let (recovery_threshold, num_storage_nodes, max_multiplicity) = (4, 6, 2); let num_polys = 3; let num_random_cases = 20; @@ -538,11 +538,10 @@ mod tests { } } - // assert!( - // nontrivial_multiplicity, - // "at least one payload size should use multiplicity > 1" - // ); - println!("nontrivial multiplicity? {nontrivial_multiplicity}"); + assert!( + nontrivial_multiplicity, + "at least one payload size should use multiplicity > 1" + ); fn make_edge_cases(min: usize, max: usize) -> Vec> { vec![ From 02edecd28cc031b163409a3197896a0029996c5e Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:12:11 -0400 Subject: [PATCH 29/40] fix payload prover with nontrivial multiplicity --- vid/src/advz/payload_prover.rs | 74 ++++++++++++++++++++++------------ 1 file changed, 49 insertions(+), 25 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index ea5686ba2..01ba7a443 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -81,16 +81,18 @@ where check_range_nonempty_and_in_bounds(payload.len(), &range)?; // index conversion + let multiplicity = self.min_multiplicity(payload.len())?; let range_elem = self.range_byte_to_elem(&range); - let range_poly = self.range_elem_to_poly(&range_elem); + let range_poly = self.range_elem_to_poly(&range_elem, multiplicity); let range_elem_byte = self.range_elem_to_byte_clamped(&range_elem, payload.len()); - let range_poly_byte = self.range_poly_to_byte_clamped(&range_poly, payload.len()); - let offset_elem = self.offset_poly_to_elem(range_poly.start, range_elem.start); + let range_poly_byte = + self.range_poly_to_byte_clamped(&range_poly, payload.len(), multiplicity); + let offset_elem = + self.offset_poly_to_elem(range_poly.start, range_elem.start, multiplicity); let final_points_range_end = - self.final_poly_points_range_end(range_elem.len(), offset_elem); + self.final_poly_points_range_end(range_elem.len(), offset_elem, multiplicity); // prepare list of input points - let multiplicity = self.min_multiplicity(payload.len())?; let points: Vec<_> = Self::eval_domain( usize::try_from(self.recovery_threshold * multiplicity).map_err(vid)?, )? @@ -100,7 +102,7 @@ where let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); for (i, evals_iter) in elems_iter - .chunks(self.recovery_threshold as usize) + .chunks((self.recovery_threshold * multiplicity) as usize) .into_iter() .enumerate() { @@ -159,10 +161,14 @@ where // index conversion let range_elem = self.range_byte_to_elem(&stmt.range); - let range_poly = self.range_elem_to_poly(&range_elem); - let offset_elem = self.offset_poly_to_elem(range_poly.start, range_elem.start); - let final_points_range_end = - self.final_poly_points_range_end(range_elem.len(), offset_elem); + let range_poly = self.range_elem_to_poly(&range_elem, stmt.common.multiplicity); + let offset_elem = + self.offset_poly_to_elem(range_poly.start, range_elem.start, stmt.common.multiplicity); + let final_points_range_end = self.final_poly_points_range_end( + range_elem.len(), + offset_elem, + stmt.common.multiplicity, + ); // prepare list of input points let points: Vec<_> = Self::eval_domain( @@ -228,11 +234,14 @@ where check_range_nonempty_and_in_bounds(payload.len(), &range)?; // index conversion + let multiplicity = self.min_multiplicity(payload.len())?; let range_elem = self.range_byte_to_elem(&range); - let range_poly = self.range_elem_to_poly(&range_elem); + let range_poly = self.range_elem_to_poly(&range_elem, multiplicity); let range_elem_byte = self.range_elem_to_byte_clamped(&range_elem, payload.len()); - let range_poly_byte = self.range_poly_to_byte_clamped(&range_poly, payload.len()); - let offset_elem = self.offset_poly_to_elem(range_poly.start, range_elem.start); + let range_poly_byte = + self.range_poly_to_byte_clamped(&range_poly, payload.len(), multiplicity); + let offset_elem = + self.offset_poly_to_elem(range_poly.start, range_elem.start, multiplicity); // compute the prefix and suffix elems let mut elems_iter = bytes_to_field::<_, KzgEval>(payload[range_poly_byte].iter()); @@ -255,7 +264,7 @@ where Self::check_stmt_consistency(&stmt)?; // index conversion - let range_poly = self.range_byte_to_poly(&stmt.range); + let range_poly = self.range_byte_to_poly(&stmt.range, stmt.common.multiplicity); // rebuild the needed payload elements from statement and proof let elems_iter = proof @@ -273,7 +282,7 @@ where // rebuild the poly commits, check against `common` for (commit_index, evals_iter) in range_poly.into_iter().zip( elems_iter - .chunks(self.recovery_threshold as usize) + .chunks((self.recovery_threshold * stmt.common.multiplicity) as usize) .into_iter(), ) { let poly = Self::interpolate_polynomial( @@ -307,34 +316,49 @@ where ..result } } - fn range_elem_to_poly(&self, range: &Range) -> Range { - range_coarsen(range, self.recovery_threshold as usize) + fn range_elem_to_poly(&self, range: &Range, multiplicity: u32) -> Range { + range_coarsen(range, (self.recovery_threshold * multiplicity) as usize) } - fn range_byte_to_poly(&self, range: &Range) -> Range { + fn range_byte_to_poly(&self, range: &Range, multiplicity: u32) -> Range { range_coarsen( range, - self.recovery_threshold as usize * elem_byte_capacity::>(), + (self.recovery_threshold * multiplicity) as usize * elem_byte_capacity::>(), ) } - fn range_poly_to_byte_clamped(&self, range: &Range, len: usize) -> Range { + fn range_poly_to_byte_clamped( + &self, + range: &Range, + len: usize, + multiplicity: u32, + ) -> Range { let result = range_refine( range, - self.recovery_threshold as usize * elem_byte_capacity::>(), + (self.recovery_threshold * multiplicity) as usize * elem_byte_capacity::>(), ); Range { end: ark_std::cmp::min(result.end, len), ..result } } - fn offset_poly_to_elem(&self, range_poly_start: usize, range_elem_start: usize) -> usize { + fn offset_poly_to_elem( + &self, + range_poly_start: usize, + range_elem_start: usize, + multiplicity: u32, + ) -> usize { let start_poly_byte = index_refine( range_poly_start, - self.recovery_threshold as usize * elem_byte_capacity::>(), + (self.recovery_threshold * multiplicity) as usize * elem_byte_capacity::>(), ); range_elem_start - index_coarsen(start_poly_byte, elem_byte_capacity::>()) } - fn final_poly_points_range_end(&self, range_elem_len: usize, offset_elem: usize) -> usize { - (range_elem_len + offset_elem - 1) % self.recovery_threshold as usize + 1 + fn final_poly_points_range_end( + &self, + range_elem_len: usize, + offset_elem: usize, + multiplicity: u32, + ) -> usize { + (range_elem_len + offset_elem - 1) % (self.recovery_threshold * multiplicity) as usize + 1 } fn check_stmt_consistency(stmt: &Statement) -> VidResult<()> { From 1199557481f5779355a15e1b769737adfda5cb57 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:16:56 -0400 Subject: [PATCH 30/40] tests use Bn254 instead of Bls12_381 as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1736228179 --- vid/src/advz/test.rs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index 08d9f862b..8be881d24 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -1,5 +1,4 @@ use super::{VidError::Argument, *}; -use ark_bls12_381::Bls12_381; use ark_bn254::Bn254; use ark_std::{ rand::{CryptoRng, RngCore}, @@ -136,7 +135,7 @@ fn sad_path_verify_share_corrupt_commit() { // 1 corrupt commit, poly_commit let common_1_poly_corruption = { let mut corrupted = common.clone(); - corrupted.poly_commits[0] = ::G1Affine::zero().into(); + corrupted.poly_commits[0] = ::G1Affine::zero().into(); corrupted }; assert_arg_err( @@ -252,7 +251,7 @@ fn verify_share_with_multiplicity() { max_multiplicity: 4, payload_len: 4000, }; - let (mut advz, payload) = advz_init_with::(advz_params); + let (mut advz, payload) = advz_init_with::(advz_params); let disperse = advz.disperse(payload).unwrap(); let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); @@ -271,7 +270,7 @@ fn sad_path_verify_share_with_multiplicity() { max_multiplicity: 32, // payload fitting into a single polynomial payload_len: 8200, }; - let (mut advz, payload) = advz_init_with::(advz_params); + let (mut advz, payload) = advz_init_with::(advz_params); let disperse = advz.disperse(payload).unwrap(); let (shares, common, commit) = (disperse.shares, disperse.common, disperse.commit); @@ -370,7 +369,7 @@ fn max_multiplicity() { let max_multiplicity = 1 << 5; // intentionally large so as to fit many payload sizes into a single polynomial let payload_byte_lens = [0, 1, 100, 10_000]; - type E = Bls12_381; + type E = Bn254; // more items as a function of the above let (mut advz, payload_bytes) = advz_init_with::(AdvzParams { @@ -463,7 +462,7 @@ struct AdvzParams { /// Returns the following tuple: /// 1. An initialized [`Advz`] instance. /// 2. A `Vec` filled with random bytes. -pub(super) fn advz_init() -> (Advz, Vec) { +pub(super) fn advz_init() -> (Advz, Vec) { let advz_params = AdvzParams { recovery_threshold: 16, num_storage_nodes: 20, From db98c03c31314c202f9a279f308cbf4ecefc1abe Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:20:41 -0400 Subject: [PATCH 31/40] uncomment some test code as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1736335690 --- vid/src/advz/test.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vid/src/advz/test.rs b/vid/src/advz/test.rs index 8be881d24..48a8a7c8b 100644 --- a/vid/src/advz/test.rs +++ b/vid/src/advz/test.rs @@ -435,8 +435,8 @@ fn max_multiplicity() { } // sanity: recover payload - // let bytes_recovered = advz.recover_payload(&shares, &common).unwrap(); - // assert_eq!(bytes_recovered, payload); + let bytes_recovered = advz.recover_payload(&shares, &common).unwrap(); + assert_eq!(bytes_recovered, payload); // sanity: verify shares for share in shares { From eb0fd1a0dee5153fb2d73aba22ae9c883f6b1c1b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:23:19 -0400 Subject: [PATCH 32/40] address https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1736349890 --- vid/src/advz.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 33e053f51..cb2b1899c 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -161,11 +161,9 @@ where ))); } - let (ck, vk) = UnivariateKzgPCS::trim_fft_size( - srs, - usize::try_from(max_multiplicity * recovery_threshold - 1).map_err(vid)?, - ) - .map_err(vid)?; + let supported_degree = + usize::try_from(max_multiplicity * recovery_threshold - 1).map_err(vid)?; + let (ck, vk) = UnivariateKzgPCS::trim_fft_size(srs, supported_degree).map_err(vid)?; Ok(Self { recovery_threshold, From 8c866544ccd2a22306a4c8f4d42ad18e9939621b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:25:24 -0400 Subject: [PATCH 33/40] address https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1736358667 --- vid/src/advz.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index cb2b1899c..0566e9af3 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -194,7 +194,7 @@ where /// # Errors /// Return [`VidError::Argument`] if /// - `num_storage_nodes < recovery_threshold` - /// - TEMPORARY `recovery_threshold` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/339) + /// - TEMPORARY `recovery_threshold` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/668) pub fn new( num_storage_nodes: u32, recovery_threshold: u32, @@ -212,7 +212,7 @@ where /// /// # Errors /// In addition to [`Advz::new`], return [`VidError::Argument`] if - /// - TEMPORARY `max_multiplicity` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/339) + /// - TEMPORARY `max_multiplicity` is not a power of two [github issue](https://github.com/EspressoSystems/jellyfish/issues/668) pub fn with_multiplicity( num_storage_nodes: u32, recovery_threshold: u32, From e16f5ca3172f152447ddb3808eb72212ada0d49b Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:32:11 -0400 Subject: [PATCH 34/40] typo as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1736383113 --- vid/src/advz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 0566e9af3..dc579591d 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -293,7 +293,7 @@ where evals: Vec>, #[serde(with = "canonical")] - // aggretate_proofs.len() equals multiplicity + // aggregate_proofs.len() equals multiplicity // TODO further aggregate into a single KZG proof. aggregate_proofs: Vec>, From d5ff0d42123f0a67e07baf4c55cbaf0c34213883 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:34:30 -0400 Subject: [PATCH 35/40] delete debugging comment as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1737065017 --- vid/src/advz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index dc579591d..686b5048a 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -615,7 +615,7 @@ where elems.append(&mut coeffs); } - assert_eq!(elems.len(), elems_capacity); // 4 vs 2 + assert_eq!(elems.len(), elems_capacity); let mut payload: Vec<_> = field_to_bytes(elems).collect(); payload.truncate(common.payload_byte_len.try_into().map_err(vid)?); From 962912b4ff4c0b7389008812ddf9eeef725571ec Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:36:14 -0400 Subject: [PATCH 36/40] remove superfluous log message as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1738176332 --- vid/src/advz.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 686b5048a..aa1e617cd 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -394,9 +394,7 @@ where let payload_byte_len = payload.len().try_into().map_err(vid)?; let multiplicity = self.min_multiplicity(payload_byte_len)?; let chunk_size = multiplicity * self.recovery_threshold; - let bytes_to_polys_time = start_timer!(|| "encode payload bytes into polynomials"); let polys = self.bytes_to_polys(payload)?; - end_timer!(bytes_to_polys_time); let poly_commits_time = start_timer!(|| "batch poly commit"); let poly_commits = >::kzg_batch_commit(self, &polys)?; From c5c160599323f689109f29432ba6161ef14eed0a Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:38:18 -0400 Subject: [PATCH 37/40] remove superfluous timer as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1738218808 --- vid/src/advz.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index aa1e617cd..be626bdab 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -715,7 +715,6 @@ where KzgEvalsMerkleTree::::from_elems(None, &all_storage_node_evals).map_err(vid)?; end_timer!(all_evals_commit_timer); - let common_timer = start_timer!(|| format!("compute {} KZG commitments", polys.len())); let common = Common { poly_commits, all_evals_digest: all_evals_commit.commitment().digest(), @@ -723,7 +722,6 @@ where num_storage_nodes: self.num_storage_nodes, multiplicity, }; - end_timer!(common_timer); let commit = Self::derive_commit( &common.poly_commits, From 74604d0631bbd34da4ac89039bf60fe0fe8d9169 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:48:21 -0400 Subject: [PATCH 38/40] paranoia as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1738317888 --- vid/src/advz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index be626bdab..3a266d885 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -907,7 +907,7 @@ where // payload is small: choose the smallest `m` such that `0 < m < // multiplicity` and the entire payload fits into `m * // recovery_threshold` elements. - let m = elems.div_ceil(self.recovery_threshold).max(1); + let m = elems.div_ceil(self.recovery_threshold.min(1)).max(1); // TODO TEMPORARY: enforce power-of-2 // https://github.com/EspressoSystems/jellyfish/issues/668 From 1b5d187febddb711765d894103737407a1fcdcd1 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 15:54:05 -0400 Subject: [PATCH 39/40] clarify comment as per https://github.com/EspressoSystems/jellyfish/pull/670#discussion_r1738324592 --- vid/src/advz/payload_prover.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/vid/src/advz/payload_prover.rs b/vid/src/advz/payload_prover.rs index 01ba7a443..46baae595 100644 --- a/vid/src/advz/payload_prover.rs +++ b/vid/src/advz/payload_prover.rs @@ -93,11 +93,14 @@ where self.final_poly_points_range_end(range_elem.len(), offset_elem, multiplicity); // prepare list of input points + // + // perf: if payload is small enough to fit into a single polynomial then + // we don't need all the points in this domain. let points: Vec<_> = Self::eval_domain( usize::try_from(self.recovery_threshold * multiplicity).map_err(vid)?, )? .elements() - .collect(); // perf: we might not need all these points + .collect(); let elems_iter = bytes_to_field::<_, KzgEval>(&payload[range_poly_byte]); let mut proofs = Vec::with_capacity(range_poly.len() * points.len()); @@ -171,11 +174,14 @@ where ); // prepare list of input points + // + // perf: if payload is small enough to fit into a single polynomial then + // we don't need all the points in this domain. let points: Vec<_> = Self::eval_domain( usize::try_from(self.recovery_threshold * stmt.common.multiplicity).map_err(vid)?, )? .elements() - .collect(); // perf: we might not need all these points + .collect(); // verify proof let mut cur_proof_index = 0; From a1f8154af1ddac84bd37d3b9cbab07567c3c7884 Mon Sep 17 00:00:00 2001 From: Gus Gutoski Date: Fri, 30 Aug 2024 16:14:42 -0400 Subject: [PATCH 40/40] stop being such a dumbass --- vid/src/advz.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vid/src/advz.rs b/vid/src/advz.rs index 3a266d885..756654076 100644 --- a/vid/src/advz.rs +++ b/vid/src/advz.rs @@ -907,7 +907,7 @@ where // payload is small: choose the smallest `m` such that `0 < m < // multiplicity` and the entire payload fits into `m * // recovery_threshold` elements. - let m = elems.div_ceil(self.recovery_threshold.min(1)).max(1); + let m = elems.div_ceil(self.recovery_threshold.max(1)).max(1); // TODO TEMPORARY: enforce power-of-2 // https://github.com/EspressoSystems/jellyfish/issues/668