Merge pull request #3001 from autonomys/refactor-records-encoding

Refactor records encoding
autonomys · Sep 3, 2024 · 5c484f3 · 5c484f3
2 parents 90728e1 + 6b73ba7
commit 5c484f3
Show file tree

Hide file tree

Showing 9 changed files with 228 additions and 125 deletions.
diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs
@@ -52,7 +52,7 @@ use subspace_core_primitives::{
 };
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_sector_sync;
-use subspace_farmer_components::plotting::{plot_sector, PlotSectorOptions};
+use subspace_farmer_components::plotting::{plot_sector, CpuRecordsEncoder, PlotSectorOptions};
 use subspace_farmer_components::reading::ReadSectorRecordChunksMode;
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::shim::ShimTable;
@@ -417,7 +417,7 @@ pub fn create_signed_vote(
     for sector_index in iter::from_fn(|| Some(rand::random())) {
         let mut plotted_sector_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+        let plotted_sector = block_on(plot_sector(PlotSectorOptions {
             public_key: &public_key,
             sector_index,
             piece_getter: archived_history_segment,
@@ -428,7 +428,11 @@ pub fn create_signed_vote(
             sector_output: &mut plotted_sector_bytes,
             downloading_semaphore: None,
             encoding_semaphore: None,
-            table_generators: slice::from_mut(&mut table_generator),
+            records_encoder: &mut CpuRecordsEncoder::<PosTable>::new(
+                slice::from_mut(&mut table_generator),
+                erasure_coding,
+                &Default::default(),
+            ),
             abort_early: &Default::default(),
         }))
         .unwrap();

diff --git a/crates/sp-lightclient/src/tests.rs b/crates/sp-lightclient/src/tests.rs
@@ -176,7 +176,7 @@ fn valid_header(
         let mut plotted_sector_bytes = Vec::new();
         let mut plotted_sector_metadata_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<PosTable, _>(
+        let plotted_sector = block_on(plot_sector(
             &public_key,
             sector_index,
             &archived_segment.pieces,
@@ -186,7 +186,11 @@ fn valid_header(
             pieces_in_sector,
             &mut plotted_sector_bytes,
             &mut plotted_sector_metadata_bytes,
-            &mut table_generator,
+            records_encoder: &mut CpuRecordsEncoder::<PosTable>::new(
+                slice::from_mut(&mut table_generator),
+                &erasure_coding,
+                &Default::default(),
+            ),
         ))
         .unwrap();
 

diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs
@@ -15,7 +15,9 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_plot_sync;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PlotSectorOptions, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, CpuRecordsEncoder, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
 };
@@ -115,7 +117,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let mut plotted_sector_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+        let plotted_sector = block_on(plot_sector(PlotSectorOptions {
             public_key,
             sector_index,
             piece_getter: &archived_history_segment,
@@ -126,7 +128,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             sector_output: &mut plotted_sector_bytes,
             downloading_semaphore: black_box(None),
             encoding_semaphore: black_box(None),
-            table_generators: slice::from_mut(&mut table_generator),
+            records_encoder: &mut CpuRecordsEncoder::<PosTable>::new(
+                slice::from_mut(&mut table_generator),
+                &erasure_coding,
+                &Default::default(),
+            ),
             abort_early: &Default::default(),
         }))
         .unwrap();

diff --git a/crates/subspace-farmer-components/benches/plotting.rs b/crates/subspace-farmer-components/benches/plotting.rs
@@ -8,7 +8,7 @@ use subspace_core_primitives::crypto::kzg;
 use subspace_core_primitives::crypto::kzg::Kzg;
 use subspace_core_primitives::{HistorySize, PublicKey, Record, RecordedHistorySegment};
 use subspace_erasure_coding::ErasureCoding;
-use subspace_farmer_components::plotting::{plot_sector, PlotSectorOptions};
+use subspace_farmer_components::plotting::{plot_sector, CpuRecordsEncoder, PlotSectorOptions};
 use subspace_farmer_components::sector::sector_size;
 use subspace_farmer_components::FarmerProtocolInfo;
 use subspace_proof_of_space::chia::ChiaTable;
@@ -74,7 +74,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     group.throughput(Throughput::Bytes(sector_size as u64));
     group.bench_function("in-memory", |b| {
         b.iter(|| {
-            block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+            block_on(plot_sector(PlotSectorOptions {
                 public_key: black_box(&public_key),
                 sector_index: black_box(sector_index),
                 piece_getter: black_box(&archived_history_segment),
@@ -85,7 +85,11 @@ fn criterion_benchmark(c: &mut Criterion) {
                 sector_output: black_box(&mut sector_bytes),
                 downloading_semaphore: black_box(None),
                 encoding_semaphore: black_box(None),
-                table_generators: black_box(&mut table_generators),
+                records_encoder: black_box(&mut CpuRecordsEncoder::<PosTable>::new(
+                    &mut table_generators,
+                    &erasure_coding,
+                    &Default::default(),
+                )),
                 abort_early: &Default::default(),
             }))
             .unwrap();

diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs
@@ -20,7 +20,9 @@ use subspace_core_primitives::{
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::auditing::audit_plot_sync;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PlotSectorOptions, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, CpuRecordsEncoder, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::reading::ReadSectorRecordChunksMode;
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
@@ -123,7 +125,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let mut plotted_sector_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+        let plotted_sector = block_on(plot_sector(PlotSectorOptions {
             public_key,
             sector_index,
             piece_getter: &archived_history_segment,
@@ -134,7 +136,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             sector_output: &mut plotted_sector_bytes,
             downloading_semaphore: black_box(None),
             encoding_semaphore: black_box(None),
-            table_generators: slice::from_mut(&mut table_generator),
+            records_encoder: &mut CpuRecordsEncoder::<PosTable>::new(
+                slice::from_mut(&mut table_generator),
+                erasure_coding,
+                &Default::default(),
+            ),
             abort_early: &Default::default(),
         }))
         .unwrap();

diff --git a/crates/subspace-farmer-components/benches/reading.rs b/crates/subspace-farmer-components/benches/reading.rs
@@ -15,7 +15,9 @@ use subspace_core_primitives::{
 };
 use subspace_erasure_coding::ErasureCoding;
 use subspace_farmer_components::file_ext::{FileExt, OpenOptionsExt};
-use subspace_farmer_components::plotting::{plot_sector, PlotSectorOptions, PlottedSector};
+use subspace_farmer_components::plotting::{
+    plot_sector, CpuRecordsEncoder, PlotSectorOptions, PlottedSector,
+};
 use subspace_farmer_components::reading::{read_piece, ReadSectorRecordChunksMode};
 use subspace_farmer_components::sector::{
     sector_size, SectorContentsMap, SectorMetadata, SectorMetadataChecksummed,
@@ -114,7 +116,7 @@ pub fn criterion_benchmark(c: &mut Criterion) {
 
         let mut plotted_sector_bytes = Vec::new();
 
-        let plotted_sector = block_on(plot_sector::<PosTable, _>(PlotSectorOptions {
+        let plotted_sector = block_on(plot_sector(PlotSectorOptions {
             public_key: &public_key,
             sector_index,
             piece_getter: &archived_history_segment,
@@ -125,7 +127,11 @@ pub fn criterion_benchmark(c: &mut Criterion) {
             sector_output: &mut plotted_sector_bytes,
             downloading_semaphore: black_box(None),
             encoding_semaphore: black_box(None),
-            table_generators: slice::from_mut(&mut table_generator),
+            records_encoder: &mut CpuRecordsEncoder::<PosTable>::new(
+                slice::from_mut(&mut table_generator),
+                &erasure_coding,
+                &Default::default(),
+            ),
             abort_early: &Default::default(),
         }))
         .unwrap();