From 2d4cffdd2fcc242377a99ca504a6962cbf574e49 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 29 Jan 2024 05:49:12 +0200 Subject: [PATCH 1/3] Replace non-decodable sector metadata with dummy expired one in order to not prevent farmer from starting --- .../subspace-farmer/src/single_disk_farm.rs | 42 +++++++++++++------ 1 file changed, 30 insertions(+), 12 deletions(-) diff --git a/crates/subspace-farmer/src/single_disk_farm.rs b/crates/subspace-farmer/src/single_disk_farm.rs index 340f9cc9fe..5c576d99a9 100644 --- a/crates/subspace-farmer/src/single_disk_farm.rs +++ b/crates/subspace-farmer/src/single_disk_farm.rs @@ -361,9 +361,6 @@ pub enum SingleDiskFarmError { /// Failed to decode metadata header #[error("Failed to decode metadata header: {0}")] FailedToDecodeMetadataHeader(parity_scale_codec::Error), - /// Failed to decode sector metadata - #[error("Failed to decode sector metadata: {0}")] - FailedToDecodeSectorMetadata(parity_scale_codec::Error), /// Unexpected metadata version #[error("Unexpected metadata version {0}")] UnexpectedMetadataVersion(u8), @@ -765,12 +762,13 @@ impl SingleDiskFarm { } }; + let metadata_file_path = directory.join(Self::METADATA_FILE); let mut metadata_file = OpenOptions::new() .read(true) .write(true) .create(true) .advise_random_access() - .open(directory.join(Self::METADATA_FILE))?; + .open(&metadata_file_path)?; metadata_file.advise_random_access()?; @@ -827,14 +825,34 @@ impl SingleDiskFarm { let mut sector_metadata_bytes = vec![0; sector_metadata_size]; for sector_index in 0..metadata_header.plotted_sector_count { - metadata_file.read_exact_at( - &mut sector_metadata_bytes, - RESERVED_PLOT_METADATA + sector_metadata_size as u64 * u64::from(sector_index), - )?; - sectors_metadata.push( - SectorMetadataChecksummed::decode(&mut sector_metadata_bytes.as_ref()) - .map_err(SingleDiskFarmError::FailedToDecodeSectorMetadata)?, - ); + let sector_offset = + RESERVED_PLOT_METADATA + sector_metadata_size as u64 * u64::from(sector_index); + metadata_file.read_exact_at(&mut sector_metadata_bytes, sector_offset)?; + + let sector_metadata = + match SectorMetadataChecksummed::decode(&mut sector_metadata_bytes.as_ref()) { + Ok(sector_metadata) => sector_metadata, + Err(error) => { + warn!( + path = %metadata_file_path.display(), + %error, + %sector_index, + "Failed to decode sector metadata, replacing with dummy expired \ + sector metadata" + ); + + let dummy_sector = SectorMetadataChecksummed::from(SectorMetadata { + sector_index, + pieces_in_sector, + s_bucket_sizes: Box::new([0; Record::NUM_S_BUCKETS]), + history_size: HistorySize::from(SegmentIndex::ZERO), + }); + metadata_file.write_all_at(&dummy_sector.encode(), sector_offset)?; + + dummy_sector + } + }; + sectors_metadata.push(sector_metadata); } Arc::new(RwLock::new(sectors_metadata)) From d7f22e49d65942c303a067b08ca387df34b18143 Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 29 Jan 2024 06:20:36 +0200 Subject: [PATCH 2/3] Make auditing fallible in case it faces read error as read errors are not recoverable --- crates/pallet-subspace/src/mock.rs | 3 +- .../benches/auditing.rs | 38 ++++++----- .../benches/proving.rs | 6 +- .../src/auditing.rs | 67 +++++++++++-------- .../bin/subspace-farmer/commands/benchmark.rs | 8 +-- .../src/single_disk_farm/farming.rs | 28 +++++--- test/subspace-test-client/src/lib.rs | 1 + 7 files changed, 89 insertions(+), 62 deletions(-) diff --git a/crates/pallet-subspace/src/mock.rs b/crates/pallet-subspace/src/mock.rs index bdbf12795b..babb8bb4bd 100644 --- a/crates/pallet-subspace/src/mock.rs +++ b/crates/pallet-subspace/src/mock.rs @@ -482,7 +482,8 @@ pub fn create_signed_vote( vote_solution_range, &plotted_sector_bytes, &plotted_sector.sector_metadata, - ); + ) + .unwrap(); let Some(audit_result) = maybe_audit_result else { // Sector didn't have any solutions diff --git a/crates/subspace-farmer-components/benches/auditing.rs b/crates/subspace-farmer-components/benches/auditing.rs index 7d13f6d617..0c28bfdf6e 100644 --- a/crates/subspace-farmer-components/benches/auditing.rs +++ b/crates/subspace-farmer-components/benches/auditing.rs @@ -151,14 +151,17 @@ pub fn criterion_benchmark(c: &mut Criterion) { group.throughput(Throughput::Elements(1)); group.bench_function("memory/sync", |b| { b.iter(|| async { - black_box(audit_plot_sync( - black_box(public_key), - black_box(global_challenge), - black_box(solution_range), - black_box(&plotted_sector_bytes), - black_box(slice::from_ref(&plotted_sector.sector_metadata)), - black_box(None), - )); + black_box( + audit_plot_sync( + black_box(public_key), + black_box(global_challenge), + black_box(solution_range), + black_box(&plotted_sector_bytes), + black_box(slice::from_ref(&plotted_sector.sector_metadata)), + black_box(None), + ) + .unwrap(), + ); }) }); @@ -193,14 +196,17 @@ pub fn criterion_benchmark(c: &mut Criterion) { group.throughput(Throughput::Elements(sectors_count)); group.bench_function("disk/sync", |b| { b.iter(|| { - black_box(audit_plot_sync( - black_box(public_key), - black_box(global_challenge), - black_box(solution_range), - black_box(&plot_file), - black_box(§ors_metadata), - black_box(None), - )); + black_box( + audit_plot_sync( + black_box(public_key), + black_box(global_challenge), + black_box(solution_range), + black_box(&plot_file), + black_box(§ors_metadata), + black_box(None), + ) + .unwrap(), + ); }); }); diff --git a/crates/subspace-farmer-components/benches/proving.rs b/crates/subspace-farmer-components/benches/proving.rs index f68aa041a5..fb79fc6358 100644 --- a/crates/subspace-farmer-components/benches/proving.rs +++ b/crates/subspace-farmer-components/benches/proving.rs @@ -166,7 +166,8 @@ pub fn criterion_benchmark(c: &mut Criterion) { &plotted_sector_bytes, slice::from_ref(&plotted_sector.sector_metadata), None, - ); + ) + .unwrap(); let solution_candidates = match audit_results.into_iter().next() { Some(audit_result) => audit_result.solution_candidates, @@ -249,7 +250,8 @@ pub fn criterion_benchmark(c: &mut Criterion) { &plot_file, §ors_metadata, None, - ); + ) + .unwrap(); let solution_candidates = audit_results .into_iter() .map(|audit_result| audit_result.solution_candidates) diff --git a/crates/subspace-farmer-components/src/auditing.rs b/crates/subspace-farmer-components/src/auditing.rs index 8f86b82ae5..5787d74646 100644 --- a/crates/subspace-farmer-components/src/auditing.rs +++ b/crates/subspace-farmer-components/src/auditing.rs @@ -2,12 +2,28 @@ use crate::proving::SolutionCandidates; use crate::sector::{sector_size, SectorContentsMap, SectorMetadataChecksummed}; use crate::{ReadAtOffset, ReadAtSync}; use rayon::prelude::*; +use std::io; use subspace_core_primitives::crypto::Scalar; use subspace_core_primitives::{ Blake3Hash, PublicKey, SBucket, SectorId, SectorIndex, SectorSlotChallenge, SolutionRange, }; use subspace_verification::is_within_solution_range; -use tracing::warn; +use thiserror::Error; + +/// Errors that happen during proving +#[derive(Debug, Error)] +pub enum AuditingError { + /// Failed read s-bucket + #[error("Failed read s-bucket {s_bucket_audit_index} of sector {sector_index}: {error}")] + SBucketReading { + /// Sector index + sector_index: SectorIndex, + /// S-bucket audit index + s_bucket_audit_index: SBucket, + /// Low-level error + error: io::Error, + }, +} /// Result of sector audit #[derive(Debug, Clone)] @@ -42,7 +58,7 @@ pub fn audit_sector_sync<'a, Sector>( solution_range: SolutionRange, sector: Sector, sector_metadata: &'a SectorMetadataChecksummed, -) -> Option> +) -> Result>, AuditingError> where Sector: ReadAtSync + 'a, { @@ -55,26 +71,24 @@ where } = collect_sector_auditing_details(public_key.hash(), global_challenge, sector_metadata); let mut s_bucket = vec![0; s_bucket_audit_size]; - let read_s_bucket_result = sector.read_at(&mut s_bucket, s_bucket_audit_offset_in_sector); - - if let Err(error) = read_s_bucket_result { - warn!( - %error, - sector_index = %sector_metadata.sector_index, - %s_bucket_audit_index, - "Failed read s-bucket", - ); - return None; - } + sector + .read_at(&mut s_bucket, s_bucket_audit_offset_in_sector) + .map_err(|error| AuditingError::SBucketReading { + sector_index: sector_metadata.sector_index, + s_bucket_audit_index, + error, + })?; - let (winning_chunks, best_solution_distance) = map_winning_chunks( + let Some((winning_chunks, best_solution_distance)) = map_winning_chunks( &s_bucket, global_challenge, §or_slot_challenge, solution_range, - )?; + ) else { + return Ok(None); + }; - Some(AuditResult { + Ok(Some(AuditResult { sector_index: sector_metadata.sector_index, solution_candidates: SolutionCandidates::new( public_key, @@ -85,7 +99,7 @@ where winning_chunks.into(), ), best_solution_distance, - }) + })) } /// Audit the whole plot and generate streams of solutions @@ -96,7 +110,7 @@ pub fn audit_plot_sync<'a, Plot>( plot: &'a Plot, sectors_metadata: &'a [SectorMetadataChecksummed], maybe_sector_being_modified: Option, -) -> Vec>> +) -> Result>>, AuditingError> where Plot: ReadAtSync + 'a, { @@ -135,14 +149,11 @@ where &mut s_bucket, sector_auditing_info.s_bucket_audit_offset_in_sector, ) { - warn!( - %error, - sector_index = %sector_metadata.sector_index, - s_bucket_audit_index = %sector_auditing_info.s_bucket_audit_index, - "Failed read s-bucket", - ); - - return None; + return Some(Err(AuditingError::SBucketReading { + sector_index: sector_metadata.sector_index, + s_bucket_audit_index: sector_auditing_info.s_bucket_audit_index, + error, + })); } let (winning_chunks, best_solution_distance) = map_winning_chunks( @@ -152,7 +163,7 @@ where solution_range, )?; - Some(AuditResult { + Some(Ok(AuditResult { sector_index: sector_metadata.sector_index, solution_candidates: SolutionCandidates::new( public_key, @@ -163,7 +174,7 @@ where winning_chunks.into(), ), best_solution_distance, - }) + })) }) .collect() } diff --git a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs index 38449330e3..dec81b82b7 100644 --- a/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs +++ b/crates/subspace-farmer/src/bin/subspace-farmer/commands/benchmark.rs @@ -250,7 +250,7 @@ fn prove( table_generator: &table_generator, }; - let mut audit_results = plot_audit.audit(options); + let mut audit_results = plot_audit.audit(options).unwrap(); group.bench_function("plot/single", |b| { b.iter_batched( @@ -259,7 +259,7 @@ fn prove( return result; } - audit_results = plot_audit.audit(options); + audit_results = plot_audit.audit(options).unwrap(); audit_results.pop().unwrap() }, @@ -293,7 +293,7 @@ fn prove( maybe_sector_being_modified: None, table_generator: &table_generator, }; - let mut audit_results = plot_audit.audit(options); + let mut audit_results = plot_audit.audit(options).unwrap(); group.bench_function("plot/rayon", |b| { b.iter_batched( @@ -302,7 +302,7 @@ fn prove( return result; } - audit_results = plot_audit.audit(options); + audit_results = plot_audit.audit(options).unwrap(); audit_results.pop().unwrap() }, diff --git a/crates/subspace-farmer/src/single_disk_farm/farming.rs b/crates/subspace-farmer/src/single_disk_farm/farming.rs index a61b55d899..e63598090b 100644 --- a/crates/subspace-farmer/src/single_disk_farm/farming.rs +++ b/crates/subspace-farmer/src/single_disk_farm/farming.rs @@ -15,10 +15,10 @@ use std::{fmt, io}; use subspace_core_primitives::crypto::kzg::Kzg; use subspace_core_primitives::{PosSeed, PublicKey, SectorIndex, Solution, SolutionRange}; use subspace_erasure_coding::ErasureCoding; -use subspace_farmer_components::auditing::audit_plot_sync; +use subspace_farmer_components::auditing::{audit_plot_sync, AuditingError}; use subspace_farmer_components::proving::{ProvableSolutions, ProvingError}; use subspace_farmer_components::sector::SectorMetadataChecksummed; -use subspace_farmer_components::{proving, ReadAtSync}; +use subspace_farmer_components::ReadAtSync; use subspace_proof_of_space::{Table, TableGenerator}; use subspace_rpc_primitives::{SlotInfo, SolutionResponse}; use thiserror::Error; @@ -88,9 +88,12 @@ pub enum FarmingError { /// Lower-level error error: node_client::Error, }, + /// Low-level auditing error + #[error("Low-level auditing error: {0}")] + LowLevelAuditing(#[from] AuditingError), /// Low-level proving error #[error("Low-level proving error: {0}")] - LowLevelProving(#[from] proving::ProvingError), + LowLevelProving(#[from] ProvingError), /// I/O error occurred #[error("I/O error: {0}")] Io(#[from] io::Error), @@ -181,10 +184,13 @@ where pub fn audit( &'a self, options: PlotAuditOptions<'a, PosTable>, - ) -> Vec<( - SectorIndex, - impl ProvableSolutions, ProvingError>> + 'a, - )> + ) -> Result< + Vec<( + SectorIndex, + impl ProvableSolutions, ProvingError>> + 'a, + )>, + AuditingError, + > where PosTable: Table, { @@ -206,9 +212,9 @@ where &self.0, sectors_metadata, maybe_sector_being_modified, - ); + )?; - audit_results + Ok(audit_results .into_iter() .filter_map(|audit_results| { let sector_index = audit_results.sector_index; @@ -239,7 +245,7 @@ where Some((sector_index, sector_solutions)) }) - .collect() + .collect()) } } @@ -311,7 +317,7 @@ where erasure_coding: &erasure_coding, maybe_sector_being_modified, table_generator: &table_generator, - }) + })? }; sectors_solutions.sort_by(|a, b| { diff --git a/test/subspace-test-client/src/lib.rs b/test/subspace-test-client/src/lib.rs index 071f62fdb7..e4d52c8f6e 100644 --- a/test/subspace-test-client/src/lib.rs +++ b/test/subspace-test-client/src/lib.rs @@ -190,6 +190,7 @@ async fn start_farming( ); let solution = audit_result + .unwrap() .unwrap() .solution_candidates .into_solutions(&public_key, &kzg, &erasure_coding, |seed: &PosSeed| { From 2c33788271a4ddb8790ec165f80fec952bd0e9dc Mon Sep 17 00:00:00 2001 From: Nazar Mokrynskyi Date: Mon, 29 Jan 2024 06:24:24 +0200 Subject: [PATCH 3/3] Ignore non-fatal farming errors, emit them as events and log as metrics --- .../subspace-farmer-components/src/proving.rs | 14 + .../subspace-farmer-components/src/reading.rs | 15 + .../src/single_disk_farm/farming.rs | 258 ++++++++++++------ 3 files changed, 200 insertions(+), 87 deletions(-) diff --git a/crates/subspace-farmer-components/src/proving.rs b/crates/subspace-farmer-components/src/proving.rs index e1cca1982f..1258165bda 100644 --- a/crates/subspace-farmer-components/src/proving.rs +++ b/crates/subspace-farmer-components/src/proving.rs @@ -60,6 +60,20 @@ pub enum ProvingError { RecordReadingError(#[from] ReadingError), } +impl ProvingError { + /// Whether this error is fatal and makes farm unusable + pub fn is_fatal(&self) -> bool { + match self { + ProvingError::InvalidErasureCodingInstance => true, + ProvingError::FailedToCreatePolynomialForRecord { .. } => false, + ProvingError::FailedToCreateChunkWitness { .. } => false, + ProvingError::FailedToDecodeSectorContentsMap(_) => false, + ProvingError::Io(_) => true, + ProvingError::RecordReadingError(error) => error.is_fatal(), + } + } +} + #[derive(Debug, Clone)] struct WinningChunk { /// Chunk offset within s-bucket diff --git a/crates/subspace-farmer-components/src/reading.rs b/crates/subspace-farmer-components/src/reading.rs index 307f52fdb1..0bfe08be0f 100644 --- a/crates/subspace-farmer-components/src/reading.rs +++ b/crates/subspace-farmer-components/src/reading.rs @@ -75,6 +75,21 @@ pub enum ReadingError { ChecksumMismatch, } +impl ReadingError { + /// Whether this error is fatal and renders farm unusable + pub fn is_fatal(&self) -> bool { + match self { + ReadingError::FailedToReadChunk { .. } => false, + ReadingError::InvalidChunk { .. } => false, + ReadingError::FailedToErasureDecodeRecord { .. } => false, + ReadingError::WrongRecordSizeAfterDecoding { .. } => false, + ReadingError::FailedToDecodeSectorContentsMap(_) => false, + ReadingError::Io(_) => true, + ReadingError::ChecksumMismatch => false, + } + } +} + /// Record contained in the plot #[derive(Debug, Clone)] pub struct PlotRecord { diff --git a/crates/subspace-farmer/src/single_disk_farm/farming.rs b/crates/subspace-farmer/src/single_disk_farm/farming.rs index e63598090b..36c48762b2 100644 --- a/crates/subspace-farmer/src/single_disk_farm/farming.rs +++ b/crates/subspace-farmer/src/single_disk_farm/farming.rs @@ -6,7 +6,7 @@ use crate::single_disk_farm::Handlers; use async_lock::RwLock; use futures::channel::mpsc; use futures::StreamExt; -use parity_scale_codec::{Decode, Encode}; +use parity_scale_codec::{Decode, Encode, Error, Input, Output}; use parking_lot::Mutex; use rayon::ThreadPoolBuildError; use std::sync::Arc; @@ -71,6 +71,23 @@ pub enum FarmingNotification { Auditing(AuditingDetails), /// Proving Proving(ProvingDetails), + /// Non-fatal farming error + NonFatalError(Arc), +} + +/// Special decoded farming error +#[derive(Debug, Encode, Decode)] +pub struct DecodedFarmingError { + /// String representation of an error + error: String, + /// Whether error is fatal + is_fatal: bool, +} + +impl fmt::Display for DecodedFarmingError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + self.error.fmt(f) + } } /// Errors that happen during farming @@ -100,6 +117,54 @@ pub enum FarmingError { /// Failed to create thread pool #[error("Failed to create thread pool: {0}")] FailedToCreateThreadPool(#[from] ThreadPoolBuildError), + /// Decoded farming error + #[error("Decoded farming error {0}")] + Decoded(DecodedFarmingError), +} + +impl Encode for FarmingError { + fn encode_to(&self, dest: &mut O) { + let error = DecodedFarmingError { + error: self.to_string(), + is_fatal: self.is_fatal(), + }; + + error.encode_to(dest) + } +} + +impl Decode for FarmingError { + fn decode(input: &mut I) -> Result { + DecodedFarmingError::decode(input).map(FarmingError::Decoded) + } +} + +impl FarmingError { + /// String variant of the error, primarily for monitoring purposes + pub fn str_variant(&self) -> &str { + match self { + FarmingError::FailedToSubscribeSlotInfo { .. } => "FailedToSubscribeSlotInfo", + FarmingError::FailedToGetFarmerInfo { .. } => "FailedToGetFarmerInfo", + FarmingError::LowLevelAuditing(_) => "LowLevelAuditing", + FarmingError::LowLevelProving(_) => "LowLevelProving", + FarmingError::Io(_) => "Io", + FarmingError::FailedToCreateThreadPool(_) => "FailedToCreateThreadPool", + FarmingError::Decoded(_) => "Decoded", + } + } + + /// Whether this error is fatal and makes farm unusable + pub fn is_fatal(&self) -> bool { + match self { + FarmingError::FailedToSubscribeSlotInfo { .. } => true, + FarmingError::FailedToGetFarmerInfo { .. } => true, + FarmingError::LowLevelAuditing(_) => true, + FarmingError::LowLevelProving(error) => error.is_fatal(), + FarmingError::Io(_) => true, + FarmingError::FailedToCreateThreadPool(_) => true, + FarmingError::Decoded(error) => error.is_fatal, + } + } } pub(super) async fn slot_notification_forwarder( @@ -298,108 +363,127 @@ where let table_generator = Arc::new(Mutex::new(PosTable::generator())); while let Some(slot_info) = slot_info_notifications.next().await { - let start = Instant::now(); - let slot = slot_info.slot_number; - let sectors_metadata = sectors_metadata.read().await; - - debug!(%slot, sector_count = %sectors_metadata.len(), "Reading sectors"); - - let mut sectors_solutions = { - let modifying_sector_guard = modifying_sector_index.read().await; - let maybe_sector_being_modified = modifying_sector_guard.as_ref().copied(); - - plot_audit.audit(PlotAuditOptions:: { - public_key: &public_key, - reward_address: &reward_address, - slot_info, - sectors_metadata: §ors_metadata, - kzg: &kzg, - erasure_coding: &erasure_coding, - maybe_sector_being_modified, - table_generator: &table_generator, - })? - }; + let result: Result<(), FarmingError> = try { + let start = Instant::now(); + let slot = slot_info.slot_number; + let sectors_metadata = sectors_metadata.read().await; + + debug!(%slot, sector_count = %sectors_metadata.len(), "Reading sectors"); + + let mut sectors_solutions = { + let modifying_sector_guard = modifying_sector_index.read().await; + let maybe_sector_being_modified = modifying_sector_guard.as_ref().copied(); + + plot_audit.audit(PlotAuditOptions:: { + public_key: &public_key, + reward_address: &reward_address, + slot_info, + sectors_metadata: §ors_metadata, + kzg: &kzg, + erasure_coding: &erasure_coding, + maybe_sector_being_modified, + table_generator: &table_generator, + })? + }; + + sectors_solutions.sort_by(|a, b| { + let a_solution_distance = + a.1.best_solution_distance().unwrap_or(SolutionRange::MAX); + let b_solution_distance = + b.1.best_solution_distance().unwrap_or(SolutionRange::MAX); + + a_solution_distance.cmp(&b_solution_distance) + }); + + handlers + .farming_notification + .call_simple(&FarmingNotification::Auditing(AuditingDetails { + sectors_count: sectors_metadata.len() as SectorIndex, + time: start.elapsed(), + })); + + 'solutions_processing: for (sector_index, sector_solutions) in sectors_solutions { + if sector_solutions.is_empty() { + continue; + } + let mut start = Instant::now(); + for maybe_solution in sector_solutions { + let solution = match maybe_solution { + Ok(solution) => solution, + Err(error) => { + error!(%slot, %sector_index, %error, "Failed to prove"); + // Do not error completely as disk corruption or other reasons why + // proving might fail + start = Instant::now(); + continue; + } + }; + + debug!(%slot, %sector_index, "Solution found"); + trace!(?solution, "Solution found"); + + if start.elapsed() >= farming_timeout { + handlers + .farming_notification + .call_simple(&FarmingNotification::Proving(ProvingDetails { + result: ProvingResult::Timeout, + time: start.elapsed(), + })); + warn!( + %slot, + %sector_index, + "Proving for solution skipped due to farming time limit", + ); - sectors_solutions.sort_by(|a, b| { - let a_solution_distance = a.1.best_solution_distance().unwrap_or(SolutionRange::MAX); - let b_solution_distance = b.1.best_solution_distance().unwrap_or(SolutionRange::MAX); + break 'solutions_processing; + } - a_solution_distance.cmp(&b_solution_distance) - }); + let response = SolutionResponse { + slot_number: slot, + solution, + }; - handlers - .farming_notification - .call_simple(&FarmingNotification::Auditing(AuditingDetails { - sectors_count: sectors_metadata.len() as SectorIndex, - time: start.elapsed(), - })); + handlers.solution.call_simple(&response); - 'solutions_processing: for (sector_index, sector_solutions) in sectors_solutions { - if sector_solutions.is_empty() { - continue; - } - let mut start = Instant::now(); - for maybe_solution in sector_solutions { - let solution = match maybe_solution { - Ok(solution) => solution, - Err(error) => { - error!(%slot, %sector_index, %error, "Failed to prove"); - // Do not error completely as disk corruption or other reasons why - // proving might fail - start = Instant::now(); - continue; + if let Err(error) = node_client.submit_solution_response(response).await { + handlers + .farming_notification + .call_simple(&FarmingNotification::Proving(ProvingDetails { + result: ProvingResult::Rejected, + time: start.elapsed(), + })); + warn!( + %slot, + %sector_index, + %error, + "Failed to send solution to node, skipping further proving for this slot", + ); + break 'solutions_processing; } - }; - - debug!(%slot, %sector_index, "Solution found"); - trace!(?solution, "Solution found"); - if start.elapsed() >= farming_timeout { handlers .farming_notification .call_simple(&FarmingNotification::Proving(ProvingDetails { - result: ProvingResult::Timeout, + result: ProvingResult::Success, time: start.elapsed(), })); - warn!( - %slot, - %sector_index, - "Proving for solution skipped due to farming time limit", - ); - - break 'solutions_processing; + start = Instant::now(); } + } + }; - let response = SolutionResponse { - slot_number: slot, - solution, - }; - - handlers.solution.call_simple(&response); - - if let Err(error) = node_client.submit_solution_response(response).await { - handlers - .farming_notification - .call_simple(&FarmingNotification::Proving(ProvingDetails { - result: ProvingResult::Rejected, - time: start.elapsed(), - })); - warn!( - %slot, - %sector_index, - %error, - "Failed to send solution to node, skipping further proving for this slot", - ); - break 'solutions_processing; - } + if let Err(error) = result { + if error.is_fatal() { + return Err(error); + } else { + warn!( + %error, + "Non-fatal farming error" + ); handlers .farming_notification - .call_simple(&FarmingNotification::Proving(ProvingDetails { - result: ProvingResult::Success, - time: start.elapsed(), - })); - start = Instant::now(); + .call_simple(&FarmingNotification::NonFatalError(Arc::new(error))); } } }