From b2121d633b4c45f52b6b2d375fbcf95ec937f182 Mon Sep 17 00:00:00 2001 From: Robin Salen Date: Sun, 3 Dec 2023 14:19:24 -0500 Subject: [PATCH 1/2] Distinguish passed tests --- evm_test_runner/src/main.rs | 2 +- evm_test_runner/src/persistent_run_state.rs | 35 ++++++++++++++++++--- evm_test_runner/src/plonky2_runner.rs | 13 +++++--- evm_test_runner/src/report_generation.rs | 7 ++++- 4 files changed, 45 insertions(+), 12 deletions(-) diff --git a/evm_test_runner/src/main.rs b/evm_test_runner/src/main.rs index 59cebd9..0265ff1 100644 --- a/evm_test_runner/src/main.rs +++ b/evm_test_runner/src/main.rs @@ -67,7 +67,7 @@ async fn run() -> anyhow::Result { let passed_t_names = skip_passed.then(|| { Arc::new( persistent_test_state - .get_tests_that_have_passed() + .get_tests_that_have_passed(witness_only) .map(|t| t.to_string()) .collect(), ) diff --git a/evm_test_runner/src/persistent_run_state.rs b/evm_test_runner/src/persistent_run_state.rs index 22fd881..82744d4 100644 --- a/evm_test_runner/src/persistent_run_state.rs +++ b/evm_test_runner/src/persistent_run_state.rs @@ -67,9 +67,18 @@ impl TestRunEntries { } } - pub(crate) fn get_tests_that_have_passed(&self) -> impl Iterator { - self.0.iter().filter_map(|(name, info)| { - matches!(info.pass_state, PassState::Passed | PassState::Ignored).then(|| name.as_str()) + /// Filters previously passed tests if the `skip_passed` argument is used. + /// The filtering will always ignore tests for which proof verification was + /// successful, but may not skip tests for which only witness generation + /// was tested, if we haven't passed the `witness_only` argument. + pub(crate) fn get_tests_that_have_passed( + &self, + witness_only: bool, + ) -> impl Iterator { + self.0.iter().filter_map(move |(name, info)| { + info.pass_state + .get_passed_status(witness_only) + .then(|| name.as_str()) }) } } @@ -90,17 +99,33 @@ impl From> for TestRunEntries { #[derive(Copy, Clone, Debug, Deserialize, Default, Serialize)] pub(crate) enum PassState { - Passed, + PassedWitness, + PassedProof, Ignored, Failed, #[default] NotRun, } +impl PassState { + // Utility method to filter out passed tests from previous runs. + fn get_passed_status(&self, witness_only: bool) -> bool { + if witness_only { + matches!( + self, + Self::PassedWitness | Self::PassedProof | Self::Ignored + ) + } else { + matches!(self, Self::PassedProof | Self::Ignored) + } + } +} + impl From for PassState { fn from(v: TestStatus) -> Self { match v { - TestStatus::Passed => PassState::Passed, + TestStatus::PassedWitness => PassState::PassedWitness, + TestStatus::PassedProof => PassState::PassedProof, TestStatus::Ignored => PassState::Ignored, TestStatus::EvmErr(_) | TestStatus::TimedOut => PassState::Failed, } diff --git a/evm_test_runner/src/plonky2_runner.rs b/evm_test_runner/src/plonky2_runner.rs index e3c9104..9661396 100644 --- a/evm_test_runner/src/plonky2_runner.rs +++ b/evm_test_runner/src/plonky2_runner.rs @@ -73,8 +73,8 @@ impl TestProgressIndicator for FancyProgressIndicator { #[derive(Clone, Debug)] pub(crate) enum TestStatus { - Passed, - #[allow(unused)] + PassedWitness, + PassedProof, Ignored, EvmErr(String), TimedOut, @@ -83,7 +83,8 @@ pub(crate) enum TestStatus { impl Display for TestStatus { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - TestStatus::Passed => write!(f, "Passed"), + TestStatus::PassedWitness => write!(f, "Passed witness generation"), + TestStatus::PassedProof => write!(f, "Passed proof verification"), TestStatus::Ignored => write!(f, "Ignored"), TestStatus::EvmErr(err) => write!(f, "Evm error: {}", err), TestStatus::TimedOut => write!(f, "Test timed out"), @@ -93,7 +94,7 @@ impl Display for TestStatus { impl TestStatus { pub(crate) fn passed(&self) -> bool { - matches!(self, TestStatus::Passed) + matches!(self, Self::PassedProof | Self::PassedWitness) } } @@ -282,6 +283,8 @@ fn run_test_and_get_test_result(test: TestVariantRunInfo, witness_only: bool) -> if let Err(evm_err) = res { return handle_evm_err(evm_err, is_gaslimit_changed, "witness generation"); } + + return TestStatus::PassedWitness; } false => { let proof_run_res = prove::( @@ -310,7 +313,7 @@ fn run_test_and_get_test_result(test: TestVariantRunInfo, witness_only: bool) -> } } - TestStatus::Passed + TestStatus::PassedProof } fn handle_evm_err( diff --git a/evm_test_runner/src/report_generation.rs b/evm_test_runner/src/report_generation.rs index 4b13911..6baa6b0 100644 --- a/evm_test_runner/src/report_generation.rs +++ b/evm_test_runner/src/report_generation.rs @@ -118,7 +118,12 @@ impl From for TemplateSubGroupResultsData { let tests: Vec = v.test_res.into_iter().collect(); let num_passed = tests .iter() - .filter(|t| matches!(t.status, TestStatus::Passed)) + .filter(|t| { + matches!( + t.status, + TestStatus::PassedProof | TestStatus::PassedWitness + ) + }) .count(); Self { From c4a564dba913af9cc88ff923fc366619a2bcc8b0 Mon Sep 17 00:00:00 2001 From: Robin Salen Date: Sun, 3 Dec 2023 15:10:11 -0500 Subject: [PATCH 2/2] Clippy --- evm_test_runner/src/persistent_run_state.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/evm_test_runner/src/persistent_run_state.rs b/evm_test_runner/src/persistent_run_state.rs index 82744d4..fba920e 100644 --- a/evm_test_runner/src/persistent_run_state.rs +++ b/evm_test_runner/src/persistent_run_state.rs @@ -78,7 +78,7 @@ impl TestRunEntries { self.0.iter().filter_map(move |(name, info)| { info.pass_state .get_passed_status(witness_only) - .then(|| name.as_str()) + .then_some(name.as_str()) }) } }