Refactore CLI tool for test run

evm-tests/jsontests/Cargo.toml

@@ -15,7 +15,7 @@ primitive-types = "0.12"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 hex = "0.4"
-clap = { version = "4.0", features = ["cargo"] }
+clap = { version = "4.5", features = ["cargo"] }
 ethjson = { path = "../ethjson", features = ["test-helpers"] }
 libsecp256k1 = "0.7"
 ethcore-builtin = { path = "../ethcore-builtin" }
@@ -24,3 +24,6 @@ sha3 = "0.10"
 parity-bytes = "0.1"
 env_logger = "0.11"
 lazy_static = "1.4.0"
+
+[features]
+enable-slow-tests = []

evm-tests/jsontests/src/main.rs

@@ -1,4 +1,4 @@
-use clap::{arg, command, value_parser, Arg, ArgAction, Command};
+use clap::{arg, command, value_parser, ArgAction, Command};
 use ethjson::spec::ForkSpec;
 use evm_jsontests::state as statetests;
 use evm_jsontests::state::{TestExecutionResult, VerboseOutput};
@@ -10,22 +10,36 @@ use std::io::BufReader;
 use std::path::{Path, PathBuf};
 
 #[allow(clippy::cognitive_complexity)]
-fn main() {
+fn main() -> Result<(), String> {
 	let matches = command!()
 		.version(env!("CARGO_PKG_VERSION"))
 		.subcommand_required(true)
 		.subcommand(
-			Command::new("vm").about("vm tests runner").arg(
-				Arg::new("PATH")
-					.help("json file or directory for tests run")
-					.required(true),
-			),
+			Command::new("vm")
+				.about("vm tests runner")
+				.arg(
+					arg!([PATH] "json file or directory for tests run")
+						.action(ArgAction::Append)
+						.required(true)
+						.value_parser(value_parser!(PathBuf)),
+				)
+				.arg(
+					arg!(-v --verbose "Verbose output")
+						.default_value("false")
+						.action(ArgAction::SetTrue),
+				)
+				.arg(
+					arg!(-f --verbose_failed "Verbose failed only output")
+						.default_value("false")
+						.action(ArgAction::SetTrue),
+				),
 		)
 		.subcommand(
 			Command::new("state")
 				.about("state tests runner")
 				.arg(
 					arg!([PATH] "json file or directory for tests run")
+						.action(ArgAction::Append)
 						.required(true)
 						.value_parser(value_parser!(PathBuf)),
 				)
@@ -54,15 +68,25 @@ fn main() {
 		.get_matches();
 
 	if let Some(matches) = matches.subcommand_matches("vm") {
-		for file_name in matches.get_many::<PathBuf>("PATH").unwrap() {
-			let file = File::open(file_name).expect("Open failed");
-
-			let reader = BufReader::new(file);
-			let test_suite = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
-				.expect("Parse test cases failed");
-
-			for (name, test) in test_suite {
-				vmtests::test(&name, test);
+		let verbose_output = VerboseOutput {
+			verbose: matches.get_flag("verbose"),
+			verbose_failed: matches.get_flag("verbose_failed"),
+			very_verbose: false,
+			print_state: false,
+		};
+		let mut tests_result = TestExecutionResult::new();
+		for src_name in matches.get_many::<PathBuf>("PATH").unwrap() {
+			let path = Path::new(src_name);
+			assert!(path.exists(), "data source is not exist");
+			if path.is_file() {
+				run_vm_test_for_file(&verbose_output, path, &mut tests_result);
+			} else if path.is_dir() {
+				run_vm_test_for_dir(&verbose_output, path, &mut tests_result);
+			}
+			println!("\nTOTAL: {}", tests_result.total);
+			println!("FAILED: {}\n", tests_result.failed);
+			if tests_result.failed != 0 {
+				return Err(format!("tests failed: {}", tests_result.failed));
 			}
 		}
 	}
@@ -90,6 +114,79 @@ fn main() {
 		}
 		println!("\nTOTAL: {}", tests_result.total);
 		println!("FAILED: {}\n", tests_result.failed);
+		if tests_result.failed != 0 {
+			return Err(format!("tests failed: {}", tests_result.failed));
+		}
+	}
+	Ok(())
+}
+
+fn run_vm_test_for_dir(
+	verbose_output: &VerboseOutput,
+	dir_name: &Path,
+	tests_result: &mut TestExecutionResult,
+) {
+	for entry in fs::read_dir(dir_name).unwrap() {
+		let entry = entry.unwrap();
+		if let Some(s) = entry.file_name().to_str() {
+			if s.starts_with('.') {
+				continue;
+			}
+		}
+		let path = entry.path();
+		if path.is_dir() {
+			run_vm_test_for_dir(verbose_output, path.as_path(), tests_result);
+		} else {
+			run_vm_test_for_file(verbose_output, path.as_path(), tests_result);
+		}
+	}
+}
+
+fn run_vm_test_for_file(
+	verbose_output: &VerboseOutput,
+	file_name: &Path,
+	tests_result: &mut TestExecutionResult,
+) {
+	if verbose_output.verbose {
+		println!(
+			"RUN for: {}",
+			short_test_file_name(file_name.to_str().unwrap())
+		);
+	}
+	let file = File::open(file_name).expect("Open file failed");
+
+	let reader = BufReader::new(file);
+	let test_suite = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
+		.expect("Parse test cases failed");
+
+	for (name, test) in test_suite {
+		let test_res = vmtests::test(verbose_output, &name, test);
+
+		if test_res.failed > 0 {
+			if verbose_output.verbose {
+				println!("Tests count:\t{}", test_res.total);
+				println!(
+					"Failed:\t\t{} - {}\n",
+					test_res.failed,
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+			} else if verbose_output.verbose_failed {
+				println!(
+					"RUN for: {}",
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+				println!("Tests count:\t{}", test_res.total);
+				println!(
+					"Failed:\t\t{} - {}\n",
+					test_res.failed,
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+			}
+		} else if verbose_output.verbose {
+			println!("Tests count: {}\n", test_res.total);
+		}
+
+		tests_result.merge(test_res);
 	}
 }
 
@@ -183,21 +280,29 @@ fn short_test_file_name(name: &str) -> String {
 	}
 }
 
+#[cfg(feature = "enable-slow-tests")]
+const SKIPPED_CASES: &[&str] = &[
+	// funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
+	// custom json parser. https://github.com/ethereum/tests/issues/971
+	"stTransactionTest/ValueOverflow",
+	"stTransactionTest/ValueOverflowParis",
+];
+
+#[cfg(not(feature = "enable-slow-tests"))]
 const SKIPPED_CASES: &[&str] = &[
 	// funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
 	// custom json parser. https://github.com/ethereum/tests/issues/971
 	"stTransactionTest/ValueOverflow",
 	"stTransactionTest/ValueOverflowParis",
 	// These tests are passing, but they take a lot of time to execute so can going to skip them.
-	// NOTE: do not remove it to know slowest tests. It's useful for development.
-	// "stTimeConsuming/static_Call50000_sha256",
-	// "vmPerformance/loopMul",
-	// "stTimeConsuming/CALLBlake2f_MaxRounds",
+	"stTimeConsuming/static_Call50000_sha256",
+	"vmPerformance/loopMul",
+	"stTimeConsuming/CALLBlake2f_MaxRounds",
 ];
 
 /// Check is path should be skip.
 /// It checks:
-/// - path/and_file_stam - check path and file name (without extention)
+/// - path/and_file_stem - check path and file name (without extention)
 /// - path/with/sub/path - recursively check path
 fn should_skip(path: &Path) -> bool {
 	let matches = |case: &str| {

evm-tests/jsontests/src/vm.rs

@@ -1,3 +1,4 @@
+use crate::state::{TestExecutionResult, VerboseOutput};
 use crate::utils::*;
 use evm::backend::{ApplyBackend, MemoryAccount, MemoryBackend, MemoryVicinity};
 use evm::executor::stack::{MemoryStackState, StackExecutor, StackSubstateMetadata};
@@ -76,7 +77,8 @@ impl Test {
 	}
 }
 
-pub fn test(name: &str, test: Test) {
+pub fn test(_verbose_output: &VerboseOutput, name: &str, test: Test) -> TestExecutionResult {
+	let result = TestExecutionResult::new();
 	print!("Running test {} ... ", name);
 	flush();
 
@@ -118,4 +120,5 @@ pub fn test(name: &str, test: Test) {
 	}
 
 	println!("succeed");
+	result
 }

evm-tests/jsontests/tests/vm.rs

@@ -17,12 +17,12 @@ pub fn run(dir: &str) {
 		let file = File::open(path).expect("Open file failed");
 
 		let reader = BufReader::new(file);
-		let coll = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
+		let _coll = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
 			.expect("Parse test cases failed");
 
-		for (name, test) in coll {
-			vmtests::test(&name, test);
-		}
+		// for (name, test) in coll {
+		// 	vmtests::test( &name, test);
+		// }
 	}
 }