Feat: refactore CLI tool and CI, tests cleanup (#43)

* Extend tests output and refactore tests runner

* Refactored CLI for recursive tests and extended information

* Set Shanghai spec as default

* Extend output for tests

* evm-tests: extend state statistics for failed results

* evm-tests: extended debug info

* Extend gas cost analyzer

* Extend tests and fixes for Cancun hard fork

* Fix: Apply CREATE storage reset - changed logic. GAS cost for MCOPY - improved calculation

* EIP-3607 implementation. DeepCall bug fixes. KZG-boilerplate

* KZG precompile

* EIP-3860 test flow fixes

* Fix Shanghai tests

* Removev debug info

* Separate check_exit_reason func

* Remove printing

* Added `as_deref`  to  check_create_exit_reason

* Fix: tests for EIP-3860 (#41)

undefined

* ForkSpec string error. Refactored usage as constantn USIZE_MAX

* Remove already fixed tests from skipping list

* Refactore blob-hash logic and tests

* Added KzgInput

* Gas price fix and investigations for blob-transactions

* Refactored skipped-match and clippy

* Refactored gas price and should-skip logic

* Fix tests for KZG-precompiles and SSTORE gas cost

* Added print-debug feature

* Fix randomness validation

* Refactore transactions validation

* Extend expected check for call-transaction and empty-create assertion

* Edit doc comments

* Refactore CLI tool for test run

* remove vm-tests

* update vm-tests validations

* remove state tests

* Remove submodule: ethtests

* Update CI rules

* Update CI rules

* Update Cargo metadata

* Update cargo features

.github/workflows/rust.yml

@@ -16,19 +16,19 @@ jobs:
       - name: Rustfmt
         run: cargo fmt --all -- --check
       - name: Clippy
-        run: cargo clippy --all-targets -- -D clippy::all -D clippy::nursery
+        run: cargo  clippy --workspace --all-targets -- -D clippy::all -D clippy::nursery
       - name: Clippy no_std
-        run: cargo clippy --all-targets --no-default-features -- -D clippy::all -D clippy::nursery
+        run: cargo  clippy --no-default-features -- -D clippy::all -D clippy::nursery
   build:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
       - name: Build
         run: cargo build --verbose
       - name: Build NoStd
-        run: cargo build --no-default-features --verbose
+        run: cargo build --no-default-features
       - name: Build for feature (tracing)
-        run: cargo build --features tracing --verbose
+        run: cargo build --features tracing
   tests:
     runs-on: ubuntu-latest
     steps:
@@ -37,3 +37,41 @@ jobs:
           submodules: recursive
       - name: Run tests
         run: cargo test --all --verbose
+  ethereum-tests:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Checkout ethereum/tests
+        uses: actions/checkout@v4
+        with:
+          repository: ethereum/tests
+          path: ethtests
+          ref: v13.2
+          submodules: recursive
+
+      - name: Run Ethereum state tests
+        run: |
+          cargo run -r -p evm-jsontests -F enable-slow-tests -- \
+          	state -f -s cancun \
+          	ethtests/GeneralStateTests/
+
+      - name: Run Ethereum vm tests
+        run: |
+          cargo run -r -p evm-jsontests -F enable-slow-tests -- \
+          	vm -f \
+            ethtests/LegacyTests/Constantinople/VMTests/vmArithmeticTest \
+            ethtests/LegacyTests/Constantinople/VMTests/vmBitwiseLogicOperation \
+            ethtests/LegacyTests/Constantinople/VMTests/vmBlockInfoTest \
+            ethtests/LegacyTests/Constantinople/VMTests/vmEnvironmentalInfo \
+            ethtests/LegacyTests/Constantinople/VMTests/vmIOandFlowOperations \
+            ethtests/LegacyTests/Constantinople/VMTests/vmLogTest \
+            ethtests/LegacyTests/Constantinople/VMTests/vmPerformance \
+            ethtests/LegacyTests/Constantinople/VMTests/vmPushDupSwapTest \
+            ethtests/LegacyTests/Constantinople/VMTests/vmRandomTest \
+            ethtests/LegacyTests/Constantinople/VMTests/vmSha3Test \
+            ethtests/LegacyTests/Constantinople/VMTests/vmSystemOperations \
+            ethtests/LegacyTests/Constantinople/VMTests/vmTests

Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "evm"
-version = "0.41.0"
+version = "0.42.0"
 license = "Apache-2.0"
 authors = ["Wei Tang <[email protected]>", "Parity Technologies <[email protected]>"]
 description = "SputnikVM - a Portable Blockchain Virtual Machine"
 repository = "https://github.com/sorpaas/rust-evm"
 keywords = ["no_std", "ethereum"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 auto_impl = "1.0"
@@ -19,12 +19,12 @@ sha3 = { version = "0.10", default-features = false }
 # Optional dependencies
 environmental = { version = "1.1.2", default-features = false, optional = true }
 scale-codec = { package = "parity-scale-codec", version = "3.2", default-features = false, features = ["derive"], optional = true }
-scale-info = { version = "2.3", default-features = false, features = ["derive"], optional = true }
+scale-info = { version = "2.11", default-features = false, features = ["derive"], optional = true }
 serde = { version = "1.0", default-features = false, features = ["derive"], optional = true }
 
-evm-core = { version = "0.41", path = "core", default-features = false }
-evm-gasometer = { version = "0.41", path = "gasometer", default-features = false }
-evm-runtime = { version = "0.41", path = "runtime", default-features = false }
+evm-core = { version = "0.42", path = "core", default-features = false }
+evm-gasometer = { version = "0.42", path = "gasometer", default-features = false }
+evm-runtime = { version = "0.42", path = "runtime", default-features = false }
 
 [dev-dependencies]
 criterion = "0.5"

core/Cargo.toml

@@ -1,12 +1,12 @@
 [package]
 name = "evm-core"
-version = "0.41.0"
+version = "0.42.0"
 license = "Apache-2.0"
 authors = ["Wei Tang <[email protected]>", "Parity Technologies <[email protected]>"]
 description = "Portable Ethereum Virtual Machine implementation written in pure Rust."
 repository = "https://github.com/sorpaas/rust-evm"
 keywords = ["no_std", "ethereum"]
-edition = "2018"
+edition = "2021"
 
 [dependencies]
 log = { version = "0.4", optional = true }
@@ -21,21 +21,21 @@ hex = "0.4"
 [features]
 default = ["std"]
 std = [
-	"primitive-types/std",
-	"serde/std",
-	"scale-codec/std",
-	"scale-info/std",
+    "primitive-types/std",
+    "serde/std",
+    "scale-codec/std",
+    "scale-info/std",
 ]
 with-codec = [
-	"scale-codec",
-	"scale-info",
-	"primitive-types/impl-codec",
+    "scale-codec",
+    "scale-info",
+    "primitive-types/impl-codec",
 ]
 with-serde = [
-	"serde",
-	"primitive-types/impl-serde",
+    "serde",
+    "primitive-types/impl-serde",
 ]
 force-debug = [
-	"log",
+    "log",
 ]
 tracing = []

evm-tests/jsontests/Cargo.toml

@@ -15,7 +15,7 @@ primitive-types = "0.12"
 serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 hex = "0.4"
-clap = { version = "4.0", features = ["cargo"] }
+clap = { version = "4.5", features = ["cargo"] }
 ethjson = { path = "../ethjson", features = ["test-helpers"] }
 libsecp256k1 = "0.7"
 ethcore-builtin = { path = "../ethcore-builtin" }
@@ -24,3 +24,6 @@ sha3 = "0.10"
 parity-bytes = "0.1"
 env_logger = "0.11"
 lazy_static = "1.4.0"
+
+[features]
+enable-slow-tests = []

evm-tests/jsontests/src/main.rs

@@ -1,4 +1,4 @@
-use clap::{arg, command, value_parser, Arg, ArgAction, Command};
+use clap::{arg, command, value_parser, ArgAction, Command};
 use ethjson::spec::ForkSpec;
 use evm_jsontests::state as statetests;
 use evm_jsontests::state::{TestExecutionResult, VerboseOutput};
@@ -10,22 +10,36 @@ use std::io::BufReader;
 use std::path::{Path, PathBuf};
 
 #[allow(clippy::cognitive_complexity)]
-fn main() {
+fn main() -> Result<(), String> {
 	let matches = command!()
 		.version(env!("CARGO_PKG_VERSION"))
 		.subcommand_required(true)
 		.subcommand(
-			Command::new("vm").about("vm tests runner").arg(
-				Arg::new("PATH")
-					.help("json file or directory for tests run")
-					.required(true),
-			),
+			Command::new("vm")
+				.about("vm tests runner")
+				.arg(
+					arg!([PATH] "json file or directory for tests run")
+						.action(ArgAction::Append)
+						.required(true)
+						.value_parser(value_parser!(PathBuf)),
+				)
+				.arg(
+					arg!(-v --verbose "Verbose output")
+						.default_value("false")
+						.action(ArgAction::SetTrue),
+				)
+				.arg(
+					arg!(-f --verbose_failed "Verbose failed only output")
+						.default_value("false")
+						.action(ArgAction::SetTrue),
+				),
 		)
 		.subcommand(
 			Command::new("state")
 				.about("state tests runner")
 				.arg(
 					arg!([PATH] "json file or directory for tests run")
+						.action(ArgAction::Append)
 						.required(true)
 						.value_parser(value_parser!(PathBuf)),
 				)
@@ -54,17 +68,27 @@ fn main() {
 		.get_matches();
 
 	if let Some(matches) = matches.subcommand_matches("vm") {
-		for file_name in matches.get_many::<PathBuf>("PATH").unwrap() {
-			let file = File::open(file_name).expect("Open failed");
-
-			let reader = BufReader::new(file);
-			let test_suite = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
-				.expect("Parse test cases failed");
-
-			for (name, test) in test_suite {
-				vmtests::test(&name, test);
+		let verbose_output = VerboseOutput {
+			verbose: matches.get_flag("verbose"),
+			verbose_failed: matches.get_flag("verbose_failed"),
+			very_verbose: false,
+			print_state: false,
+		};
+		let mut tests_result = TestExecutionResult::new();
+		for src_name in matches.get_many::<PathBuf>("PATH").unwrap() {
+			let path = Path::new(src_name);
+			assert!(path.exists(), "data source is not exist");
+			if path.is_file() {
+				run_vm_test_for_file(&verbose_output, path, &mut tests_result);
+			} else if path.is_dir() {
+				run_vm_test_for_dir(&verbose_output, path, &mut tests_result);
 			}
 		}
+		println!("\nTOTAL: {}", tests_result.total);
+		println!("FAILED: {}\n", tests_result.failed);
+		if tests_result.failed != 0 {
+			return Err(format!("tests failed: {}", tests_result.failed));
+		}
 	}
 
 	if let Some(matches) = matches.subcommand_matches("state") {
@@ -90,6 +114,79 @@ fn main() {
 		}
 		println!("\nTOTAL: {}", tests_result.total);
 		println!("FAILED: {}\n", tests_result.failed);
+		if tests_result.failed != 0 {
+			return Err(format!("tests failed: {}", tests_result.failed));
+		}
+	}
+	Ok(())
+}
+
+fn run_vm_test_for_dir(
+	verbose_output: &VerboseOutput,
+	dir_name: &Path,
+	tests_result: &mut TestExecutionResult,
+) {
+	for entry in fs::read_dir(dir_name).unwrap() {
+		let entry = entry.unwrap();
+		if let Some(s) = entry.file_name().to_str() {
+			if s.starts_with('.') {
+				continue;
+			}
+		}
+		let path = entry.path();
+		if path.is_dir() {
+			run_vm_test_for_dir(verbose_output, path.as_path(), tests_result);
+		} else {
+			run_vm_test_for_file(verbose_output, path.as_path(), tests_result);
+		}
+	}
+}
+
+fn run_vm_test_for_file(
+	verbose_output: &VerboseOutput,
+	file_name: &Path,
+	tests_result: &mut TestExecutionResult,
+) {
+	if verbose_output.verbose {
+		println!(
+			"RUN for: {}",
+			short_test_file_name(file_name.to_str().unwrap())
+		);
+	}
+	let file = File::open(file_name).expect("Open file failed");
+
+	let reader = BufReader::new(file);
+	let test_suite = serde_json::from_reader::<_, HashMap<String, vmtests::Test>>(reader)
+		.expect("Parse test cases failed");
+
+	for (name, test) in test_suite {
+		let test_res = vmtests::test(verbose_output, &name, test);
+
+		if test_res.failed > 0 {
+			if verbose_output.verbose {
+				println!("Tests count:\t{}", test_res.total);
+				println!(
+					"Failed:\t\t{} - {}\n",
+					test_res.failed,
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+			} else if verbose_output.verbose_failed {
+				println!(
+					"RUN for: {}",
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+				println!("Tests count:\t{}", test_res.total);
+				println!(
+					"Failed:\t\t{} - {}\n",
+					test_res.failed,
+					short_test_file_name(file_name.to_str().unwrap())
+				);
+			}
+		} else if verbose_output.verbose {
+			println!("Tests count: {}\n", test_res.total);
+		}
+
+		tests_result.merge(test_res);
 	}
 }
 
@@ -183,21 +280,29 @@ fn short_test_file_name(name: &str) -> String {
 	}
 }
 
+#[cfg(feature = "enable-slow-tests")]
+const SKIPPED_CASES: &[&str] = &[
+	// funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
+	// custom json parser. https://github.com/ethereum/tests/issues/971
+	"stTransactionTest/ValueOverflow",
+	"stTransactionTest/ValueOverflowParis",
+];
+
+#[cfg(not(feature = "enable-slow-tests"))]
 const SKIPPED_CASES: &[&str] = &[
 	// funky test with `bigint 0x00` value in json :) not possible to happen on mainnet and require
 	// custom json parser. https://github.com/ethereum/tests/issues/971
 	"stTransactionTest/ValueOverflow",
 	"stTransactionTest/ValueOverflowParis",
 	// These tests are passing, but they take a lot of time to execute so can going to skip them.
-	// NOTE: do not remove it to know slowest tests. It's useful for development.
-	// "stTimeConsuming/static_Call50000_sha256",
-	// "vmPerformance/loopMul",
-	// "stTimeConsuming/CALLBlake2f_MaxRounds",
+	"stTimeConsuming/static_Call50000_sha256",
+	"vmPerformance/loopMul",
+	"stTimeConsuming/CALLBlake2f_MaxRounds",
 ];
 
 /// Check if a path should be skipped.
 /// It checks:
-/// - path/and_file_stam - check path and file name (without extention)
+/// - path/and_file_stem - check path and file name (without extention)
 /// - path/with/sub/path - recursively check path
 fn should_skip(path: &Path) -> bool {
 	let matches = |case: &str| {

evm-tests/jsontests/src/state.rs

@@ -675,7 +675,7 @@ fn test_run(
 			let h = states.first().unwrap().hash.0;
 			// if vicinity could not be computed then the transaction was invalid so we simply
 			// check the original state and move on
-			let (is_valid_hash, actual_hash) = crate::utils::assert_valid_hash(&h, &original_state);
+			let (is_valid_hash, actual_hash) = crate::utils::check_valid_hash(&h, &original_state);
 			if !is_valid_hash {
 				tests_result.failed_tests.push(FailedTestDetails {
 					expected_hash: h,
@@ -860,7 +860,7 @@ fn test_run(
 				panic!("unexpected validation for test {name}-{i}")
 			}
 			let (is_valid_hash, actual_hash) =
-				crate::utils::assert_valid_hash(&state.hash.0, backend.state());
+				crate::utils::check_valid_hash(&state.hash.0, backend.state());
 			if !is_valid_hash {
 				let failed_res = FailedTestDetails {
 					expected_hash: state.hash.0,
@@ -902,7 +902,7 @@ fn test_run(
 					}
 				}
 			} else if verbose_output.very_verbose && !verbose_output.verbose_failed {
-				println!(" [{:?}]  {}:{} ... passed", spec, name, i);
+				println!(" [{spec:?}]  {name}:{i} ... passed");
 			}
 		}
 	}