diff --git a/Cargo.lock b/Cargo.lock index ee40f336911dd..5d8eafd9c1b4d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3479,15 +3479,19 @@ dependencies = [ "inferno", "itertools 0.14.0", "mockall", + "num-bigint", "opener", "parking_lot", "paste", "path-slash", "proptest", "quick-junit", + "rand 0.8.5", "rayon", "regex", "reqwest", + "revm", + "rstest", "semver 1.0.26", "serde", "serde_json", @@ -4372,6 +4376,12 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988" +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + [[package]] name = "futures-util" version = "0.3.31" @@ -7391,6 +7401,12 @@ version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +[[package]] +name = "relative-path" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2" + [[package]] name = "reqwest" version = "0.12.15" @@ -7591,6 +7607,36 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "rstest" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fc39292f8613e913f7df8fa892b8944ceb47c247b78e1b1ae2f09e019be789d" +dependencies = [ + "futures-timer", + "futures-util", + "rstest_macros", + "rustc_version 0.4.1", +] + +[[package]] +name = "rstest_macros" +version = "0.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f168d99749d307be9de54d23fd226628d99768225ef08f6ffb52e0182a27746" +dependencies = [ + "cfg-if", + "glob", + "proc-macro-crate", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version 0.4.1", + "syn 2.0.100", + "unicode-ident", +] + [[package]] name = "rtoolbox" version = "0.0.2" diff --git a/cache/solidity-files-cache.json b/cache/solidity-files-cache.json new file mode 100644 index 0000000000000..62081cb1cc34f --- /dev/null +++ b/cache/solidity-files-cache.json @@ -0,0 +1 @@ +{"_format":"","paths":{"artifacts":"out","build_infos":"out/build-info","sources":"src","tests":"test","scripts":"script","libraries":["lib"]},"files":{"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"409e7881b1f12b7eda886da0aadf38bd","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_BitNot/src/Counter.sol/Counter.json","build_id":"611b06cf8f63cf633d8f3e1cf644bbfe"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_BitNot/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_BitNot/test/CounterTest.t.sol/CounterTest.json","build_id":"611b06cf8f63cf633d8f3e1cf644bbfe"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"23ec06146f4e636071d9d2f687bb39b8","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_PostDec/src/Counter.sol/Counter.json","build_id":"b733849b9e35a95f8a64b14dda9151ef"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PostDec/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"mutation_211_203_UnaryOperator_PostDec/test/CounterTest.t.sol/CounterTest.json","build_id":"b733849b9e35a95f8a64b14dda9151ef"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"fa61ead0fe3f3d3cf9a74043ca8a1cca","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"src/Counter.sol/Counter.json","build_id":"2068d3446326d754b730a2df5ea568f8"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreDec/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"test/CounterTest.t.sol/CounterTest.json","build_id":"2068d3446326d754b730a2df5ea568f8"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol":{"lastModificationDate":1743435364548,"contentHash":"1129c6e517e1881612a094060e2866bd","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol","imports":[],"versionRequirement":"^0.8.13","artifacts":{"Counter":{"0.8.28":{"default":{"path":"Counter.sol/Counter.json","build_id":"32fa3bbd0fbf144bf1fd0af3ea20e90e"}}}},"seenByCompiler":true},"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/test/CounterTest.t.sol":{"lastModificationDate":1743435364534,"contentHash":"7ca0c40d195ed5b4cdc882be42cba837","sourceName":"/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/test/CounterTest.t.sol","imports":["/private/var/folders/zt/hvj1qf5j1fv7vrdb2c8hxr7m0000gn/T/.tmpADxVq0/Counter_sol/mutation_211_203_UnaryOperator_PreInc/src/Counter.sol"],"versionRequirement":"^0.8.13","artifacts":{"CounterTest":{"0.8.28":{"default":{"path":"CounterTest.t.sol/CounterTest.json","build_id":"32fa3bbd0fbf144bf1fd0af3ea20e90e"}}}},"seenByCompiler":true}},"builds":["2068d3446326d754b730a2df5ea568f8","32fa3bbd0fbf144bf1fd0af3ea20e90e","611b06cf8f63cf633d8f3e1cf644bbfe","b733849b9e35a95f8a64b14dda9151ef"],"profiles":{"default":{"solc":{"optimizer":{"enabled":false,"runs":200},"metadata":{"useLiteralContent":false,"bytecodeHash":"ipfs","appendCBOR":true},"outputSelection":{"*":{"*":["abi","evm.bytecode.object","evm.bytecode.sourceMap","evm.bytecode.linkReferences","evm.deployedBytecode.object","evm.deployedBytecode.sourceMap","evm.deployedBytecode.linkReferences","evm.deployedBytecode.immutableReferences","evm.methodIdentifiers","metadata"]}},"evmVersion":"cancun","viaIR":false,"libraries":{}},"vyper":{"evmVersion":"cancun","outputSelection":{"*":{"*":["abi","evm.bytecode","evm.deployedBytecode"]}}}}}} \ No newline at end of file diff --git a/crates/forge/Cargo.toml b/crates/forge/Cargo.toml index f2265703f879e..dad17bb02d739 100644 --- a/crates/forge/Cargo.toml +++ b/crates/forge/Cargo.toml @@ -67,14 +67,17 @@ indicatif.workspace = true inferno = { version = "0.12", default-features = false } itertools.workspace = true parking_lot.workspace = true +rand.workspace = true regex = { workspace = true, default-features = false } reqwest = { workspace = true, features = ["json"] } +revm.workspace = true semver.workspace = true serde_json.workspace = true similar = { version = "2", features = ["inline"] } solang-parser.workspace = true solar-parse.workspace = true strum = { workspace = true, features = ["derive"] } +tempfile.workspace = true thiserror.workspace = true tokio = { workspace = true, features = ["time"] } toml = { workspace = true, features = ["preserve_order"] } @@ -84,6 +87,7 @@ watchexec-events = "5.0" watchexec-signals = "4.0" clearscreen = "4.0" evm-disassembler.workspace = true +num-bigint = "0.4" # doc server axum = { workspace = true, features = ["ws"] } @@ -105,6 +109,7 @@ mockall = "0.13" globset = "0.4" paste = "1.0" path-slash = "0.2" +rstest = "0.25.0" similar-asserts.workspace = true svm = { package = "svm-rs", version = "0.5", default-features = false, features = [ "rustls", diff --git a/crates/forge/src/cmd/test/mod.rs b/crates/forge/src/cmd/test/mod.rs index 4fc3ef2a8e7d9..53e05f3c156a9 100644 --- a/crates/forge/src/cmd/test/mod.rs +++ b/crates/forge/src/cmd/test/mod.rs @@ -3,6 +3,7 @@ use crate::{ decode::decode_console_logs, gas_report::GasReport, multi_runner::matches_contract, + mutation::{MutationHandler, MutationReporter, MutationsSummary}, result::{SuiteResult, TestOutcome, TestStatus}, traces::{ debug::{ContractSources, DebugTraceIdentifier}, @@ -18,7 +19,7 @@ use clap::{Parser, ValueHint}; use eyre::{bail, Context, OptionExt, Result}; use foundry_cli::{ opts::{BuildOpts, GlobalArgs}, - utils::{self, LoadConfig}, + utils::{self, FoundryPathExt, LoadConfig}, }; use foundry_common::{compile::ProjectCompiler, evm::EvmArgs, fs, shell, TestFunctionExt}; use foundry_compilers::{ @@ -193,6 +194,19 @@ pub struct TestArgs { #[command(flatten)] pub watch: WatchArgs, + + /// Enable mutation testing. + /// If passed with file paths, only those files will be tested. + #[arg(long, num_args(0..), value_name = "PATH")] + pub mutate: Option>, + + /// Specify which files to mutate with glob pattern matching. + #[arg(long, value_name = "PATTERN", requires = "mutate")] + pub mutate_path: Option, + + /// Only run tests in contracts matching the specified regex pattern. + #[arg(long, value_name = "REGEX", requires = "mutate")] + pub mutate_contract: Option, } impl TestArgs { @@ -291,6 +305,13 @@ impl TestArgs { config.invariant.gas_report_samples = 0; } + let should_mutate = self.mutate.is_some(); + + // Mutation test uses cache to avoid recompiling non-mutated contracts -> force it + if should_mutate && !config.cache { + config.cache = true; + } + // Install missing dependencies. if install::install_missing_dependencies(&mut config) && config.auto_detect_remappings { // need to re-configure here to also catch additional remappings @@ -349,10 +370,11 @@ impl TestArgs { .with_fork(evm_opts.get_fork(&config, env.clone())) .enable_isolation(evm_opts.isolate) .odyssey(evm_opts.odyssey) - .build::(project_root, &output, env, evm_opts)?; + .build::(project_root, &output, env.clone(), evm_opts.clone())?; let libraries = runner.libraries.clone(); - let mut outcome = self.run_tests(runner, config, verbosity, &filter, &output).await?; + let mut outcome = + self.run_tests(runner, config.clone(), verbosity, &filter, &output).await?; if should_draw { let (suite_name, test_name, mut test_result) = @@ -416,13 +438,109 @@ impl TestArgs { } let mut debugger = builder.build(); - if let Some(dump_path) = self.dump { + if let Some(dump_path) = self.dump.clone() { debugger.dump_to_file(&dump_path)?; } else { debugger.try_run_tui()?; } } + // All test have been run once before reaching this point + if should_mutate { + // check outcome here, stop if any test failed + // @todo rather set non-allowed failed tests in config and ensure_ok() here? + // @todo other checks: no fork (or just exclude based on clap arg?) + if outcome.failed() > 0 { + eyre::bail!("Cannot run mutation testing with failed tests"); + } + + let mutate_paths = if let Some(pattern) = &self.mutate_path { + // If --mutate-path is provided, use it to filter paths + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + // @todo filter out interfaces here? + // we do it in lexing for now + entry.is_sol() && !entry.is_sol_test() && pattern.is_match(entry) + }) + .collect() + } else if let Some(contract_pattern) = &self.mutate_contract { + // If --mutate-contract is provided, use it to filter contracts + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| { + entry.is_sol() && + !entry.is_sol_test() && + output + .artifact_ids() + .find(|(id, _)| id.source == *entry) + .is_some_and(|(id, _)| contract_pattern.is_match(&id.name)) + }) + .collect() + } else if self.mutate.as_ref().unwrap().is_empty() { + // If --mutate is passed without arguments, use all Solidity files + source_files_iter(&project.paths.sources, MultiCompilerLanguage::FILE_EXTENSIONS) + .filter(|entry| entry.is_sol() && !entry.is_sol_test()) + .collect() + } else { + // If --mutate is passed with arguments, use those paths + self.mutate.as_ref().unwrap().clone() + }; + + sh_println!("Running mutation tests...").unwrap(); + let mut mutation_summary = MutationsSummary::new(); + + for path in mutate_paths { + let mut handler = MutationHandler::new(path, config.clone()); + + handler.read_source_contract()?; + handler.generate_ast().await; + handler.create_mutation_folders(); + + let mutants = handler.generate_and_compile().await; + + // @todo ugly - needs to be refactored + for mutant in mutants { + if let Some(compile_output) = mutant.1 { + let mutant_path = mutant.0.path.clone(); + + let mut new_config = (*config).clone(); + + new_config.root = mutant_path.clone(); + new_config.src = mutant_path.clone().join(config.src.file_name().unwrap()); + new_config.out = mutant_path.clone().join(config.out.file_name().unwrap()); + new_config.cache_path = + mutant_path.clone().join(config.cache_path.file_name().unwrap()); + + let new_config = Arc::new(new_config); + let new_filter = self.filter(&new_config).unwrap(); + + let mut runner = MultiContractRunnerBuilder::new(new_config.clone()) + .set_debug(false) + .initial_balance(evm_opts.initial_balance) + .evm_spec(config.evm_spec_id()) + .sender(evm_opts.sender) + .odyssey(evm_opts.odyssey) + .build::( + &mutant_path, + &compile_output, + env.clone(), + evm_opts.clone(), + )?; + + let results = runner.test_collect(&new_filter); + + let outcome = TestOutcome::new(results, self.allow_failure); + mutation_summary.update_valid_mutant(&outcome); + } else { + mutation_summary.update_invalid_mutant(); + } + } + } + + MutationReporter::new().report(&mutation_summary); + + outcome = TestOutcome::empty(true); + } + Ok(outcome) } diff --git a/crates/forge/src/lib.rs b/crates/forge/src/lib.rs index 4c7d135767719..da9ab28b693f8 100644 --- a/crates/forge/src/lib.rs +++ b/crates/forge/src/lib.rs @@ -19,6 +19,8 @@ pub mod gas_report; pub mod multi_runner; pub use multi_runner::{MultiContractRunner, MultiContractRunnerBuilder}; +pub mod mutation; + mod runner; pub use runner::ContractRunner; diff --git a/crates/forge/src/mutation/mod.rs b/crates/forge/src/mutation/mod.rs new file mode 100644 index 0000000000000..f35f9c24b6671 --- /dev/null +++ b/crates/forge/src/mutation/mod.rs @@ -0,0 +1,265 @@ +mod mutant; +mod mutators; +mod reporter; +mod visitor; + +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use solar_parse::{ + ast::interface::{source_map::FileName, Session}, + Parser, +}; +use std::sync::Arc; + +use crate::mutation::{mutant::Mutant, visitor::MutantVisitor}; + +pub use crate::mutation::reporter::MutationReporter; + +use crate::result::TestOutcome; +use foundry_compilers::{project::ProjectCompiler, ProjectCompileOutput}; +use foundry_config::Config; +use rayon::prelude::*; +use solar_parse::ast::visit::Visit; +use std::path::{Path, PathBuf}; +use tempfile::TempDir; +pub struct MutationsSummary { + total: usize, + dead: usize, + survived: usize, + invalid: usize, +} + +impl MutationsSummary { + pub fn new() -> Self { + Self { total: 0, dead: 0, survived: 0, invalid: 0 } + } + + pub fn update_valid_mutant(&mut self, outcome: &TestOutcome) { + self.total += 1; + + if outcome.failures().count() > 0 { + self.dead += 1; + } else { + self.survived += 1; + } + } + + pub fn update_invalid_mutant(&mut self) { + self.total += 1; + self.invalid += 1; + } + + pub fn total(&self) -> usize { + self.total + } + + pub fn dead(&self) -> usize { + self.dead + } + + pub fn survived(&self) -> usize { + self.survived + } + + pub fn invalid(&self) -> usize { + self.invalid + } +} + +pub struct MutationHandler { + contract_to_mutate: PathBuf, + src: Arc, + mutations: Vec, + config: Arc, + report: MutationsSummary, + // Ensure we don't clean it between creation and mutant generation (been there, done that) + temp_dir: Option, +} + +impl MutationHandler { + pub fn new(contract_to_mutate: PathBuf, config: Arc) -> Self { + Self { + contract_to_mutate, + src: Arc::default(), + mutations: vec![], + config, + temp_dir: None, + report: MutationsSummary::new(), + } + } + + /// Keep the source contract in memory (in the hashmap), as we'll use it to create the mutants + /// in spooled tmp files + pub fn read_source_contract(&mut self) -> Result<(), std::io::Error> { + let content = std::fs::read_to_string(&self.contract_to_mutate)?; + self.src = Arc::new(content); + Ok(()) + } + + /// Read a source string, and for each contract found, gets its ast and visit it to list + /// all mutations to conduct + pub async fn generate_ast(&mut self) { + let path = &self.contract_to_mutate; + let target_content = Arc::clone(&self.src); + let sess = Session::builder().with_silent_emitter(None).build(); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let arena = solar_parse::ast::Arena::new(); + let mut parser = + Parser::from_lazy_source_code(&sess, &arena, FileName::from(path.clone()), || { + Ok((*target_content).to_string()) + })?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + let mut mutant_visitor = MutantVisitor::default(); + mutant_visitor.visit_source_unit(&ast); + self.mutations.extend(mutant_visitor.mutation_to_conduct); + Ok(()) + }); + } + + /// Create a folder for each mutation, naming based on the type and span + pub fn create_mutation_folders(&mut self) { + let temp_dir_root = tempfile::tempdir().unwrap(); + let target_contract_path = &self.contract_to_mutate; + + for mutant in &mut self.mutations { + let mutation_dir = temp_dir_root + .path() + .join( + target_contract_path + .file_name() + .unwrap_or_default() + .to_string_lossy() + .replace('.', "_"), + ) + .join(format!("mutation_{}", mutant.get_unique_id())); + std::fs::create_dir_all(&mutation_dir).expect("Failed to create mutation directory"); + + let config = Arc::clone(&self.config); + Self::copy_origin(&mutation_dir, target_contract_path, config); + + mutant.path = mutation_dir; + } + + self.temp_dir = Some(temp_dir_root); + } + + /// Emit the solidity of the mutated contract, write it to disk and (try to) compile it + pub async fn generate_and_compile(&self) -> Vec<(&Mutant, Option)> { + let src_path = &self.contract_to_mutate; + + self.mutations.iter().for_each(|mutant| { + self.generate_mutant(mutant, src_path); + }); + + self.mutations + .par_iter() + .map(|mutant| { + if let Some(output) = self.compile_mutant(mutant) { + (mutant, Some(output)) + } else { + (mutant, None) + } + }) + .collect() + } + + /// Copy the src, cache and out folders to one of the mutant temp folder + /// @todo use symlinks for the untouched part of the src folder + fn copy_origin(path: &Path, src_contract_path: &Path, config: Arc) { + let cache_src = &config.cache_path; + let out_src = &config.out; + let contract_src = &config.src; + + let cache_dest = path.join(cache_src.file_name().unwrap()); + let out_dest = path.join(out_src.file_name().unwrap()); + let contract_dest = path.join(contract_src.file_name().unwrap()); + + std::fs::create_dir_all(&cache_dest).expect("Failed to create temp cache directory"); + std::fs::create_dir_all(&out_dest).expect("Failed to create temp out directory"); + std::fs::create_dir_all(&contract_dest).expect("Failed to create temp src directory"); + + Self::copy_dir_except(cache_src, cache_dest, src_contract_path) + .expect("Failed to copy in temp cache"); + Self::copy_dir_except(out_src, out_dest, src_contract_path) + .expect("Failed to copy in temp out directory"); + Self::copy_dir_except(contract_src, contract_dest, src_contract_path) + .expect("Failed to copy in temp src directory"); + } + + /// Recursively copy all files except one, from a src to a dst folder + fn copy_dir_except( + src: impl AsRef, + dst: impl AsRef, + except: &Path, + ) -> std::io::Result<()> { + std::fs::create_dir_all(&dst)?; + + for entry in std::fs::read_dir(src)? { + let entry = entry?; + let ty = entry.file_type()?; + + if ty.is_dir() { + Self::copy_dir_except(entry.path(), dst.as_ref().join(entry.file_name()), except)?; + } else if entry.file_name() != except.file_name().unwrap_or_default() { + // std::os::unix::fs::symlink(entry.path(), + // &dst.as_ref().join(entry.file_name()))?; // and for windows, would be + // std::os::windows::fs::symlink_file + std::fs::copy(entry.path(), dst.as_ref().join(entry.file_name()))?; + } + } + Ok(()) + } + + /// Based on a given mutation, emit the corresponding mutated solidity code and write it to disk + fn generate_mutant(&self, mutation: &Mutant, src_contract_path: &Path) { + let temp_dir_path = &mutation.path; + + let span = mutation.span; + let replacement = mutation.mutation.to_string(); + + let target_path = temp_dir_path + .ancestors() + .next() + .unwrap() + .join("src") + .join(src_contract_path.file_name().unwrap()); + let src_content = Arc::clone(&self.src); + + let start_pos = span.lo().0 as usize; + let end_pos = span.hi().0 as usize; + + let before = &src_content[..start_pos]; + let after = &src_content[end_pos..]; + + let mut new_content = String::with_capacity(before.len() + replacement.len() + after.len()); + new_content.push_str(before); + new_content.push_str(&replacement); + new_content.push_str(after); + + std::fs::write(&target_path, new_content) + .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &target_path)); + } + + /// Compile a directory and get the compilation output + fn compile_mutant(&self, mutant: &Mutant) -> Option { + let temp_folder = &mutant.path; + + let mut config = (*self.config).clone(); + config.src = temp_folder.clone(); + config.cache_path = temp_folder.join("cache"); + config.out = temp_folder.join("out"); + let project = config.project().unwrap(); + + let compiler = ProjectCompiler::new(&project).unwrap(); + + let output = compiler.compile().unwrap(); + + match output.has_compiler_errors() { + true => None, + false => Some(output), + } + } +} diff --git a/crates/forge/src/mutation/mutant.rs b/crates/forge/src/mutation/mutant.rs new file mode 100644 index 0000000000000..f0fb9f679420c --- /dev/null +++ b/crates/forge/src/mutation/mutant.rs @@ -0,0 +1,196 @@ +// Generate mutants then run tests (reuse the whole unit test flow for now, including compilation to +// select mutants) Use Solar: +use super::visitor::AssignVarTypes; +use solar_parse::ast::{BinOpKind, LitKind, Span, UnOpKind}; +use std::{fmt::Display, path::PathBuf}; + +/// Wraps an unary operator mutated, to easily store pre/post-fix op swaps +#[derive(Debug)] +pub struct UnaryOpMutated { + /// String containing the whole new expression (operator and its target) + /// eg `a++` + new_expression: String, + + /// The underlying operator used by this mutant + pub resulting_op_kind: UnOpKind, +} + +impl UnaryOpMutated { + pub fn new(new_expression: String, resulting_op_kind: UnOpKind) -> Self { + Self { new_expression, resulting_op_kind } + } +} + +impl Display for UnaryOpMutated { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}", self.new_expression) + } +} + +// @todo add a mutation from universalmutator: line swap (swap two lines of code, as it +// could theoretically uncover untested reentrancies +#[derive(Debug)] +pub enum MutationType { + // @todo Solar doesn't differentiate numeric type in LitKind (only on declaration?) -> for + // now, planket and let solc filter out the invalid mutants -> we might/should add a + // hashtable of the var to their underlying type (signed or not) so we avoid *a lot* of + // invalid mutants + /// For an initializer x, of type + /// bool: replace x with !x + /// uint: replace x with 0 + /// int: replace x with 0; replace x with -x (temp: this is mutated for uint as well) + /// + /// For a binary op y: apply BinaryOp(y) + Assignment(AssignVarTypes), + + /// For a binary op y in BinOpKind ("+", "-", ">=", etc) + /// replace y with each non-y in op + BinaryOp(BinOpKind), + + /// For a delete expr x `delete foo`, replace x with `assert(true)` + DeleteExpression, + + /// replace "delegatecall" with "call" + ElimDelegate, + + /// Gambit doesn't implement nor define it? + FunctionCall, + + // /// For a if(x) condition x: + // /// replace x with true; replace x with false + // This mutation is not used anymore, as we mutate the condition as an expression, + // which will creates true/false mutant as well as more complex conditions (eg if(foo++ > + // --bar) ) IfStatementMutation, + /// For a require(x) condition: + /// replace x with true; replace x with false + // Same as for IfStatementMutation, the expression inside the require is mutated as an + // expression to handle increment etc + Require, + + // @todo review if needed -> this might creates *a lot* of combinations for super-polyadic fn + // tho only swapping same type (to avoid obvious compilation failure), but should + // take into account implicit casting too... + /// For 2 args of the same type x,y in a function args: + /// swap(x, y) + SwapArgumentsFunction, + + // @todo same remark as above, might end up in a space too big to explore + filtering out + // based on type + /// For an expr taking 2 expression x, y (x+y, x-y, x = x + ...): + /// swap(x, y) + SwapArgumentsOperator, + + /// For an unary operator x in UnOpKind (eg "++", "--", "~", "!"): + /// replace x with all other operator in op + /// Pre or post- are different UnOp + UnaryOperator(UnaryOpMutated), +} + +impl MutationType { + fn get_name(&self) -> String { + match self { + Self::Assignment(var_type) => match var_type { + AssignVarTypes::Literal(kind) => { + format!("{}_{}", "Assignment", kind.description()) + } + AssignVarTypes::Identifier(ident) => { + format!("{}_{}", "Assignment", ident) + } + }, + Self::BinaryOp(kind) => { + format!("{}_{:?}", "BinaryOp", kind) + } + Self::DeleteExpression => "DeleteExpression".to_string(), + Self::ElimDelegate => "ElimDelegate".to_string(), + Self::FunctionCall => "FunctionCall".to_string(), + Self::Require => "Require".to_string(), + Self::SwapArgumentsFunction => "SwapArgumentsFunction".to_string(), + Self::SwapArgumentsOperator => "SwapArgumentsOperator".to_string(), + Self::UnaryOperator(mutated) => { + // avoid operator in tmp dir name + format!("{}_{:?}", "UnaryOperator", mutated.resulting_op_kind) + } + } + } +} + +impl Display for MutationType { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + Self::Assignment(kind) => match kind { + AssignVarTypes::Literal(kind) => match kind { + LitKind::Number(val) => write!(f, "{val}"), + _ => todo!(), + }, + AssignVarTypes::Identifier(ident) => write!(f, "{ident}"), + }, + Self::BinaryOp(kind) => write!(f, "{}", kind.to_str()), + Self::DeleteExpression => write!(f, "assert(true)"), + Self::ElimDelegate => write!(f, "call"), + Self::UnaryOperator(mutated) => write!(f, "{mutated}"), + + Self::FunctionCall | + Self::Require | + Self::SwapArgumentsFunction | + Self::SwapArgumentsOperator => write!(f, ""), + } + } +} + +#[derive(Debug)] +pub enum MutationResult { + Dead, + Alive, + Invalid, +} + +/// A given mutation +#[derive(Debug)] +pub struct Mutant { + /// The path to the project root where this mutant (tries to) live + pub path: PathBuf, + pub span: Span, + pub mutation: MutationType, +} + +impl Mutant { + /// Get a temp folder name based on the span and the mutation to conduct + pub fn get_unique_id(&self) -> String { + format!( + "{}_{}_{}", + self.span.hi().to_u32(), + self.span.lo().to_u32(), + self.mutation.get_name() + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use solar_parse::ast::{BinOpKind, LitKind, Span, UnOpKind}; + + #[test] + fn test_mutation_type_get_name() { + assert_eq!(MutationType::DeleteExpression.get_name(), "DeleteExpression"); + assert_eq!(MutationType::ElimDelegate.get_name(), "ElimDelegate"); + assert_eq!(MutationType::FunctionCall.get_name(), "FunctionCall"); + assert_eq!(MutationType::Require.get_name(), "Require"); + assert_eq!(MutationType::SwapArgumentsFunction.get_name(), "SwapArgumentsFunction"); + assert_eq!(MutationType::SwapArgumentsOperator.get_name(), "SwapArgumentsOperator"); + + assert_eq!(MutationType::BinaryOp(BinOpKind::Add).get_name(), "BinaryOp_Add"); + + let lit_num = LitKind::Number(123.into()); + assert_eq!( + MutationType::Assignment(AssignVarTypes::Literal(lit_num)).get_name(), + "Assignment_number" + ); + + let ident = AssignVarTypes::Identifier("myVar".to_string()); + assert_eq!(MutationType::Assignment(ident).get_name(), "Assignment_myVar"); + + let unary_mutated = UnaryOpMutated::new("a--".to_string(), UnOpKind::PreInc); + assert_eq!(MutationType::UnaryOperator(unary_mutated).get_name(), "UnaryOperator_PreInc"); + } +} diff --git a/crates/forge/src/mutation/mutators/assignement_mutator.rs b/crates/forge/src/mutation/mutators/assignement_mutator.rs new file mode 100644 index 0000000000000..a8905d9e5ab7a --- /dev/null +++ b/crates/forge/src/mutation/mutators/assignement_mutator.rs @@ -0,0 +1,113 @@ +use crate::mutation::{ + mutant::{Mutant, MutationType}, + mutators::{MutationContext, Mutator}, + visitor::AssignVarTypes, +}; + +use eyre::Result; +use solar_parse::ast::{Expr, ExprKind, LitKind, Span}; +use std::path::PathBuf; + +pub struct AssignmentMutator; + +impl Mutator for AssignmentMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let (assign_var_type, replacement_span) = match extract_rhs_info(context) { + Some(info) => info, + None => return Ok(vec![]), // is_applicable should filter this + }; + + match assign_var_type { + AssignVarTypes::Literal(lit) => match lit { + LitKind::Bool(val) => Ok(vec![Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal(LitKind::Bool( + !val, + ))), + path: PathBuf::default(), + }]), + LitKind::Number(val) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + LitKind::Number(num_bigint::BigInt::ZERO), + )), + path: PathBuf::default(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal( + LitKind::Number(-val), + )), + path: PathBuf::default(), + }, + ]), + _ => { + eyre::bail!("AssignmentMutator: unhandled literal kind on RHS: {:?}", lit) + } + }, + AssignVarTypes::Identifier(ident) => Ok(vec![ + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Literal(LitKind::Number( + num_bigint::BigInt::ZERO, + ))), + path: PathBuf::default(), + }, + Mutant { + span: replacement_span, + mutation: MutationType::Assignment(AssignVarTypes::Identifier(format!( + "-{ident}" + ))), + path: PathBuf::default(), + }, + ]), + } + } + + /// Match is the expr is an assign with a var definiton having a literal or identifier as + /// initializer + fn is_applicable(&self, context: &MutationContext<'_>) -> bool { + if let Some(expr) = context.expr { + if let ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) = &expr.kind { + matches!((&**rhs_actual_expr).kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // Not an assign + } + } else if let Some(var_definition) = context.var_definition { + if let Some(init) = &var_definition.initializer { + matches!(&init.kind, ExprKind::Lit(..) | ExprKind::Ident(..)) + } else { + false // No initializer + } + } else { + false // Not an expression or var_definition + } + } +} + +fn extract_rhs_info(context: &MutationContext<'_>) -> Option<(AssignVarTypes, Span)> { + let relevant_expr_for_rhs = if let Some(var_definition) = context.var_definition { + var_definition.initializer.as_ref()? + } else if let Some(expr) = context.expr { + match &expr.kind { + ExprKind::Assign(_lhs, _op_opt, rhs_actual_expr) => &**rhs_actual_expr, + // If the context.expr is already what we want to get the type from + // (e.g. a simple Lit or Ident being passed directly, though is_applicable filters this) + ExprKind::Lit(..) | ExprKind::Ident(..) => expr, + _ => return None, + } + } else { + return None; // No var_definition or expr in context (shouldn't happen?) + }; + + match &relevant_expr_for_rhs.kind { + ExprKind::Lit(kind, _) => { + Some((AssignVarTypes::Literal(kind.kind.clone()), relevant_expr_for_rhs.span)) + } + ExprKind::Ident(val) => { + Some((AssignVarTypes::Identifier(val.to_string()), relevant_expr_for_rhs.span)) + } + _ => None, + } +} diff --git a/crates/forge/src/mutation/mutators/binary_op_mutator.rs b/crates/forge/src/mutation/mutators/binary_op_mutator.rs new file mode 100644 index 0000000000000..3b6ee71636459 --- /dev/null +++ b/crates/forge/src/mutation/mutators/binary_op_mutator.rs @@ -0,0 +1,82 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use eyre::{OptionExt, Result}; +use solar_parse::ast::{BinOp, BinOpKind, ExprKind}; +use std::path::PathBuf; + +pub struct BinaryOpMutator; + +// @todo Add the other way to get there + +impl Mutator for BinaryOpMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let bin_op = get_bin_op(context)?; + let op = bin_op.kind; + + let operations_bools = vec![ + // Bool + BinOpKind::Lt, + BinOpKind::Le, + BinOpKind::Gt, + BinOpKind::Ge, + BinOpKind::Eq, + BinOpKind::Ne, + BinOpKind::Or, + BinOpKind::And, + ]; // this cover the "if" mutations, as every other mutant is tested, at least once + // @todo to optimize -> replace whole stmt (need new visitor override for visit_stmt tho) + // with true/false and skip operations_bools here (mayve some "level"/depth of + // mutation as param?) + + let operations_num_bitwise = vec![ + // Arithm + BinOpKind::Shr, + BinOpKind::Shl, + BinOpKind::Sar, + BinOpKind::BitAnd, + BinOpKind::BitOr, + BinOpKind::BitXor, + BinOpKind::Add, + BinOpKind::Sub, + BinOpKind::Pow, + BinOpKind::Mul, + BinOpKind::Div, + BinOpKind::Rem, + ]; + + let operations = + if operations_bools.contains(&op) { operations_bools } else { operations_num_bitwise }; + + Ok(operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| Mutant { + span: context.span, + mutation: MutationType::BinaryOp(kind), + path: PathBuf::default(), + }) + .collect()) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if ctxt.expr.is_none() { + return false; + } + + match ctxt.expr.unwrap().kind { + ExprKind::Binary(_, _, _) => true, + ExprKind::Assign(_, bin_op, _) => bin_op.is_some(), + _ => false, + } + } +} + +fn get_bin_op(ctxt: &MutationContext<'_>) -> Result { + let expr = ctxt.expr.ok_or_eyre("BinaryOpMutator: unexpected expression")?; + + match expr.kind { + ExprKind::Assign(_, Some(bin_op), _) => Ok(bin_op), + ExprKind::Binary(_, op, _) => Ok(op), + _ => eyre::bail!("BinaryOpMutator: unexpected expression kind"), + } +} diff --git a/crates/forge/src/mutation/mutators/delete_expression_mutator.rs b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs new file mode 100644 index 0000000000000..a04a52e3e9ddb --- /dev/null +++ b/crates/forge/src/mutation/mutators/delete_expression_mutator.rs @@ -0,0 +1,26 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; +use solar_parse::ast::ExprKind; + +use eyre::Result; +use std::path::PathBuf; + +pub struct DeleteExpressionMutator; + +impl Mutator for DeleteExpressionMutator { + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: ctxt.span, + mutation: MutationType::DeleteExpression, + path: PathBuf::default(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { + matches!(expr.kind, ExprKind::Delete(_)) + } else { + false + } + } +} diff --git a/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs new file mode 100644 index 0000000000000..5fde5ab0e8117 --- /dev/null +++ b/crates/forge/src/mutation/mutators/elim_delegate_mutator.rs @@ -0,0 +1,38 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType}; + +use eyre::Result; +use solar_parse::ast::ExprKind; +use std::{fmt::Display, path::PathBuf}; + +pub struct ElimDelegateMutator; + +impl Mutator for ElimDelegateMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + Ok(vec![Mutant { + span: context.span, + mutation: MutationType::ElimDelegate, + path: PathBuf::default(), + }]) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + ctxt.expr + .as_ref() + .and_then(|expr| match &expr.kind { + ExprKind::Call(callee, _) => Some(callee), + _ => None, + }) + .and_then(|callee| match &callee.kind { + ExprKind::Member(_, ident) => Some(ident), + _ => None, + }) + .is_some_and(|ident| ident.to_string() == "delegatecall") + } +} + +impl Display for ElimDelegateMutator { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "") + } +} diff --git a/crates/forge/src/mutation/mutators/mod.rs b/crates/forge/src/mutation/mutators/mod.rs new file mode 100644 index 0000000000000..fceff75b327cb --- /dev/null +++ b/crates/forge/src/mutation/mutators/mod.rs @@ -0,0 +1,75 @@ +pub mod assignement_mutator; +pub mod binary_op_mutator; +pub mod delete_expression_mutator; +pub mod elim_delegate_mutator; +pub mod unary_op_mutator; + +pub mod mutator_registry; + +use solar_parse::ast::{Expr, Span, VariableDefinition}; + +use eyre::Result; + +use crate::mutation::Mutant; + +pub trait Mutator { + /// Generate all mutant corresponding to a given context + fn generate_mutants(&self, ctxt: &MutationContext<'_>) -> Result>; + /// True if a mutator can be applied to an expression/node + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool; +} + +#[derive(Debug)] +pub struct MutationContext<'a> { + pub span: Span, + /// The expression to mutate + pub expr: Option<&'a Expr<'a>>, + + pub var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContext<'a> { + pub fn builder() -> MutationContextBuilder<'a> { + MutationContextBuilder::new() + } +} + +pub struct MutationContextBuilder<'a> { + span: Option, + expr: Option<&'a Expr<'a>>, + var_definition: Option<&'a VariableDefinition<'a>>, +} + +impl<'a> MutationContextBuilder<'a> { + // Create a new empty builder + pub fn new() -> Self { + MutationContextBuilder { span: None, expr: None, var_definition: None } + } + + // Required + pub fn with_span(mut self, span: Span) -> Self { + self.span = Some(span); + self + } + + // Optional + pub fn with_expr(mut self, expr: &'a Expr<'a>) -> Self { + self.expr = Some(expr); + self + } + + // Optional + pub fn with_var_definition(mut self, var_definition: &'a VariableDefinition<'a>) -> Self { + self.var_definition = Some(var_definition); + self + } + + pub fn build(self) -> Result, &'static str> { + let span = self.span.ok_or("Span is required for MutationContext")?; + + Ok(MutationContext { span, expr: self.expr, var_definition: self.var_definition }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/forge/src/mutation/mutators/mutator_registry.rs b/crates/forge/src/mutation/mutators/mutator_registry.rs new file mode 100644 index 0000000000000..c24321458dcf6 --- /dev/null +++ b/crates/forge/src/mutation/mutators/mutator_registry.rs @@ -0,0 +1,40 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::Mutant; + +use super::{ + assignement_mutator, binary_op_mutator, delete_expression_mutator, elim_delegate_mutator, + unary_op_mutator, +}; + +/// Registry of all available mutators (ie implementing the Mutator trait) +pub struct MutatorRegistry { + mutators: Vec>, +} + +impl MutatorRegistry { + pub fn default() -> Self { + let mut registry = Self { mutators: Vec::new() }; + + registry.mutators.push(Box::new(assignement_mutator::AssignmentMutator)); + registry.mutators.push(Box::new(binary_op_mutator::BinaryOpMutator)); + registry.mutators.push(Box::new(delete_expression_mutator::DeleteExpressionMutator)); + registry.mutators.push(Box::new(elim_delegate_mutator::ElimDelegateMutator)); + registry.mutators.push(Box::new(unary_op_mutator::UnaryOperatorMutator)); + + registry + } + + pub fn new_with_mutators(mutators: Vec>) -> Self { + Self { mutators } + } + + /// Find all applicable mutators for a given context and return the corresponding mutations + pub fn generate_mutations(&self, context: &MutationContext<'_>) -> Vec { + self.mutators + .iter() + .filter(|mutator| mutator.is_applicable(context)) + .filter_map(|mutator| mutator.generate_mutants(context).ok()) + .flatten() + .collect() + } +} diff --git a/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs new file mode 100644 index 0000000000000..ff11e843d2009 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/assignement_mutator_test.rs @@ -0,0 +1,24 @@ +use crate::mutation::mutators::{ + assignement_mutator::AssignmentMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for AssignmentMutator {} + +#[rstest] +#[case::assign_lit("x = y", Some(vec!["x = 0", "x = -y"]))] +#[case::assign_number("x = 123", Some(vec!["x = 0", "x = -123"]))] +#[case::assign_bool("x = true", Some(vec!["x = false"]))] +#[case::assign_bool("x = false", Some(vec!["x = true"]))] +#[case::assign_declare("uint256 x = 123", Some(vec!["uint256 x = 0", "uint256 x = -123"]))] +#[case::non_assign("a = b + c", None)] +fn test_mutator_assignment( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: AssignmentMutator = AssignmentMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + AssignmentMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs new file mode 100644 index 0000000000000..84a874d017043 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/binary_op_mutator_test.rs @@ -0,0 +1,31 @@ +use crate::mutation::mutators::{ + binary_op_mutator::BinaryOpMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for BinaryOpMutator {} + +#[rstest] +#[case::add("x + y", Some(vec!["x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::sub("x - y", Some(vec!["x + y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::mul("x * y", Some(vec!["x + y", "x - y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::div("x / y", Some(vec!["x + y", "x - y", "x * y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::modulus("x % y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::pow("x ** y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x << y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_left("x << y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x >> y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right("x >> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >>> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_shift_right_unsigned("x >>> y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x & y", "x | y", "x ^ y"]))] +#[case::bit_and("x & y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x | y", "x ^ y"]))] +#[case::bit_or("x | y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x ^ y"]))] +#[case::bit_xor("x ^ y", Some(vec!["x + y", "x - y", "x * y", "x / y", "x % y", "x ** y", "x << y", "x >> y", "x >>> y", "x & y", "x | y"]))] +#[case::non_binary("a = true", None)] +fn test_mutator_bitwise( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: BinaryOpMutator = BinaryOpMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + BinaryOpMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs new file mode 100644 index 0000000000000..ba94e5295837d --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/delete_expression_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + delete_expression_mutator::DeleteExpressionMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for DeleteExpressionMutator {} + +#[rstest] +#[case::delete_expr("delete x", Some(vec!["x"]))] +#[case::non_delete("a = b + c", None)] +fn test_mutator_delete_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: DeleteExpressionMutator = DeleteExpressionMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + DeleteExpressionMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs new file mode 100644 index 0000000000000..05828beb24822 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/elim_delegate_mutator_test.rs @@ -0,0 +1,20 @@ +use crate::mutation::mutators::{ + elim_delegate_mutator::ElimDelegateMutator, + tests::helper::{MutatorTestCase, MutatorTester}, +}; + +use rstest::*; + +impl MutatorTester for ElimDelegateMutator {} + +#[rstest] +#[case::delegate_expr("address(this).delegatecall{value: 1 ether}(0)", Some(vec!["address(this).call{value: 1 ether}(0)"]))] +#[case::non_delegate("address(this).call{value: 1 ether}(0)", None)] +fn test_mutator_delegate_expr( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: ElimDelegateMutator = ElimDelegateMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + ElimDelegateMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/tests/helper.rs b/crates/forge/src/mutation/mutators/tests/helper.rs new file mode 100644 index 0000000000000..7a9a4e177e489 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/helper.rs @@ -0,0 +1,55 @@ +use crate::mutation::{mutators::Mutator, visitor::MutantVisitor, Session}; +use solar_parse::{ + ast::{interface::source_map::FileName, visit::Visit, Arena}, + Parser, +}; + +use std::path::PathBuf; +pub struct MutatorTestCase<'a> { + /// @dev needs to be in a function, to avoid parsing error from solar + /// eg `let input = "function f() { x = 1; }"` to test x = 1 + pub input: &'a str, + /// All the mutations expected for this input, using this mutator + pub expected_mutations: Option>, +} + +pub trait MutatorTester { + fn test_mutator(mutator: M, test_case: MutatorTestCase<'_>) { + let arena = Arena::new(); + let sess = Session::builder().with_silent_emitter(None).build(); + + // let mut mutations: Vec = Vec::new(); + let mut mutant_visitor = MutantVisitor::new_with_mutators(vec![Box::new(mutator)]); + + let _ = sess.enter(|| -> solar_parse::interface::Result<()> { + let mut parser = Parser::from_lazy_source_code( + &sess, + &arena, + FileName::Real(PathBuf::from(test_case.input)), + || Ok(test_case.input.to_string()), + )?; + + let ast = parser.parse_file().map_err(|e| e.emit())?; + + mutant_visitor.visit_source_unit(&ast); + + let mutations = mutant_visitor.mutation_to_conduct; + + // @todo test mutants content... + if let Some(expected) = test_case.expected_mutations { + assert_eq!(mutations.len(), expected.len()); + + for mutation in mutations { + assert!(expected.contains(&mutation.mutation.to_string().as_str())); + } + } else { + assert_eq!(mutations.len(), 0); + } + + Ok(()) + }); + } +} + +// Implement for unit test module +impl MutatorTester for () {} diff --git a/crates/forge/src/mutation/mutators/tests/mod.rs b/crates/forge/src/mutation/mutators/tests/mod.rs new file mode 100644 index 0000000000000..403ee2be95bd9 --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/mod.rs @@ -0,0 +1,10 @@ +mod assignement_mutator_test; +mod binary_op_mutator_test; + +mod delete_expression_mutator_test; + +mod elim_delegate_mutator_test; + +mod helper; + +mod unary_op_mutator_test; diff --git a/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs new file mode 100644 index 0000000000000..4c7ece82de2da --- /dev/null +++ b/crates/forge/src/mutation/mutators/tests/unary_op_mutator_test.rs @@ -0,0 +1,26 @@ +use crate::mutation::mutators::{ + tests::helper::{MutatorTestCase, MutatorTester}, + unary_op_mutator::UnaryOperatorMutator, +}; + +use rstest::*; + +impl MutatorTester for UnaryOperatorMutator {} + +#[rstest] +#[case::pre_inc("++x", Some(vec!["--x", "~x", "-x", "x++", "x--"]))] +#[case::pre_dec("--x", Some(vec!["++x", "~x", "-x", "x++", "x--"]))] +#[case::neg("-x", Some(vec!["++x", "--x", "~x", "x++", "x--"]))] +#[case::bit_not("~x", Some(vec!["++x", "--x", "-x", "x++", "x--"]))] +#[case::post_inc("x++",Some(vec!["++x", "--x", "~x", "-x", "x--"]))] +#[case::post_dec("x--",Some(vec!["++x", "--x", "~x", "-x", "x++"]))] +#[case::bool("!x", Some(vec!["x"]))] +#[case::non_unary("a = b + c", None)] +fn test_unary_op_mutator_arithmetic( + #[case] input: &'static str, + #[case] expected_mutations: Option>, +) { + let mutator: UnaryOperatorMutator = UnaryOperatorMutator; + let test_case = MutatorTestCase { input, expected_mutations }; + UnaryOperatorMutator::test_mutator(mutator, test_case); +} diff --git a/crates/forge/src/mutation/mutators/unary_op_mutator.rs b/crates/forge/src/mutation/mutators/unary_op_mutator.rs new file mode 100644 index 0000000000000..e8f4523af2394 --- /dev/null +++ b/crates/forge/src/mutation/mutators/unary_op_mutator.rs @@ -0,0 +1,107 @@ +use super::{MutationContext, Mutator}; +use crate::mutation::mutant::{Mutant, MutationType, UnaryOpMutated}; +use eyre::Result; +use solar_parse::ast::{ExprKind, LitKind, UnOpKind}; +use std::path::PathBuf; + +pub struct UnaryOperatorMutator; + +impl Mutator for UnaryOperatorMutator { + fn generate_mutants(&self, context: &MutationContext<'_>) -> Result> { + let operations = vec![ + UnOpKind::PreInc, // number + UnOpKind::PreDec, // n + UnOpKind::Neg, // n @todo filter this one only for int + UnOpKind::BitNot, // n + ]; + + let post_fixed_operations = vec![UnOpKind::PostInc, UnOpKind::PostDec]; + + let expr = context.expr.unwrap(); + + let target_kind; + let op; + + match &expr.kind { + ExprKind::Unary(un_op, target) => { + target_kind = &target.kind; + op = un_op.kind; + } + _ => unreachable!(), + }; + + let target_content = match target_kind { + ExprKind::Lit(lit, _) => match &lit.kind { + LitKind::Bool(val) => val.to_string(), + LitKind::Number(val) => val.to_string(), + _ => String::new(), + }, + ExprKind::Ident(inner) => inner.to_string(), + ExprKind::Member(expr, ident) => { + match expr.kind { + ExprKind::Ident(inner) => { + format!("{}{}", ident.as_str(), inner.to_string()) + } // @todo not supporting something like a.b[0]++ + _ => String::new(), + } + } + _ => String::new(), + }; + + // Bool has only the Not operator as possible target -> we try removing it + if op == UnOpKind::Not { + return Ok(vec![Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(UnaryOpMutated::new( + target_content.to_string(), + UnOpKind::Not, + )), + path: PathBuf::default(), + }]); + } + + let mut mutations: Vec; + + mutations = operations + .into_iter() + .filter(|&kind| kind != op) + .map(|kind| { + let new_expression = format!("{}{}", kind.to_str(), target_content); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: PathBuf::default(), + } + }) + .collect(); + + mutations.extend(post_fixed_operations.into_iter().filter(|&kind| kind != op).map( + |kind| { + let new_expression = format!("{}{}", target_content, kind.to_str()); + + let mutated = UnaryOpMutated::new(new_expression, kind); + + Mutant { + span: expr.span, + mutation: MutationType::UnaryOperator(mutated), + path: PathBuf::default(), + } + }, + )); + + Ok(mutations) + } + + fn is_applicable(&self, ctxt: &MutationContext<'_>) -> bool { + if let Some(expr) = ctxt.expr { + if let ExprKind::Unary(_, _) = &expr.kind { + return true; + } + } + + false + } +} diff --git a/crates/forge/src/mutation/reporter.rs b/crates/forge/src/mutation/reporter.rs new file mode 100644 index 0000000000000..1d23447f19dcc --- /dev/null +++ b/crates/forge/src/mutation/reporter.rs @@ -0,0 +1,53 @@ +use crate::mutation::MutationsSummary; +use comfy_table::{modifiers::UTF8_ROUND_CORNERS, Attribute, Cell, Color, Row, Table}; +pub struct MutationReporter { + table: Table, +} + +impl MutationReporter { + pub fn new() -> Self { + let mut table = Table::new(); + + table.apply_modifier(UTF8_ROUND_CORNERS); + + table.set_header(vec![ + Cell::new("Status"), + Cell::new("# Mutants"), + Cell::new("% of Total"), + ]); + + Self { table } + } + + pub fn report(&mut self, summary: &MutationsSummary) { + let mut row = Row::new(); + row.add_cell(Cell::new("Survived").fg(Color::Red)) + .add_cell(Cell::new(summary.survived().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.survived() as f64 / summary.total() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Dead").fg(Color::Green)) + .add_cell(Cell::new(summary.dead().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.dead() as f64 / summary.total() as f64 * 100. + ))); + self.table.add_row(row); + + row = Row::new(); + row.add_cell(Cell::new("Invalid").fg(Color::Green)) + .add_cell(Cell::new(summary.invalid().to_string())) + .add_cell(Cell::new(format!( + "{:.2}%", + summary.invalid() as f64 / summary.total() as f64 * 100. + ))); + self.table.add_row(row); + + sh_println!("Total number of mutants generated: {}", summary.total()); + sh_println!("\n{}", self.table); + } +} diff --git a/crates/forge/src/mutation/visitor.rs b/crates/forge/src/mutation/visitor.rs new file mode 100644 index 0000000000000..7ab5a17d91913 --- /dev/null +++ b/crates/forge/src/mutation/visitor.rs @@ -0,0 +1,62 @@ +use crate::mutation::mutators::Mutator; +use solar_parse::ast::{visit::Visit, Expr, LitKind, SourceUnit, VariableDefinition}; +use std::ops::ControlFlow; + +use crate::mutation::{ + mutant::Mutant, + mutators::{mutator_registry::MutatorRegistry, MutationContext}, +}; + +#[derive(Debug, Clone)] +pub enum AssignVarTypes { + Literal(LitKind), + Identifier(String), /* not using Ident as the symbol is slow to convert as to_str() <-- + * maybe will have to switch back if validating more aggressively */ +} + +/// A visitor which collect all expression to mutate as well as the mutation types +pub struct MutantVisitor { + pub mutation_to_conduct: Vec, + pub mutator_registry: MutatorRegistry, +} + +impl MutantVisitor { + /// Use all mutator from registry::default + pub fn default() -> Self { + Self { mutation_to_conduct: Vec::new(), mutator_registry: MutatorRegistry::default() } + } + + /// Use only a set of mutators + pub fn new_with_mutators(mutators: Vec>) -> Self { + Self { + mutation_to_conduct: Vec::new(), + mutator_registry: MutatorRegistry::new_with_mutators(mutators), + } + } +} + +impl<'ast> Visit<'ast> for MutantVisitor { + type BreakValue = (); + + fn visit_variable_definition( + &mut self, + var: &'ast VariableDefinition<'ast>, + ) -> ControlFlow { + let context = MutationContext::builder() + .with_span(var.span) + .with_var_definition(var) + .build() + .unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_variable_definition(var) + } + + fn visit_expr(&mut self, expr: &'ast Expr<'ast>) -> ControlFlow { + let context = + MutationContext::builder().with_span(expr.span).with_expr(expr).build().unwrap(); + + self.mutation_to_conduct.extend(self.mutator_registry.generate_mutations(&context)); + self.walk_expr(expr) + } +} diff --git a/crates/forge/tests/it/main.rs b/crates/forge/tests/it/main.rs index aaa129796a39a..aa692225aa4e9 100644 --- a/crates/forge/tests/it/main.rs +++ b/crates/forge/tests/it/main.rs @@ -8,6 +8,7 @@ mod fs; mod fuzz; mod inline; mod invariant; +mod mutation; mod repros; mod spec; mod vyper; diff --git a/crates/forge/tests/it/mutation.rs b/crates/forge/tests/it/mutation.rs new file mode 100644 index 0000000000000..fda00f2eb6181 --- /dev/null +++ b/crates/forge/tests/it/mutation.rs @@ -0,0 +1,118 @@ +use forge::mutation::MutationHandler; +use forge_script::ScriptArgs; +use foundry_common::shell::{ColorChoice, OutputFormat, OutputMode, Shell}; +use std::sync::Arc; + +#[tokio::test(flavor = "multi_thread")] +async fn test_mutation_test_lifecycle() { + let contract = r#" + // SPDX-License-Identifier: UNLICENSED + pragma solidity ^0.8.13; + + contract Counter { + uint256 public number; + + function increment() public { + number++; + // This should result in 5 mutants: ++number, --number, -number, ~number, number-- + // -number should be invalid + // ++number should be alive + // the rest should be dead + } + }"#; + + let test = r#" + // SPDX-License-Identifier: UNLICENSED + pragma solidity ^0.8.13; + + // Avoid having to manage a libs folder + import {Counter} from "../src/Counter.sol"; + + contract CounterTest { + Counter public counter; + + function setUp() public { + counter = new Counter(); + } + + function test_Increment() public { + uint256 _countBefore = counter.number(); + + counter.increment(); + + assert(counter.number() == _countBefore + 1); + } + }"#; + + let temp_dir = tempfile::tempdir().unwrap(); + + let src_dir = temp_dir.path().join("src"); + std::fs::create_dir_all(&src_dir).expect("Failed to create src directory"); + + let test_dir = temp_dir.path().join("test"); + std::fs::create_dir_all(&test_dir).expect("Failed to create test directory"); + + let cache_dir = temp_dir.path().join("cache"); + std::fs::create_dir_all(&cache_dir).expect("Failed to create test directory"); + + let out_dir = temp_dir.path().join("out"); + std::fs::create_dir_all(&out_dir).expect("Failed to create test directory"); + + std::fs::write(&src_dir.join("Counter.sol"), contract) + .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + + std::fs::write(&test_dir.join("CounterTest.t.sol"), test) + .unwrap_or_else(|_| panic!("Failed to write to target file {:?}", &src_dir)); + + let mut config = foundry_config::Config::default(); + config.cache_path = cache_dir; + config.out = out_dir; + config.src = src_dir.clone(); + config.test = test_dir.clone(); + + let mut mutation_handler = MutationHandler::new(src_dir.join("Counter.sol"), Arc::new(config)); + + mutation_handler.read_source_contract(); + mutation_handler.generate_ast().await; + mutation_handler.create_mutation_folders(); + let mutants = mutation_handler.generate_and_compile().await; + + // Test if we compile and collect the valid/invalid mutants + assert_eq!(mutants.iter().filter(|(_, output)| output.is_none()).count(), 1); + assert_eq!(mutants.iter().filter(|(_, output)| output.is_some()).count(), 4); + + // @todo run the tests + let mut invalids = 0; + let mut alive = 0; + let mut dead = 0; + + // Create a new shell to suppress any script output + let shell = Shell::new_with(OutputFormat::Json, OutputMode::Quiet, ColorChoice::Never, 0); + shell.set(); + + // Run the tests as scripts, for convenience + for mutant in mutants { + if mutant.1.is_some() { + let result = ScriptArgs { + path: mutant.0.path.join("test/CounterTest.t.sol").to_string_lossy().to_string(), + sig: "test_Increment".to_string(), + args: vec![], + ..Default::default() + } + .run_script() + .await; + + if result.is_err() { + dead += 1; + } else { + alive += 1; + } + } else { + invalids += 1; + } + } + + assert_eq!(invalids, 1); + assert_eq!(alive, 1); + assert_eq!(dead, 3); +}