diff --git a/docs/release-notes.d/ptest.md b/docs/release-notes.d/ptest.md new file mode 100644 index 000000000..caca800ec --- /dev/null +++ b/docs/release-notes.d/ptest.md @@ -0,0 +1,8 @@ +- Update ptest to support test selection. Ptest can now be invoked with `list` + to show the available tests and `run` to run them. The `-t` argument will + select specific tests to run. +- Allow sim tests to skip slow tests. By setting `MCUBOOT_SKIP_SLOW_TESTS` in + the environment, the sim will skip two tests that are very slow. In one + instance this reduces the test time from 2 hours to about 5 minutes. These + slow tests are useful, in that they test bad powerdown recovery, but are + inconvenient when testing other areas. diff --git a/ptest/Cargo.lock b/ptest/Cargo.lock index faca3590a..84f0e485c 100644 --- a/ptest/Cargo.lock +++ b/ptest/Cargo.lock @@ -32,7 +32,7 @@ version = "0.2.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", "winapi", ] @@ -58,6 +58,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + [[package]] name = "cc" version = "1.0.73" @@ -83,6 +89,43 @@ dependencies = [ "winapi", ] +[[package]] +name = "clap" +version = "4.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42dfd32784433290c51d92c438bb72ea5063797fc3cc9a21a8c4346bebbb2098" +dependencies = [ + "bitflags", + "clap_derive", + "clap_lex", + "is-terminal", + "once_cell", + "strsim", + "termcolor", +] + +[[package]] +name = "clap_derive" +version = "4.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fddf67631444a3a3e3e5ac51c36a5e01335302de677bd78759eaa90ab1f46644" +dependencies = [ + "heck", + "proc-macro-error", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "033f6b7a4acb1f358c742aaca805c939ee73b4c6209ae4318ec7aca81c42e646" +dependencies = [ + "os_str_bytes", +] + [[package]] name = "env_logger" version = "0.9.0" @@ -124,6 +167,12 @@ version = "0.26.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78cc372d058dcf6d5ecd98510e7fbc9e5aec4d21de70f65fea8fecebcd881bd4" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.1.19" @@ -133,12 +182,29 @@ dependencies = [ "libc", ] +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "humantime" version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9a3a5bfb195931eeb336b2a7b4d761daec841b97f947d34394601737a7bba5e4" +[[package]] +name = "is-terminal" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f23ff5ef2b80d608d61efee834934d862cd92461afc0560dedf493e4c033738b" +dependencies = [ + "hermit-abi 0.3.9", + "libc", + "windows-sys", +] + [[package]] name = "libc" version = "0.2.126" @@ -200,7 +266,7 @@ version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ - "hermit-abi", + "hermit-abi 0.1.19", "libc", ] @@ -213,11 +279,47 @@ dependencies = [ "memchr", ] +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "os_str_bytes" +version = "6.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2355d85b9a3786f481747ced0e0ff2ba35213a1f9bd406ed906554d7af805a1" + +[[package]] +name = "proc-macro-error" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da25490ff9892aab3fcf7c36f08cfb902dd3e71ca0f9f9517bea02a73a5ce38c" +dependencies = [ + "proc-macro-error-attr", + "proc-macro2", + "quote", + "syn", + "version_check", +] + +[[package]] +name = "proc-macro-error-attr" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a1be40180e52ecc98ad80b184934baf3d0d29f979574e439af5a55274b35f869" +dependencies = [ + "proc-macro2", + "quote", + "version_check", +] + [[package]] name = "proc-macro2" -version = "1.0.40" +version = "1.0.79" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd96a1e8ed2596c337f8eae5f24924ec83f5ad5ab21ea8e455d3566c69fbcaf7" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" dependencies = [ "unicode-ident", ] @@ -227,6 +329,7 @@ name = "ptest" version = "0.1.0" dependencies = [ "chrono", + "clap", "env_logger", "failure", "log", @@ -273,6 +376,12 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33ae9eec00137a8eed469fb4148acd9fc6ac8c3f9b110f52cd34698c8b5bfa0e" +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "syn" version = "1.0.98" @@ -328,6 +437,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "wasi" version = "0.10.0+wasi-snapshot-preview1" @@ -365,6 +480,72 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + +[[package]] +name = "windows-targets" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7dd37b7e5ab9018759f893a1952c9420d060016fc19a472b4bb20d1bdd694d1b" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bcf46cf4c365c6f2d1cc93ce535f2c8b244591df96ceee75d8e83deb70a9cac9" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da9f259dd3bcf6990b55bffd094c4f7235817ba4ceebde8e6d11cd0c5633b675" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b474d8268f99e0995f25b9f095bc7434632601028cf86590aea5c8a5cb7801d3" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1515e9a29e5bed743cb4415a9ecf5dfca648ce85ee42e15873c3cd8610ff8e02" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5eee091590e89cc02ad514ffe3ead9eb6b660aedca2183455434b93546371a03" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ca79f2451b49fa9e2af39f0747fe999fcda4f5e241b2898624dca97a1f2177" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32b752e52a2da0ddfbdbcc6fceadfeede4c939ed16d13e648833a61dfb611ed8" + [[package]] name = "yaml-rust" version = "0.4.5" diff --git a/ptest/Cargo.toml b/ptest/Cargo.toml index 4086bf121..bb38f9332 100644 --- a/ptest/Cargo.toml +++ b/ptest/Cargo.toml @@ -12,3 +12,7 @@ log = "0.4.17" num_cpus = "1.13.1" std-semaphore = "0.1" yaml-rust = "0.4" + +[dependencies.clap] +version = "4.0" +features = ["derive"] diff --git a/ptest/src/main.rs b/ptest/src/main.rs index 053448bf8..9a75ccb74 100644 --- a/ptest/src/main.rs +++ b/ptest/src/main.rs @@ -11,6 +11,7 @@ //! jobs->environment->strategy->matric->features use chrono::Local; +use clap::{Parser, Subcommand}; use log::{debug, error, warn}; use std::{ collections::HashSet, @@ -37,7 +38,9 @@ type Result = result::Result; fn main() -> Result<()> { env_logger::init(); - let workflow_text = fs::read_to_string("../.github/workflows/sim.yaml")?; + let args = Cli::parse(); + + let workflow_text = fs::read_to_string(&args.workflow)?; let workflow = YamlLoader::load_from_str(&workflow_text)?; let ncpus = num_cpus::get(); @@ -45,6 +48,18 @@ fn main() -> Result<()> { let matrix = Matrix::from_yaml(&workflow); + let matrix = if args.test.len() == 0 { matrix } else { + matrix.only(&args.test) + }; + + match args.command { + Commands::List => { + matrix.show(); + return Ok(()); + } + Commands::Run => (), + } + let mut children = vec![]; let state = State::new(matrix.envs.len()); let st2 = state.clone(); @@ -76,6 +91,31 @@ fn main() -> Result<()> { Ok(()) } +/// The main Cli. +#[derive(Debug, Parser)] +#[command(name = "ptest")] +#[command(about = "Run MCUboot CI tests stand alone")] +struct Cli { + /// The workflow file to use. + #[arg(short, long, default_value = "../.github/workflows/sim.yaml")] + workflow: String, + + /// The tests to run (defaults to all) + #[arg(short, long)] + test: Vec, + + #[command(subcommand)] + command: Commands, +} + +#[derive(Debug, Subcommand)] +enum Commands { + /// Runs the tests. + Run, + /// List available tests. + List, +} + /// State, for printing status. struct State { running: HashSet, @@ -220,6 +260,27 @@ impl Matrix { envs, } } + + /// Print out all of the feature sets. + fn show(&self) { + for (i, feature) in self.envs.iter().enumerate() { + println!("{:3}. {}", i + 1, feature.simple_textual()); + } + } + + /// Replace this matrix with one that only has the chosen tests in it. Note + /// that the original order is preserved, not that given in `pick`. + fn only(self, pick: &[usize]) -> Self { + let pick: HashSet = pick.iter().cloned().collect(); + let envs: Vec<_> = self + .envs + .into_iter() + .enumerate() + .filter(|(ind, _)| pick.contains(&(ind + 1))) + .map(|(_, item)| item) + .collect(); + Matrix { envs } + } } impl FeatureSet { @@ -278,6 +339,18 @@ impl FeatureSet { buf } + + /// Generate a simpler textual representation, without showing the environment. + fn simple_textual(&self) -> String { + use std::fmt::Write; + + let mut buf = String::new(); + for v in &self.values { + write!(&mut buf, " {}", v).unwrap(); + } + + buf + } } fn lookup_matrix(y: &Yaml) -> Option<&Vec> { diff --git a/sim/src/image.rs b/sim/src/image.rs index 632dfa568..270ef248a 100644 --- a/sim/src/image.rs +++ b/sim/src/image.rs @@ -689,6 +689,10 @@ impl Images { let mut fails = 0; let total_flash_ops = self.total_count.unwrap(); + if skip_slow_test() { + return false; + } + // Let's try an image halfway through. for i in 1 .. total_flash_ops { info!("Try interruption at {}", i); @@ -775,6 +779,10 @@ impl Images { let mut fails = 0; + if skip_slow_test() { + return false; + } + if self.is_swap_upgrade() { for i in 1 .. self.total_count.unwrap() { info!("Try interruption at {}", i); @@ -2314,3 +2322,14 @@ fn test_alignments() -> &'static [usize] { fn test_alignments() -> &'static [usize] { &[32] } + +/// For testing, some of the tests are quite slow. This will query for an +/// environment variable `MCUBOOT_SKIP_SLOW_TESTS`, which can be set to avoid +/// running these tests. +fn skip_slow_test() -> bool { + if let Ok(_) = std::env::var("MCUBOOT_SKIP_SLOW_TESTS") { + true + } else { + false + } +}