misc: add compare test runs tool (#4495)

book/src/03_tools.md

@@ -0,0 +1,35 @@
+# Tools
+
+The `tools` crate contains various utilities useful for development.
+
+## Benchmarking
+
+Run the hardhat test command in a repo 5 times and report the times it took:
+
+```bash
+# From the repo root
+ cargo run --bin tools benchmark -i 5 /repo/path -t "npx hardhat test"
+```
+
+## Compare test run execution times
+
+Create a provider test execution report for the base branch:
+
+```bash
+# From packages/hardhat-core in the base branch
+yarn build && yarn test:provider --reporter json | tee base-test-provider-logs.json
+```
+
+Create a provider test execution report for the candidate branch:
+
+```bash
+# From packages/hardhat-core in the candidate branch
+yarn build && yarn test:provider --reporter json | tee candidate-test-provider-logs.json
+```
+
+Generate a comparison report that will list slower tests in the candidate branch:
+
+```bash
+# From the repo root
+cargo run --bin tools compare-test-runs base-test-provider-logs.json candidate-test-provider-logs.json > comparisions.txt
+```

book/src/SUMMARY.md

@@ -6,5 +6,7 @@
   - [Setup](./01_getting_started/01_setup.md)
   - [Build](./01_getting_started/02_build.md)
   - [Test](./01_getting_started/03_test.md)
+  - [Release](./02_release.md)
+  - [Tools](./03_tools.md)
 
 - [Contributing](./99_contributing.md)

crates/tools/Cargo.toml

@@ -9,5 +9,7 @@ cfg-if = "1.0.0"
 clap = { version = "3.2.20", features = ["derive"] }
 difference = { version = "2.0.0", default-features = false }
 reqwest = { version = "0.11.12", features = ["blocking"] }
+serde_json = "1.0.107"
+serde = { version = "1.0.189", features = ["derive"] }
 tempfile = "3.7.1"
 toml = { version = "0.5.9", default-features = false }

crates/tools/src/compare_test_runs.rs

@@ -0,0 +1,80 @@
+use std::collections::BTreeMap;
+use std::path::Path;
+
+use anyhow::Result;
+
+#[derive(Debug, PartialEq, Eq, serde::Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct TestRun {
+    title: String,
+    full_title: String,
+    file: String,
+    duration: Option<u32>,
+    current_retry: u32,
+    speed: Option<String>,
+    err: serde_json::Value,
+}
+
+#[derive(Debug, PartialEq, Eq, serde::Deserialize)]
+struct TestRuns {
+    tests: Vec<TestRun>,
+}
+
+#[derive(Debug)]
+struct Diff {
+    test_title: String,
+    absolute_diff: u32,
+    increase_percent: f64,
+}
+
+pub(crate) fn compare(baseline: &Path, candidate: &Path) -> Result<()> {
+    let baseline_runs = read_test_runs(baseline)?;
+    let candidate_runs = read_test_runs(candidate)?;
+    let mut diffs = Vec::new();
+    for (key, candidate_run) in candidate_runs.iter() {
+        if let Some(baseline_run) = baseline_runs.get(key) {
+            match (baseline_run.duration, candidate_run.duration) {
+                (Some(base_duration), Some(candidate_duration)) => {
+                    if base_duration < candidate_duration {
+                        let absolute_diff = candidate_duration - base_duration;
+                        let increase_percent = if base_duration == 0 {
+                            100.0
+                        } else {
+                            f64::from(candidate_duration) / f64::from(base_duration) * 100.0 - 100.0
+                        };
+                        diffs.push(Diff {
+                            test_title: candidate_run.title.clone(),
+                            absolute_diff,
+                            increase_percent,
+                        });
+                    }
+                }
+                (Some(_), None) => println!("No candidate duration for `{key}`"),
+                (None, Some(_)) => println!("No baseline duration for `{key}`"),
+                (None, None) => {}
+            }
+        } else {
+            println!("`{}` is a new test", candidate_run.title);
+        }
+    }
+
+    diffs.sort_by(|a, b| b.absolute_diff.cmp(&a.absolute_diff));
+    for diff in &diffs {
+        println!(
+            "`{}` is slower than baseline. Absolute diff: {} ms. Increase: {:.2} %",
+            diff.test_title, diff.absolute_diff, diff.increase_percent
+        );
+    }
+
+    Ok(())
+}
+
+fn read_test_runs(json_path: &Path) -> Result<BTreeMap<String, TestRun>> {
+    let runs: TestRuns = serde_json::from_reader(std::fs::File::open(json_path)?)?;
+    let result = runs
+        .tests
+        .into_iter()
+        .map(|run| (run.full_title.clone(), run))
+        .collect();
+    Ok(result)
+}

crates/tools/src/main.rs

@@ -2,6 +2,7 @@ use clap::{Parser, Subcommand};
 use std::path::PathBuf;
 
 mod benchmark;
+mod compare_test_runs;
 mod execution_api;
 mod update;
 
@@ -25,13 +26,24 @@ enum Command {
         #[clap(long, short, default_value = "3")]
         iterations: usize,
     },
+    /// Compare JSON format test execution outputs for slower tests. Pass the --reporter json argument to mocha to generate the input files.
+    CompareTestRuns {
+        /// The path to the baseline test run
+        baseline: PathBuf,
+        /// The path to the candidate test run
+        candidate: PathBuf,
+    },
     /// Generate Ethereum execution API
     GenExecutionApi,
 }
 
 fn main() -> anyhow::Result<()> {
     let args = Args::parse();
     match args.command {
+        Command::CompareTestRuns {
+            baseline,
+            candidate,
+        } => compare_test_runs::compare(&baseline, &candidate),
         Command::Benchmark {
             working_directory,
             test_command,