test: Dump simulator RNG seeds (#2165)

* test: Dump simulator RNG seeds Into the directory given in the `DUMP_SIMULATION_SEEDS` environment variable. Also, export them as artifacts from the CI runs. Fixes #1645 * mkdir * Print leading zeros * TypoPrint leading zeros * Fixes * Conditional * Separate out sanitizer runs * Per sanitizer * Update test-fixture/src/sim/rng.rs Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Update .github/workflows/check.yml Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Update test-fixture/src/sim/mod.rs Co-authored-by: Martin Thomson <[email protected]> Signed-off-by: Lars Eggert <[email protected]> * Prefix seed with test name * Merge --------- Signed-off-by: Lars Eggert <[email protected]> Co-authored-by: Martin Thomson <[email protected]>
mozilla · Oct 11, 2024 · 9ef0355 · 9ef0355
1 parent 249d062
commit 9ef0355
Show file tree

Hide file tree

Showing 4 changed files with 48 additions and 16 deletions.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -17,6 +17,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  DUMP_SIMULATION_SEEDS: /tmp/simulation-seeds
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}
@@ -82,12 +83,14 @@ jobs:
           cargo +${{ matrix.rust-toolchain }} check $BUILD_TYPE --all-targets --features ci
 
       - name: Run tests and determine coverage
+        env:
+          RUST_LOG: trace
         run: |
           # shellcheck disable=SC2086
           if [ "${{ matrix.rust-toolchain }}" == "stable" ]; then
-            RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
+            cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
           else
-            RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast
+            cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast
           fi
 
       - name: Run client/server transfer
@@ -117,6 +120,14 @@ jobs:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         if: matrix.type == 'debug' && matrix.rust-toolchain == 'stable'
 
+      - name: Save simulation seeds artifact
+        if: env.DUMP_SIMULATION_SEEDS
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: simulation-seeds-${{ matrix.os }}-${{ matrix.rust-toolchain }}-${{ matrix.type }}
+          path: ${{ env.DUMP_SIMULATION_SEEDS }}
+          compression-level: 9
+
   bench:
     needs: [check]
     if: >

diff --git a/.github/workflows/sanitize.yml b/.github/workflows/sanitize.yml
@@ -11,6 +11,7 @@ on:
 env:
   CARGO_TERM_COLOR: always
   RUST_BACKTRACE: 1
+  DUMP_SIMULATION_SEEDS: /tmp/simulation-seeds
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.ref_name }}
@@ -76,3 +77,11 @@ jobs:
             export LSAN_OPTIONS="suppressions=$PWD/suppressions.txt"
           fi
           cargo nextest run -Z build-std --features ci --target "$TARGET"
+
+      - name: Save simulation seeds artifact
+        if: env.DUMP_SIMULATION_SEEDS
+        uses: actions/upload-artifact@50769540e7f4bd5e21e526ee35c689e35e0d6874 # v4.4.0
+        with:
+          name: simulation-seeds-${{ matrix.os }}-sanitizer-${{ matrix.sanitizer }}
+          path: ${{ env.DUMP_SIMULATION_SEEDS }}
+          compression-level: 9
diff --git a/test-fixture/src/sim/mod.rs b/test-fixture/src/sim/mod.rs
@@ -15,12 +15,14 @@ use std::{
     cell::RefCell,
     cmp::min,
     fmt::Debug,
+    fs::{create_dir_all, File},
     ops::{Deref, DerefMut},
+    path::PathBuf,
     rc::Rc,
     time::{Duration, Instant},
 };
 
-use neqo_common::{qdebug, qinfo, qtrace, Datagram, Encoder};
+use neqo_common::{qdebug, qerror, qinfo, qtrace, Datagram, Encoder};
 use neqo_transport::Output;
 use rng::Random;
 use NodeState::{Active, Idle, Waiting};
@@ -64,11 +66,7 @@ macro_rules! simulate {
                 let f: Box<dyn FnOnce(&_) -> _> = Box::new($v);
                 nodes.push(Box::new(f(&fixture)));
             )*
-            let mut sim = Simulator::new(stringify!($n), nodes);
-            if let Ok(seed) = std::env::var("SIMULATION_SEED") {
-                sim.seed_str(seed);
-            }
-            sim.run();
+            Simulator::new(stringify!($n), nodes).run();
         }
     };
 }
@@ -151,23 +149,37 @@ impl Simulator {
             .into_iter()
             .chain(it.map(|node| NodeHolder { node, state: Idle }))
             .collect::<Vec<_>>();
-        Self {
+        let mut sim = Self {
             name,
             nodes,
             rng: Rc::default(),
+        };
+        // Seed from the `SIMULATION_SEED` environment variable, if set.
+        if let Ok(seed) = std::env::var("SIMULATION_SEED") {
+            sim.seed_str(seed);
         }
-    }
-
-    pub fn seed(&mut self, seed: [u8; 32]) {
-        self.rng = Rc::new(RefCell::new(Random::new(&seed)));
+        // Dump the seed to a file to the directory in the `DUMP_SIMULATION_SEEDS` environment
+        // variable, if set.
+        if let Ok(dir) = std::env::var("DUMP_SIMULATION_SEEDS") {
+            if create_dir_all(&dir).is_err() {
+                qerror!("Failed to create directory {dir}");
+            } else {
+                let seed_str = sim.rng.borrow().seed_str();
+                let path = PathBuf::from(format!("{dir}/{}-{seed_str}", sim.name));
+                if File::create(&path).is_err() {
+                    qerror!("Failed to write seed to {}", path.to_string_lossy());
+                }
+            }
+        }
+        sim
     }
 
     /// Seed from a hex string.
     /// # Panics
     /// When the provided string is not 32 bytes of hex (64 characters).
     pub fn seed_str(&mut self, seed: impl AsRef<str>) {
-        let seed = Encoder::from_hex(seed);
-        self.seed(<[u8; 32]>::try_from(seed.as_ref()).unwrap());
+        let seed = <[u8; 32]>::try_from(Encoder::from_hex(seed).as_ref()).unwrap();
+        self.rng = Rc::new(RefCell::new(Random::new(&seed)));
     }
 
     fn next_time(&self, now: Instant) -> Instant {

diff --git a/test-fixture/src/sim/rng.rs b/test-fixture/src/sim/rng.rs
@@ -70,7 +70,7 @@ impl Random {
     #[must_use]
     pub fn seed_str(&self) -> String {
         format!(
-            "{:8x}{:8x}{:8x}{:8x}",
+            "{:016x}{:016x}{:016x}{:016x}",
             self.state[0], self.state[1], self.state[2], self.state[3],
         )
     }