add --fuzz-seed and --fuzz-futexes

Summary:
This adds the first non-thread-order, non-RNG source of fuzzing, as described in this issue:

   #34

Before this diff, hermit is choosing an arbitrary (but deterministic) set of waiting threads to wake on each FUTEX_WAKE. After this diff, that selection is randomized, using deterministic PRNG from a new seed.

Reviewed By: jasonwhite

Differential Revision: D41721535

fbshipit-source-id: 8c340df049ffa0792cfaaead20d7441d6145c2b7

detcore-model/src/config.rs

@@ -65,6 +65,12 @@ pub struct Config {
     #[clap(long, value_name = "uint64")]
     pub rng_seed: Option<u64>,
 
+    /// Seeds the PRNG which drives syscall response fuzzing (i.e. chaotically exercising syscall
+    /// nondeterminism).  Like other seeds, this is initialized from the `--seed` if not
+    /// specifically provided.
+    #[clap(long, value_name = "uint64")]
+    pub fuzz_seed: Option<u64>,
+
     /// Logical clock multiplier. Values above one make time appear to go faster within the sandbox.
     #[clap(long, value_name = "float")]
     pub clock_multiplier: Option<f64>,
@@ -93,6 +99,10 @@ pub struct Config {
     #[clap(long)]
     pub chaos: bool,
 
+    /// Uses the `--fuzz-seed` to generate randomness and fuzz nondeterminism in the futex semantics.
+    #[clap(long)]
+    pub fuzz_futexes: bool,
+
     /// Record the timing of preemption events for future replay or experimentation.
     /// This is only useful in chaos modes.
     #[clap(long)]
@@ -566,6 +576,12 @@ impl Config {
         self.rng_seed.unwrap_or(self.seed)
     }
 
+    /// Returns the fuzz_seed, as specified by the user or defaulting to the primary seed if
+    /// unspecified.
+    pub fn fuzz_seed(&self) -> u64 {
+        self.fuzz_seed.unwrap_or(self.seed)
+    }
+
     /// Returns effective "sched-seed" parameter taking in account "seed"
     /// parameter if former isn't specified
     pub fn sched_seed(&self) -> u64 {

detcore/src/scheduler.rs

@@ -28,6 +28,9 @@ use detcore_model::collections::ReplayCursor;
 use nix::sys::signal;
 use nix::sys::signal::Signal;
 use nix::unistd::Pid;
+use rand::seq::SliceRandom;
+use rand::SeedableRng;
+use rand_pcg::Pcg64Mcg;
 use reverie::syscalls::Syscall;
 use reverie::syscalls::SyscallInfo;
 pub use runqueue::entropy_to_priority;
@@ -202,7 +205,7 @@ fn assert_continue_request(req: &Resources) {
 }
 
 /// The state for the deterministic scheduler.
-#[derive(Debug, Default)]
+#[derive(Debug)]
 pub struct Scheduler {
     /// Monotonically count upwards.
     pub turn: u64,
@@ -289,6 +292,9 @@ pub struct Scheduler {
     /// the (original) replay_cursor trace.
     pub stacktrace_events: Option<StacktraceEventsIter>,
 
+    /// PRNG to drive any fuzzing of OS semantics (other than scheduling).
+    fuzz_prng: Pcg64Mcg,
+
     /// A cached copy of the same (immutable) field in Config.
     stop_after_turn: Option<u64>,
     /// A cached copy of the same (immutable) field in Config.
@@ -299,6 +305,8 @@ pub struct Scheduler {
     die_on_desync: bool,
     /// A cached copy of the same (immutable) field in Config.
     replay_exhausted_panic: bool,
+    /// A cached copy of the same (immutable) field in Config.
+    fuzz_futexes: bool,
 }
 
 type StacktraceEventsIter = Peekable<IntoIter<(u64, Option<PathBuf>)>>;
@@ -860,6 +868,8 @@ impl Scheduler {
             thread_tree: Default::default(),
             priorities: Default::default(),
             timeslices: Default::default(),
+            fuzz_futexes: cfg.fuzz_futexes,
+            fuzz_prng: Pcg64Mcg::seed_from_u64(cfg.fuzz_seed()),
         }
     }
 
@@ -1180,8 +1190,33 @@ impl Scheduler {
         // is ready to run in normal order.
     }
 
+    fn choose_futex_wakees(
+        &mut self,
+        vec: &mut Vec<(DetPid, Ivar<SchedResponse>)>,
+        num_woken: usize,
+    ) -> Vec<(DetPid, Ivar<SchedResponse>)> {
+        if self.fuzz_futexes {
+            let rng = &mut self.fuzz_prng;
+            debug!(
+                "[fuzz-futexes] selecting {} tids, pre shuffle: {:?}",
+                num_woken,
+                vec.iter().map(|x| x.0).collect::<Vec<DetPid>>()
+            );
+
+            // No need to actually use the results here since vec was mutated:
+            let (_extracted, _remain) = &vec[..].partial_shuffle(rng, num_woken);
+
+            info!(
+                "[fuzz-futexes] selecting {} tids, post shuffle: {:?}",
+                num_woken,
+                vec.iter().map(|x| x.0).collect::<Vec<DetPid>>()
+            );
+        }
+        // just take the first N, in whatever deterministic order they are in:
+        vec.split_off(vec.len() - num_woken)
+    }
+
     /// Reschedule all threads blocked on a particular futex.
-    /// TODO: support rescheduling exactly K threads.
     pub fn wake_futex_waiters(
         &mut self,
         _waker_dettid: DetTid,
@@ -1210,7 +1245,8 @@ impl Scheduler {
             vec.len(),
         );
         let num_woken: usize = std::cmp::min(vec.len(), max_to_wake.try_into().unwrap());
-        let to_wake = vec.split_off(vec.len() - num_woken);
+        let to_wake = self.choose_futex_wakees(&mut vec, num_woken);
+
         assert_eq!(to_wake.len(), num_woken);
         for waiter in to_wake {
             self.wake_futex_waiter(waiter);

detcore/tests/testutils/src/lib.rs

@@ -66,6 +66,7 @@ lazy_static! {
     seed: DEFAULT_CFG.seed,
     rng_seed: None,
     sched_seed: None,
+    fuzz_seed: None,
     gdbserver: false,
     gdbserver_port: 1234,
     preemption_timeout: NonZeroU64::new(5000000),
@@ -94,6 +95,7 @@ lazy_static! {
     sysinfo_uptime_offset: 60,
     memory: 1024 * 1024 * 1024, //1 GiB
     interrupt_at: vec![],
+    fuzz_futexes: false,
   };
 
   /// Standardized test config: common options on.
@@ -115,6 +117,7 @@ lazy_static! {
     seed: DEFAULT_CFG.seed,
     rng_seed: None,
     sched_seed: None,
+    fuzz_seed: None,
     gdbserver: false,
     gdbserver_port: 1234,
     preemption_timeout: NonZeroU64::new(5000000),
@@ -143,6 +146,7 @@ lazy_static! {
     sysinfo_uptime_offset: 60,
     memory: 1024 * 1024 * 1024, //1 GiB
     interrupt_at: vec![],
+    fuzz_futexes: false,
   };
 
   /// Standardized test config: all options on.
@@ -164,6 +168,7 @@ lazy_static! {
     seed: DEFAULT_CFG.seed,
     rng_seed: None,
     sched_seed: None,
+    fuzz_seed: None,
     gdbserver: false,
     gdbserver_port: 1234,
     preemption_timeout: NonZeroU64::new(5000000),
@@ -192,6 +197,7 @@ lazy_static! {
     sysinfo_uptime_offset: 60,
     memory: 1024 * 1024 * 1024, //1 GiB
     interrupt_at: vec![],
+    fuzz_futexes: false,
   };
 }
 

hermit-cli/src/bin/hermit/run.rs

@@ -465,7 +465,13 @@ impl fmt::Display for RunOpts {
         if let Some(rng_seed) = dop.rng_seed {
             write!(f, " --rng-seed={}", rng_seed)?;
         }
+        if let Some(fuzz_seed) = dop.fuzz_seed {
+            write!(f, " --fuzz-seed={}", fuzz_seed)?;
+        }
 
+        if dop.fuzz_futexes {
+            write!(f, " --fuzz-futexes")?;
+        }
         if let Some(m) = dop.clock_multiplier {
             write!(f, " --clock-multiplier={}", m)?;
         }

hermit-cli/src/metadata.rs

@@ -193,6 +193,8 @@ pub fn record_or_replay_config(data: &Path) -> detcore::Config {
         sysinfo_uptime_offset: 120,
         memory: 1024 * 1024 * 1024,
         interrupt_at: vec![],
+        fuzz_futexes: false,
+        fuzz_seed: None,
     };
     if config.preemption_timeout.is_some() && !reverie_ptrace::is_perf_supported() {
         tracing::warn!(

tests/rust/futex_wake_some.rs

@@ -16,7 +16,7 @@ use nix::errno::errno;
 use nix::errno::Errno;
 
 const TOTAL: i64 = 4;
-const TO_WAKE: i64 = 3;
+const TO_WAKE: i64 = 2;
 
 fn main() {
     let layout = std::alloc::Layout::from_size_align(4, 4).unwrap(); // u32
@@ -45,6 +45,7 @@ fn main() {
             };
             if val == 0 {
                 children_post.fetch_add(1, Ordering::SeqCst);
+                eprintln!("tid {} woken", ix);
             } else {
                 println!(
                     "UNLIKELY: child thread {} issued wait after wake already happened, errno {}",