Merge pull request #58 from AFLplusplus/real_time_benchmarks

Introspection

README.md

@@ -42,11 +42,6 @@ Rust directly, instructions can be found [here](https://www.rust-lang.org/tools/
 git clone https://github.com/AFLplusplus/LibAFL
 ```
 
-If you want to get the latest and greatest features,
-```
-git checkout dev
-```
-
 Build the library using
 
 ```
@@ -65,7 +60,6 @@ cargo doc
 cd docs && mdbook serve
 ```
 
-
 We collect all example fuzzers in [`./fuzzers`](./fuzzers/).
 Be sure to read their documentation (and source), this is *the natural way to get started!*
 

clippy.sh

@@ -3,16 +3,12 @@
 cargo clean -p libafl
 RUST_BACKTRACE=full cargo clippy --all --all-features --tests -- \
    -D clippy::pedantic \
-   -W clippy::cast_sign_loss \
    -W clippy::similar-names \
-   -W clippy::cast_ptr_alignment \
-   -W clippy::cast_possible_wrap \
    -W clippy::unused_self \
    -W clippy::too_many_lines \
    -W clippy::option_if_let_else \
    -W clippy::must-use-candidate \
    -W clippy::if-not-else \
-   -W clippy::doc-markdown \
    -A clippy::type_repetition_in_bounds \
    -A clippy::missing-errors-doc \
    -A clippy::cast-possible-truncation \

fuzzers/libfuzzer_libpng/Cargo.toml

@@ -20,7 +20,8 @@ which = { version = "4.0.2" }
 num_cpus = "1.0"
 
 [dependencies]
-libafl = { path = "../../libafl/" }
+libafl = { path = "../../libafl/", features = ["default", "introspection"] }
+# libafl = { path = "../../libafl/", features = ["default"] }
 libafl_targets = { path = "../../libafl_targets/", features = ["pcguard_hitcounts", "libfuzzer"] }
 # TODO Include it only when building cc
 libafl_cc = { path = "../../libafl_cc/" }

fuzzers/libfuzzer_libpng/Makefile

@@ -0,0 +1,63 @@
+PWD=`pwd`
+FUZZER_NAME="fuzzer_libpng"
+
+all:
+	# Build the libpng libfuzzer library
+	cargo build --release
+
+    # Build the libpng harness	
+	$(PWD)/target/release/libafl_cxx \
+		$(PWD)/harness.cc \
+		$(PWD)/libpng-1.6.37/.libs/libpng16.a \
+		-I$(PWD)/libpng-1.6.37/ \
+		-o $(FUZZER_NAME) \
+		-lm -lz
+
+run: all
+	./$(FUZZER_NAME) &
+	sleep 0.2
+	./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+
+short_test: all
+	timeout 11s ./$(FUZZER_NAME) &
+	sleep 0.2
+	timeout 10s taskset -c 0 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 10s taskset -c 1 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 10s taskset -c 2 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 10s taskset -c 3 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+
+test: all
+	timeout 60s ./$(FUZZER_NAME) &
+	sleep 0.2
+	timeout 59s taskset 0x00000001 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 59s taskset 0x00000002 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 59s taskset 0x00000004 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	timeout 59s taskset 0x00000008 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000010 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000020 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000040 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000080 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000100 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000200 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000400 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00000800 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00001000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00002000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00004000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00008000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00010000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00020000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00040000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00080000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00100000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00200000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00400000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x00800000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x01000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x02000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x04000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x08000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x10000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x20000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x40000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &
+	# timeout 59s taskset 0x80000000 ./$(FUZZER_NAME) >/dev/null 2>/dev/null &

fuzzers/libfuzzer_libpng/README.md

@@ -6,6 +6,8 @@ In contrast to other fuzzer examples, this setup uses `fuzz_loop_for`, to occasi
 While this costs performance, it can be useful for targets with memory leaks or other instabilities.
 If your target is really instable, however, consider exchanging the `InProcessExecutor` for a `ForkserverExecutor` instead.
 
+It also uses the `introspection` feature, printing fuzzer stats during execution.
+
 To show off crash detection, we added a `ud2` instruction to the harness, edit harness.cc if you want a non-crashing example.
 It has been tested on Linux.
 
@@ -51,7 +53,7 @@ This allows you to run multiple different builds of the same fuzzer alongside, f
 
 ## Run
 
-The first time you run the binary, the broker will open a tcp port (currently on port `1337`), waiting for fuzzer clients to connect. This port is local and only used for the initial handshake. All further communication happens via shared map, to be independent of the kernel. Currently you must run the clients from the libfuzzer_libpng directory for them to be able to access the PNG corpus.
+The first time you run the binary, the broker will open a tcp port (currently on port `1337`), waiting for fuzzer clients to connect. This port is local and only used for the initial handshake. All further communication happens via shared map, to be independent of the kernel. Currently, you must run the clients from the libfuzzer_libpng directory for them to be able to access the PNG corpus.
 
 ```
 ./fuzzer_libpng

fuzzers/libfuzzer_libpng/src/lib.rs

@@ -63,7 +63,7 @@ fn fuzz(corpus_dirs: Vec<PathBuf>, objective_dir: PathBuf, broker_port: u16) ->
             }
         },
     };
-    
+
     // Create an observation channel using the coverage map
     let edges = unsafe { &mut EDGES_MAP[0..MAX_EDGES_NUM] };
     let edges_observer = HitcountsMapObserver::new(StdMapObserver::new("edges", edges));

libafl/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "libafl"
-version = "0.2.0"
+version = "0.2.1"
 authors = ["Andrea Fioraldi <[email protected]>", "Dominik Maier <[email protected]>"]
 description = "Slot your own fuzzers together and extend their features using Rust"
 documentation = "https://docs.rs/libafl"
@@ -40,10 +40,11 @@ default = ["std", "anymap_debug", "derive", "llmp_compression"]
 std = [] # print, sharedmap, ... support
 anymap_debug = ["serde_json"] # uses serde_json to Debug the anymap trait. Disable for smaller footprint.
 derive = ["libafl_derive"] # provide derive(SerdeAny) macro.
-llmp_small_maps = [] # reduces initial map size for llmp
-llmp_debug = ["backtrace"] # Enables debug output for LLMP
-llmp_compression = [] # llmp compression using GZip
 llmp_bind_public = [] # If set, llmp will bind to 0.0.0.0, allowing cross-device communication. Binds to localhost by default.
+llmp_compression = [] # llmp compression using GZip
+llmp_debug = ["backtrace"] # Enables debug output for LLMP
+llmp_small_maps = [] # reduces initial map size for llmp
+introspection = [] # Include performance statistics of the fuzzing pipeline
 
 [[example]]
 name = "llmp_test"
@@ -60,7 +61,7 @@ erased-serde = "0.3.12"
 postcard = { version = "0.5.1", features = ["alloc"] } # no_std compatible serde serialization fromat
 static_assertions = "1.1.0"
 ctor = "0.1.20"
-libafl_derive = { version = "0.1.0", optional = true, path = "../libafl_derive" }
+libafl_derive = { optional = true, path = "../libafl_derive", version = "0.2.1" }
 serde_json = { version = "1.0", optional = true, default-features = false, features = ["alloc"] } # an easy way to debug print SerdeAnyMap
 compression = { version = "0.1.5" }
 num_enum = "0.5.1"

libafl/src/bolts/llmp.rs

@@ -55,6 +55,8 @@ For broker2broker communication, all messages are forwarded via network sockets.
 
 Check out the `llmp_test` example in ./examples, or build it with `cargo run --example llmp_test`.
 
+For broker2broker communication, all messages are forwarded via network sockets.
+
 */
 
 use alloc::{string::String, vec::Vec};

libafl/src/cpu.rs

@@ -0,0 +1,31 @@
+//! Architecture agnostic processor features
+
+#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
+use crate::utils::current_nanos;
+
+// TODO: Add more architectures, using C code, see
+// https://github.com/google/benchmark/blob/master/src/cycleclock.h
+// Or using llvm intrinsics (if they ever should become available in stable rust?)
+
+/// Read a timestamp for measurements.
+///
+/// This function is a wrapper around different ways to get a timestamp, fast
+/// In this way, an experiment only has to
+/// change this implementation rather than every instead of [`cpu::read_time_counter`]
+/// It is using [`rdtsc`] on `x86_64` and `x86`.
+#[cfg(any(target_arch = "x86_64", target_arch = "x86"))]
+#[must_use]
+pub fn read_time_counter() -> u64 {
+    unsafe { core::arch::x86_64::_rdtsc() }
+}
+
+/// Read a timestamp for measurements.
+///
+/// This function is a wrapper around different ways to get a timestamp, fast
+/// In this way, an experiment only has to
+/// change this implementation rather than every instead of [`cpu::read_time_counter`]
+/// On unsupported architectures, it's falling back to normal system time, in millis.
+#[cfg(not(any(target_arch = "x86_64", target_arch = "x86")))]
+pub fn read_time_counter() -> u64 {
+    current_nanos()
+}

libafl/src/events/llmp.rs

@@ -230,7 +230,7 @@ where
                 let client = stats.client_stats_mut_for(sender_id);
                 client.update_corpus_size(*corpus_size as u64);
                 client.update_executions(*executions as u64, *time);
-                stats.display(event.name().to_string() + " #" + &sender_id.to_string());
+                // stats.display(event.name().to_string() + " #" + &sender_id.to_string());
                 Ok(BrokerEventResult::Forward)
             }
             Event::UpdateStats {
@@ -241,7 +241,35 @@ where
                 // TODO: The stats buffer should be added on client add.
                 let client = stats.client_stats_mut_for(sender_id);
                 client.update_executions(*executions as u64, *time);
-                stats.display(event.name().to_string() + " #" + &sender_id.to_string());
+                if sender_id == 1 {
+                    stats.display(event.name().to_string() + " #" + &sender_id.to_string());
+                }
+                Ok(BrokerEventResult::Handled)
+            }
+            #[cfg(feature = "introspection")]
+            Event::UpdatePerfStats {
+                time,
+                executions,
+                introspection_stats,
+                phantom: _,
+            } => {
+                // TODO: The stats buffer should be added on client add.
+
+                // Get the client for the sender ID
+                let client = stats.client_stats_mut_for(sender_id);
+
+                // Update the normal stats for this client
+                client.update_executions(*executions as u64, *time);
+
+                // Update the performance stats for this client
+                client.update_introspection_stats(**introspection_stats);
+
+                // Display the stats via `.display` only on core #1
+                if sender_id == 1 {
+                    stats.display(event.name().to_string() + " #" + &sender_id.to_string());
+                }
+
+                // Correctly handled the event
                 Ok(BrokerEventResult::Handled)
             }
             Event::Objective { objective_size } => {

libafl/src/events/mod.rs

@@ -17,6 +17,9 @@ use crate::{
     Error,
 };
 
+#[cfg(feature = "introspection")]
+use crate::stats::ClientPerfStats;
+
 /// The log event severity
 #[derive(Serialize, Deserialize, Debug, Clone, Copy)]
 pub enum LogSeverity {
@@ -98,6 +101,20 @@ where
         /// [`PhantomData`]
         phantom: PhantomData<I>,
     },
+    /// New stats with performance stats.
+    #[cfg(feature = "introspection")]
+    UpdatePerfStats {
+        /// The time of generation of the event
+        time: Duration,
+
+        /// The executions of this client
+        executions: usize,
+
+        /// Current performance statistics
+        introspection_stats: Box<ClientPerfStats>,
+
+        phantom: PhantomData<I>,
+    },
     /// A new objective was found
     Objective {
         /// Objective corpus size
@@ -138,6 +155,13 @@ where
                 executions: _,
                 phantom: _,
             } => "Stats",
+            #[cfg(feature = "introspection")]
+            Event::UpdatePerfStats {
+                time: _,
+                executions: _,
+                introspection_stats: _,
+                phantom: _,
+            } => "PerfStats",
             Event::Objective { objective_size: _ } => "Objective",
             Event::Log {
                 severity_level: _,

libafl/src/events/simple.rs

@@ -106,6 +106,19 @@ where
                 stats.display(event.name().to_string());
                 Ok(BrokerEventResult::Handled)
             }
+            #[cfg(feature = "introspection")]
+            Event::UpdatePerfStats {
+                time,
+                executions,
+                introspection_stats,
+                phantom: _,
+            } => {
+                // TODO: The stats buffer should be added on client add.
+                stats.client_stats_mut()[0].update_executions(*executions as u64, *time);
+                stats.client_stats_mut()[0].update_introspection_stats(**introspection_stats);
+                stats.display(event.name().to_string());
+                Ok(BrokerEventResult::Handled)
+            }
             Event::Objective { objective_size } => {
                 stats
                     .client_stats_mut_for(0)