final afl++ cmplog experiment (google#1106)

* add to afl++ to papers

* fix afl++

* final cmplog round

* add new havoc change test

.github/workflows/fuzzers.yml

@@ -45,10 +45,14 @@ jobs:
           - aflplusplus_dict2file
           - aflplusplus_cmplog
           - aflplusplus_cmplog_introspection
+          - aflplusplus_cmplog_4k
           - aflplusplus_cmplog_12k
+          - aflplusplus_cmplog_16k
+          - aflplusplus_cmplog_24k
           - aflplusplus_cmplog_fail96
           - aflplusplus_flcnt
           - aflplusplus_optimal_flcnt
+          - aflplusplus_havoc
 
         benchmark_type:
           - oss-fuzz

fuzzers/aflplusplus/builder.Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
 # Download and compile afl++.
 RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
     cd /afl && \
-    git checkout a2f40aa285faa75e78ac1ffffe8d79e2ac1a40da
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902
 
 # Build without Python support as we don't need it.
 # Set AFL_NO_X86 to skip flaky tests.

fuzzers/aflplusplus_cmplog/builder.Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
 # Download and compile afl++.
 RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
     cd /afl && \
-    git checkout a2f40aa285faa75e78ac1ffffe8d79e2ac1a40da
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902
 
 # Build without Python support as we don't need it.
 # Set AFL_NO_X86 to skip flaky tests.

fuzzers/aflplusplus_cmplog_12k/builder.Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
 # Download and compile afl++.
 RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
     cd /afl && \
-    git checkout a2f40aa285faa75e78ac1ffffe8d79e2ac1a40da && \
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902 && \
     sed -i 's|CMPLOG_POSITIONS_MAX .*|CMPLOG_POSITIONS_MAX 12288U|' include/config.h
 
 # Build without Python support as we don't need it.

fuzzers/aflplusplus_cmplog_16k/builder.Dockerfile

@@ -0,0 +1,36 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Install libstdc++ to use llvm_mode.
+RUN apt-get update && \
+    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+                       libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
+                       apt-utils apt-transport-https ca-certificates
+
+# Download and compile afl++.
+RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
+    cd /afl && \
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902 && \
+    sed -i 's|CMPLOG_POSITIONS_MAX .*|CMPLOG_POSITIONS_MAX 16384U|' include/config.h
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \
+    export CC=clang && export AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && make install && \
+    make -C utils/aflpp_driver && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_cmplog_16k/description.md

@@ -0,0 +1,13 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)

fuzzers/aflplusplus_cmplog_16k/fuzzer.py

@@ -0,0 +1,38 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFLplusplus fuzzer."""
+
+# This optimized afl++ variant should always be run together with
+# "aflplusplus" to show the difference - a default configured afl++ vs.
+# a hand-crafted optimized one. afl++ is configured not to enable the good
+# stuff by default to be as close to vanilla afl as possible.
+# But this means that the good stuff is hidden away in this benchmark
+# otherwise.
+
+from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer
+
+
+def build():  # pylint: disable=too-many-branches,too-many-statements
+    """Build benchmark."""
+    aflplusplus_fuzzer.build("tracepc", "cmplog", "dict2file")
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run fuzzer."""
+    run_options = ['-p', 'fast']
+
+    aflplusplus_fuzzer.fuzz(input_corpus,
+                            output_corpus,
+                            target_binary,
+                            flags=(run_options))

fuzzers/aflplusplus_cmplog_16k/runner.Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image
+
+# This makes interactive docker runs painless:
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"
+ENV AFL_MAP_SIZE=2621440
+ENV PATH="$PATH:/out"
+ENV AFL_SKIP_CPUFREQ=1
+ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
+ENV AFL_TESTCACHE_SIZE=2

fuzzers/aflplusplus_cmplog_24k/builder.Dockerfile

@@ -0,0 +1,36 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Install libstdc++ to use llvm_mode.
+RUN apt-get update && \
+    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+                       libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
+                       apt-utils apt-transport-https ca-certificates
+
+# Download and compile afl++.
+RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
+    cd /afl && \
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902 && \
+    sed -i 's|CMPLOG_POSITIONS_MAX .*|CMPLOG_POSITIONS_MAX 24576U|' include/config.h
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \
+    export CC=clang && export AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && make install && \
+    make -C utils/aflpp_driver && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_cmplog_24k/description.md

@@ -0,0 +1,13 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)

fuzzers/aflplusplus_cmplog_24k/fuzzer.py

@@ -0,0 +1,38 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFLplusplus fuzzer."""
+
+# This optimized afl++ variant should always be run together with
+# "aflplusplus" to show the difference - a default configured afl++ vs.
+# a hand-crafted optimized one. afl++ is configured not to enable the good
+# stuff by default to be as close to vanilla afl as possible.
+# But this means that the good stuff is hidden away in this benchmark
+# otherwise.
+
+from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer
+
+
+def build():  # pylint: disable=too-many-branches,too-many-statements
+    """Build benchmark."""
+    aflplusplus_fuzzer.build("tracepc", "cmplog", "dict2file")
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run fuzzer."""
+    run_options = ['-p', 'fast']
+
+    aflplusplus_fuzzer.fuzz(input_corpus,
+                            output_corpus,
+                            target_binary,
+                            flags=(run_options))

fuzzers/aflplusplus_cmplog_24k/runner.Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image
+
+# This makes interactive docker runs painless:
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"
+ENV AFL_MAP_SIZE=2621440
+ENV PATH="$PATH:/out"
+ENV AFL_SKIP_CPUFREQ=1
+ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
+ENV AFL_TESTCACHE_SIZE=2

fuzzers/aflplusplus_cmplog_4k/builder.Dockerfile

@@ -0,0 +1,36 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG parent_image
+FROM $parent_image
+
+# Install libstdc++ to use llvm_mode.
+RUN apt-get update && \
+    apt-get install -y wget libstdc++-5-dev libtool-bin automake flex bison \
+                       libglib2.0-dev libpixman-1-dev python3-setuptools unzip \
+                       apt-utils apt-transport-https ca-certificates
+
+# Download and compile afl++.
+RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
+    cd /afl && \
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902 && \
+    sed -i 's|CMPLOG_POSITIONS_MAX .*|CMPLOG_POSITIONS_MAX 4096U|' include/config.h
+
+# Build without Python support as we don't need it.
+# Set AFL_NO_X86 to skip flaky tests.
+RUN cd /afl && unset CFLAGS && unset CXXFLAGS && \
+    export CC=clang && export AFL_NO_X86=1 && \
+    PYTHON_INCLUDE=/ make && make install && \
+    make -C utils/aflpp_driver && \
+    cp utils/aflpp_driver/libAFLDriver.a /

fuzzers/aflplusplus_cmplog_4k/description.md

@@ -0,0 +1,13 @@
+# aflplusplus
+
+AFL++ fuzzer instance that has the following config active for all benchmarks:
+  - PCGUARD instrumentation 
+  - cmplog feature
+  - "fast" power schedule
+  - persistent mode + shared memory test cases
+
+Repository: [https://github.com/AFLplusplus/AFLplusplus/](https://github.com/AFLplusplus/AFLplusplus/)
+
+[builder.Dockerfile](builder.Dockerfile)
+[fuzzer.py](fuzzer.py)
+[runner.Dockerfile](runner.Dockerfile)

fuzzers/aflplusplus_cmplog_4k/fuzzer.py

@@ -0,0 +1,38 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Integration code for AFLplusplus fuzzer."""
+
+# This optimized afl++ variant should always be run together with
+# "aflplusplus" to show the difference - a default configured afl++ vs.
+# a hand-crafted optimized one. afl++ is configured not to enable the good
+# stuff by default to be as close to vanilla afl as possible.
+# But this means that the good stuff is hidden away in this benchmark
+# otherwise.
+
+from fuzzers.aflplusplus import fuzzer as aflplusplus_fuzzer
+
+
+def build():  # pylint: disable=too-many-branches,too-many-statements
+    """Build benchmark."""
+    aflplusplus_fuzzer.build("tracepc", "cmplog", "dict2file")
+
+
+def fuzz(input_corpus, output_corpus, target_binary):
+    """Run fuzzer."""
+    run_options = ['-p', 'fast']
+
+    aflplusplus_fuzzer.fuzz(input_corpus,
+                            output_corpus,
+                            target_binary,
+                            flags=(run_options))

fuzzers/aflplusplus_cmplog_4k/runner.Dockerfile

@@ -0,0 +1,23 @@
+# Copyright 2020 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM gcr.io/fuzzbench/base-image
+
+# This makes interactive docker runs painless:
+ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/out"
+ENV AFL_MAP_SIZE=2621440
+ENV PATH="$PATH:/out"
+ENV AFL_SKIP_CPUFREQ=1
+ENV AFL_I_DONT_CARE_ABOUT_MISSING_CRASHES=1
+ENV AFL_TESTCACHE_SIZE=2

fuzzers/aflplusplus_cmplog_fail96/builder.Dockerfile

@@ -24,7 +24,7 @@ RUN apt-get update && \
 # Download and compile afl++.
 RUN git clone https://github.com/AFLplusplus/AFLplusplus.git /afl && \
     cd /afl && \
-    git checkout a2f40aa285faa75e78ac1ffffe8d79e2ac1a40da && \
+    git checkout 9b3d8c327d33191b181219ffce411b40bdbe8902 && \
     sed -i 's|CMPLOG_FAIL_MAX .*|CMPLOG_FAIL_MAX 96|' include/config.h
 
 # Build without Python support as we don't need it.