Merge pull request #123 from fitzgen/update-libfuzzer-vendoring

Update how we do libfuzzer vendoring

Author: fitzgen

.github/workflows/rust.yml

@@ -3,12 +3,22 @@ name: Rust
 on: [push, pull_request]
 
 jobs:
+  check_libfuzzer_checkout:
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Re-vendor libfuzzer
+      run: ./update-libfuzzer.sh
+    - name: Check that nothing changed
+      run: git diff --exit-code
+
   build:
 
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v1
+    - uses: actions/checkout@v4
 
     - name: Install nightly Rust
       run: |

README.md

@@ -86,12 +86,18 @@ Then link to your own runtime in your `build.rs`.
 
 ## Updating libfuzzer from upstream
 
-```
-./update-libfuzzer.sh <github.com/llvm-mirror/llvm-project SHA1>
-```
+* Update the `COMMIT=...` variable in `./update-libfuzzer.sh` with the new
+  commit hash from [llvm-mirror/llvm-project](github.com/llvm-mirror/llvm-project)
+  that you are vendoring.
+
+* Re-run the script:
+
+  ```
+  $ ./update-libfuzzer.sh <github.com/llvm-mirror/llvm-project SHA1>
+  ```
 
 ## License
 
-All files in `libfuzzer` directory are licensed NCSA.
+All files in the `libfuzzer` directory are licensed NCSA.
 
 Everything else is dual-licensed Apache 2.0 and MIT.

libfuzzer/FuzzerDriver.cpp

@@ -229,6 +229,7 @@ static void PulseThread() {
 
 static void WorkerThread(const Command &BaseCmd, std::atomic<unsigned> *Counter,
                          unsigned NumJobs, std::atomic<bool> *HasErrors) {
+  ScopedDisableMsanInterceptorChecks S;
   while (true) {
     unsigned C = (*Counter)++;
     if (C >= NumJobs) break;

libfuzzer/FuzzerFork.cpp

@@ -349,7 +349,7 @@ void FuzzWithFork(Random &Rand, const FuzzingOptions &Options,
                         &NewFeatures, Env.Cov, &NewCov, CFPath,
                         /*Verbose=*/false, /*IsSetCoverMerge=*/false);
     Env.Features.insert(NewFeatures.begin(), NewFeatures.end());
-    Env.Cov.insert(NewFeatures.begin(), NewFeatures.end());
+    Env.Cov.insert(NewCov.begin(), NewCov.end());
     RemoveFile(CFPath);
   }
 

libfuzzer/FuzzerUtilFuchsia.cpp

@@ -292,7 +292,7 @@ void CrashHandler() {
     zx_wait_item_t WaitItems[] = {
         {
             .handle = SignalHandlerEvent,
-            .waitfor = ZX_SIGNAL_HANDLE_CLOSED,
+            .waitfor = ZX_USER_SIGNAL_1,
             .pending = 0,
         },
         {
@@ -378,10 +378,11 @@ void CrashHandler() {
 }
 
 void StopSignalHandler() {
-  _zx_handle_close(SignalHandlerEvent);
+  _zx_object_signal(SignalHandlerEvent, 0, ZX_USER_SIGNAL_1);
   if (SignalHandler.joinable()) {
     SignalHandler.join();
   }
+  _zx_handle_close(SignalHandlerEvent);
 }
 
 } // namespace

libfuzzer/FuzzerUtilLinux.cpp

@@ -44,7 +44,7 @@ void SetThreadName(std::thread &thread, const std::string &name) {
 #if LIBFUZZER_LINUX || LIBFUZZER_FREEBSD
   (void)pthread_setname_np(thread.native_handle(), name.c_str());
 #elif LIBFUZZER_NETBSD
-  (void)pthread_set_name_np(thread.native_handle(), "%s", name.c_str());
+  (void)pthread_setname_np(thread.native_handle(), "%s", const_cast<char *>(name.c_str()));
 #endif
 }
 

libfuzzer/FuzzerUtilWindows.cpp

@@ -21,10 +21,15 @@
 #include <signal.h>
 #include <stdio.h>
 #include <sys/types.h>
+// clang-format off
 #include <windows.h>
-
-// This must be included after windows.h.
+// These must be included after windows.h.
+// archicture need to be set before including
+// libloaderapi
+#include <libloaderapi.h>
+#include <stringapiset.h>
 #include <psapi.h>
+// clang-format on
 
 namespace fuzzer {
 
@@ -234,8 +239,25 @@ size_t PageSize() {
 }
 
 void SetThreadName(std::thread &thread, const std::string &name) {
-  // TODO ?
-  // to UTF-8 then SetThreadDescription ?
+#if defined(_LIBCPP_HAS_THREAD_API_PTHREAD) ||                                 \
+    defined(_GLIBCXX_GCC_GTHR_POSIX_H)
+  (void)pthread_setname_np(thread.native_handle(), name.c_str());
+#else
+  typedef HRESULT(WINAPI * proc)(HANDLE, PCWSTR);
+  HMODULE kbase = GetModuleHandleA("KernelBase.dll");
+  proc ThreadNameProc =
+      reinterpret_cast<proc>(GetProcAddress(kbase, "SetThreadDescription"));
+  if (ThreadNameProc) {
+    std::wstring buf;
+    auto sz = MultiByteToWideChar(CP_UTF8, 0, name.data(), -1, nullptr, 0);
+    if (sz > 0) {
+      buf.resize(sz);
+      if (MultiByteToWideChar(CP_UTF8, 0, name.data(), -1, &buf[0], sz) > 0) {
+        (void)ThreadNameProc(thread.native_handle(), buf.c_str());
+      }
+    }
+  }
+#endif
 }
 
 } // namespace fuzzer

libfuzzer/build.sh

@@ -2,7 +2,7 @@
 LIBFUZZER_SRC_DIR=$(dirname $0)
 CXX="${CXX:-clang}"
 for f in $LIBFUZZER_SRC_DIR/*.cpp; do
-  $CXX -g -O2 -fno-omit-frame-pointer -std=c++14 $f -c &
+  $CXX -g -O2 -fno-omit-frame-pointer -std=c++17 $f -c &
 done
 wait
 rm -f libFuzzer.a

libfuzzer/tests/CMakeLists.txt

@@ -12,10 +12,10 @@ if (APPLE)
 endif()
 
 add_custom_target(FuzzerUnitTests)
-set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "Compiler-RT Tests")
+set_target_properties(FuzzerUnitTests PROPERTIES FOLDER "Compiler-RT/Tests")
 
 add_custom_target(FuzzedDataProviderUnitTests)
-set_target_properties(FuzzedDataProviderUnitTests PROPERTIES FOLDER "Compiler-RT Tests")
+set_target_properties(FuzzedDataProviderUnitTests PROPERTIES FOLDER "Compiler-RT/Tests")
 
 set(LIBFUZZER_UNITTEST_LINK_FLAGS ${COMPILER_RT_UNITTEST_LINK_FLAGS})
 list(APPEND LIBFUZZER_UNITTEST_LINK_FLAGS --driver-mode=g++)
@@ -58,7 +58,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH)
     ${LIBFUZZER_TEST_RUNTIME_OBJECTS})
   set_target_properties(${LIBFUZZER_TEST_RUNTIME} PROPERTIES
     ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
-    FOLDER "Compiler-RT Runtime tests")
+    FOLDER "Compiler-RT/Tests/Runtime")
 
   if(CMAKE_SYSTEM_NAME STREQUAL "Linux" AND
      COMPILER_RT_LIBCXX_PATH AND
@@ -74,7 +74,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH)
     FuzzerUnitTests "Fuzzer-${arch}-Test" ${arch}
     SOURCES FuzzerUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
     RUNTIME ${LIBFUZZER_TEST_RUNTIME}
-    DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS}
+    DEPS ${LIBFUZZER_TEST_RUNTIME_DEPS}
     CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
     LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
   set_target_properties(FuzzerUnitTests PROPERTIES
@@ -84,7 +84,7 @@ if(COMPILER_RT_DEFAULT_TARGET_ARCH IN_LIST FUZZER_SUPPORTED_ARCH)
   generate_compiler_rt_tests(FuzzedDataProviderTestObjects
     FuzzedDataProviderUnitTests "FuzzerUtils-${arch}-Test" ${arch}
     SOURCES FuzzedDataProviderUnittest.cpp ${COMPILER_RT_GTEST_SOURCE}
-    DEPS llvm_gtest ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
+    DEPS ${LIBFUZZER_TEST_RUNTIME_DEPS} ${COMPILER_RT_SOURCE_DIR}/include/fuzzer/FuzzedDataProvider.h
     CFLAGS ${LIBFUZZER_UNITTEST_CFLAGS} ${LIBFUZZER_TEST_RUNTIME_CFLAGS}
     LINK_FLAGS ${LIBFUZZER_UNITTEST_LINK_FLAGS} ${LIBFUZZER_TEST_RUNTIME_LINK_FLAGS})
   set_target_properties(FuzzedDataProviderUnitTests PROPERTIES

update-libfuzzer.sh

@@ -2,20 +2,26 @@
 
 # Usage:
 #
-#     ./update-libfuzzer $commit_hash
-#
-# Where `$commit_hash` is a commit hash from
-# https://github.com/llvm-mirror/llvm-project
+#     $ ./update-libfuzzer.sh
 
 set -ex
 
+# The LLVM commit from which we are vendoring libfuzzer. This must be a commit
+# hash from https://github.com/llvm/llvm-project
+COMMIT=ab51eccf88f5321e7c60591c5546b254b6afab99
+
 cd "$(dirname $0)"
 project_dir="$(pwd)"
 
 tmp_dir="$(mktemp -d)"
-
-git clone https://github.com/llvm/llvm-project.git "$tmp_dir"
 cd "$tmp_dir"
-git checkout "$1"
+
+git init
+git remote add llvm https://github.com/llvm/llvm-project.git
+git sparse-checkout set compiler-rt/lib/fuzzer
+
+git fetch --depth 1 llvm "$COMMIT" --filter=blob:none
+git checkout "$COMMIT"
+
 rm -rf "$project_dir/libfuzzer/"
 mv "$tmp_dir/compiler-rt/lib/fuzzer/" "$project_dir/libfuzzer/"