From 5b733fa2eae477fe61180e345ecb43d4fa9ebd23 Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Mon, 27 May 2024 17:02:38 -0100 Subject: [PATCH 01/25] Handle TPMI initilization for systems without access to MCFG tables e.g. docker/VM --- src/cpucounters.cpp | 37 +++++++++++++++++++++---------------- src/pci.cpp | 8 ++------ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/src/cpucounters.cpp b/src/cpucounters.cpp index 66b84b56..48a25844 100644 --- a/src/cpucounters.cpp +++ b/src/cpucounters.cpp @@ -1969,26 +1969,31 @@ void PCM::initUncoreObjects() initUncorePMUsDirect(); } - // TPMIHandle::setVerbose(true); - if (TPMIHandle::getNumInstances() == (size_t)num_sockets) - { - // std::cerr << "DEBUG: TPMIHandle::getNumInstances(): " << TPMIHandle::getNumInstances() << "\n"; - UFSStatus.resize(num_sockets); - for (uint32 s = 0; s < (uint32)num_sockets; ++s) - { - try { - TPMIHandle h(s, UFS_ID, UFS_FABRIC_CLUSTER_OFFSET * sizeof(uint64)); - // std::cerr << "DEBUG: Socket " << s << " dies: " << h.getNumEntries() << "\n"; - for (size_t die = 0; die < h.getNumEntries(); ++die) + //TPMIHandle::setVerbose(true); + try { + if (TPMIHandle::getNumInstances() == (size_t)num_sockets) + { + // std::cerr << "DEBUG: TPMIHandle::getNumInstances(): " << TPMIHandle::getNumInstances() << "\n"; + UFSStatus.resize(num_sockets); + for (uint32 s = 0; s < (uint32)num_sockets; ++s) + { + try { + TPMIHandle h(s, UFS_ID, UFS_FABRIC_CLUSTER_OFFSET * sizeof(uint64)); + // std::cerr << "DEBUG: Socket " << s << " dies: " << h.getNumEntries() << "\n"; + for (size_t die = 0; die < h.getNumEntries(); ++die) + { + const auto clusterOffset = extract_bits(h.read64(die), 0, 7); + UFSStatus[s].push_back(std::make_shared(s, UFS_ID, (clusterOffset + UFS_STATUS)* sizeof(uint64))); + } + } catch (std::exception & e) { - const auto clusterOffset = extract_bits(h.read64(die), 0, 7); - UFSStatus[s].push_back(std::make_shared(s, UFS_ID, (clusterOffset + UFS_STATUS)* sizeof(uint64))); + std::cerr << "ERROR: Could not open UFS TPMI register on socket " << s << ". Uncore frequency metrics will be unavailable. Excaption details: " << e.what() << "\n"; } - } catch (std::exception & ) - { - std::cerr << "ERROR: Could not open UFS TPMI register on socket " << s << ". Uncore frequency metrics will be unavailable.\n"; } } + } catch (std::exception & e) + { + std::cerr << "ERROR: Could not initialize TPMI. Uncore frequency metrics will be unavailable. Exception details: " << e.what() << "\n"; } for (uint32 s = 0; s < (uint32)num_sockets; ++s) diff --git a/src/pci.cpp b/src/pci.cpp index 3afcbbfd..f2e30306 100644 --- a/src/pci.cpp +++ b/src/pci.cpp @@ -477,7 +477,7 @@ PciHandleM::PciHandleM(uint32 bus_, uint32 device_, uint32 function_) : fd = handle; int mcfg_handle = PciHandle::openMcfgTable(); - if (mcfg_handle < 0) throw std::exception(); + if (mcfg_handle < 0) throw std::runtime_error("Cannot open any of /[pcm]/sys/firmware/acpi/tables/MCFG* files!"); int32 result = ::pread(mcfg_handle, (void *)&base_addr, sizeof(uint64), 44); @@ -569,11 +569,7 @@ void PciHandleMM::readMCFG() return; // already initialized int mcfg_handle = PciHandle::openMcfgTable(); - - if (mcfg_handle < 0) - { - throw std::exception(); - } + if (mcfg_handle < 0) throw std::runtime_error("cannot open any of /[pcm]/sys/firmware/acpi/tables/MCFG* files!"); ssize_t read_bytes = ::read(mcfg_handle, (void *)&mcfgHeader, sizeof(MCFGHeader)); From fbd9d5d43cd020e6f8cb775a8a52127e5466a9bd Mon Sep 17 00:00:00 2001 From: Pawel Palucki Date: Mon, 3 Jun 2024 16:04:54 +0200 Subject: [PATCH 02/25] Fix typo --- src/cpucounters.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpucounters.cpp b/src/cpucounters.cpp index 48a25844..ef6bdc89 100644 --- a/src/cpucounters.cpp +++ b/src/cpucounters.cpp @@ -1987,7 +1987,7 @@ void PCM::initUncoreObjects() } } catch (std::exception & e) { - std::cerr << "ERROR: Could not open UFS TPMI register on socket " << s << ". Uncore frequency metrics will be unavailable. Excaption details: " << e.what() << "\n"; + std::cerr << "ERROR: Could not open UFS TPMI register on socket " << s << ". Uncore frequency metrics will be unavailable. Exception details: " << e.what() << "\n"; } } } From f25c80bcf5175e563b9e52653300729fb8c416e0 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Wed, 5 Jun 2024 15:13:14 +0200 Subject: [PATCH 03/25] address clang warnings Change-Id: I6ee31341dfa3bf52ce9a4c8981a2449bbe6b319d --- CMakeLists.txt | 16 ++++++++++++---- src/cpucounters.cpp | 15 ++++++++++----- src/pcm-sensor-server.cpp | 12 +++++++----- src/tpmi.cpp | 6 +++--- src/tpmi.h | 1 + src/uncore_pmu_discovery.cpp | 2 -- 6 files changed, 33 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 84e6af68..3de9d970 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -48,7 +48,10 @@ if(UNIX) # APPLE, LINUX, FREE_BSD set (PCM_COMMON_FLAGS "-Wno-unknown-pragmas -fPIC") if(LINUX) - set (PCM_COMMON_FLAGS "${PCM_COMMON_FLAGS} -Wl,-z,now -Wextra -DPCM_USE_PERF") + set (PCM_COMMON_FLAGS "${PCM_COMMON_FLAGS} -Wextra -DPCM_USE_PERF") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set (PCM_COMMON_FLAGS "${PCM_COMMON_FLAGS} -Wl,-z,now") + endif() if(NOT DEFINED LINUX_SYSTEMD) set(LINUX_SYSTEMD FALSE) @@ -74,9 +77,14 @@ if(UNIX) # APPLE, LINUX, FREE_BSD message(STATUS "CMAKE_C_FLAGS: ${CMAKE_C_FLAGS}") set(PCM_OPTIONAL_FLAGS "-Wall") - set(CMAKE_CXX_FLAGS_RELEASE "${PCM_OPTIONAL_FLAGS} -O3 -D_FORTIFY_SOURCE=1 -rdynamic") - set(CMAKE_CXX_FLAGS_DEBUG "${PCM_OPTIONAL_FLAGS} -O0 -g -rdynamic") - set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${PCM_OPTIONAL_FLAGS} -O3 -g -D_FORTIFY_SOURCE=1 -rdynamic") + if ("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU") + set (PCM_DYNAMIC "-rdynamic") + elseif() + set (PCM_DYNAMIC "") + endif() + set(CMAKE_CXX_FLAGS_RELEASE "${PCM_OPTIONAL_FLAGS} -O3 ${PCM_DYNAMIC}") + set(CMAKE_CXX_FLAGS_DEBUG "${PCM_OPTIONAL_FLAGS} -O0 -g ${PCM_DYNAMIC}") + set(CMAKE_CXX_FLAGS_RELWITHDEBINFO "${PCM_OPTIONAL_FLAGS} -O3 -g ${PCM_DYNAMIC}") if(FREE_BSD) set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -lexecinfo") set(CMAKE_CXX_FLAGS_DEBUG "${CMAKE_CXX_FLAGS_DEBUG} -lexecinfo") diff --git a/src/cpucounters.cpp b/src/cpucounters.cpp index ef6bdc89..895396ba 100644 --- a/src/cpucounters.cpp +++ b/src/cpucounters.cpp @@ -1019,12 +1019,13 @@ class QATTelemetryVirtualCounterRegister : public HWRegister { std::shared_ptr gConfigReg; std::shared_ptr controlReg; - int ctr_id; + // int ctr_id; // unused public: - QATTelemetryVirtualCounterRegister( std::shared_ptr gConfigReg_, std::shared_ptr controlReg_, int ctr_id_) : + QATTelemetryVirtualCounterRegister( std::shared_ptr gConfigReg_, + std::shared_ptr controlReg_, + int /* ctr_id_ */ ) : gConfigReg(gConfigReg_), - controlReg(controlReg_), - ctr_id(ctr_id_) + controlReg(controlReg_) { } void operator = (uint64 /* val */) override @@ -5738,7 +5739,11 @@ PCM::ErrorCode PCM::program(const RawPMUConfigs& curPMUConfigs_, const bool sile return true; }; FixedEventControlRegister fixedReg; - auto setOtherConf = [&conf, &fixedReg, &globalRegPos](const RawPMUConfig& corePMUConfig) + auto setOtherConf = [&conf, &fixedReg +#ifdef _MSC_VER + , &globalRegPos +#endif + ](const RawPMUConfig& corePMUConfig) { if ((size_t)globalRegPos < corePMUConfig.programmable.size()) { diff --git a/src/pcm-sensor-server.cpp b/src/pcm-sensor-server.cpp index 72a89ec5..26eccff6 100644 --- a/src/pcm-sensor-server.cpp +++ b/src/pcm-sensor-server.cpp @@ -4,7 +4,9 @@ // Use port allocated for PCM in prometheus: // https://github.com/prometheus/prometheus/wiki/Default-port-allocations constexpr unsigned int DEFAULT_HTTP_PORT = 9738; +#if defined (USE_SSL) constexpr unsigned int DEFAULT_HTTPS_PORT = DEFAULT_HTTP_PORT; +#endif #include "pcm-accel-common.h" #include @@ -969,7 +971,7 @@ class basic_socketbuf : public std::basic_streambuf { return 0; } - virtual int_type overflow( int_type ch ) { + virtual int_type overflow( int_type ch ) override { // send data in buffer and reset it if ( traits_type::eof() != ch ) { *Base::pptr() = ch; @@ -982,7 +984,7 @@ class basic_socketbuf : public std::basic_streambuf { return bytesWritten; // Anything but traits_type::eof() to signal ok. } - virtual int_type underflow() { + virtual int_type underflow() override { std::fill(inputBuffer_, inputBuffer_ + SIZE, 0); ssize_t bytesReceived; @@ -2547,9 +2549,9 @@ class HTTPConnection : public Work { public: HTTPConnection() = delete; #if defined (USE_SSL) - HTTPConnection( HTTPServer* hs, int socketFD, struct sockaddr_in clientAddr, std::vector const & cl, SSL* ssl = nullptr ) : hs_( hs ), socketStream_( socketFD, ssl ), clientAddress_( clientAddr ), callbackList_( cl ) {} + HTTPConnection( HTTPServer* hs, int socketFD, struct sockaddr_in /* clientAddr */, std::vector const & cl, SSL* ssl = nullptr ) : hs_( hs ), socketStream_( socketFD, ssl ), /* clientAddress_( clientAddr ), */ callbackList_( cl ) {} #else - HTTPConnection( HTTPServer* hs, int socketFD, struct sockaddr_in clientAddr, std::vector const & cl ) : hs_( hs ), socketStream_( socketFD ), clientAddress_( clientAddr ), callbackList_( cl ) {} + HTTPConnection( HTTPServer* hs, int socketFD, struct sockaddr_in /* clientAddr */, std::vector const & cl ) : hs_( hs ), socketStream_( socketFD ), /* clientAddress_( clientAddr ), */ callbackList_( cl ) {} #endif HTTPConnection( HTTPConnection const & ) = delete; void operator=( HTTPConnection const & ) = delete; @@ -2644,7 +2646,7 @@ class HTTPConnection : public Work { private: HTTPServer* hs_; socketstream socketStream_; - struct sockaddr_in clientAddress_; + // struct sockaddr_in clientAddress_; // Not used yet std::vector const & callbackList_; std::vector responseHeader_; std::string responseBody_; diff --git a/src/tpmi.cpp b/src/tpmi.cpp index 9056b9e0..07aacfda 100644 --- a/src/tpmi.cpp +++ b/src/tpmi.cpp @@ -197,7 +197,7 @@ class TPMIHandleDriver : public TPMIHandleInterface const size_t instance; const size_t ID; const size_t offset; - const bool readonly; + // const bool readonly; // not used size_t nentries; struct TPMIEntry { unsigned int offset{0}; @@ -271,11 +271,11 @@ class TPMIHandleDriver : public TPMIHandleInterface } public: static size_t getNumInstances(); - TPMIHandleDriver(const size_t instance_, const size_t ID_, const size_t offset_, const bool readonly_ = true) : + TPMIHandleDriver(const size_t instance_, const size_t ID_, const size_t offset_, const bool /* readonly_ */ = true) : instance(instance_), ID(ID_), offset(offset_), - readonly(readonly_), + // readonly(readonly_), // not used nentries(0) { assert(available > 0); diff --git a/src/tpmi.h b/src/tpmi.h index dfee7197..ab819dec 100644 --- a/src/tpmi.h +++ b/src/tpmi.h @@ -21,6 +21,7 @@ class TPMIHandleInterface virtual size_t getNumEntries() const = 0; virtual uint64 read64(size_t entryPos) = 0; virtual void write64(size_t entryPos, uint64 val) = 0; + virtual ~TPMIHandleInterface() {} }; class TPMIHandle : public TPMIHandleInterface diff --git a/src/uncore_pmu_discovery.cpp b/src/uncore_pmu_discovery.cpp index aa2685a0..61fa4493 100644 --- a/src/uncore_pmu_discovery.cpp +++ b/src/uncore_pmu_discovery.cpp @@ -9,8 +9,6 @@ namespace pcm { -constexpr auto UNCORE_DISCOVERY_MAP_SIZE = 0x80000; - UncorePMUDiscovery::UncorePMUDiscovery() { if (safe_getenv("PCM_NO_UNCORE_PMU_DISCOVERY") == std::string("1")) From 484f541e28033a0b26efa8d9bf7e9370ed16a935 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 7 Jun 2024 06:03:34 -0700 Subject: [PATCH 04/25] add fuzzing poc --- tests/CMakeLists.txt | 11 +++++++++++ tests/fuzz.sh | 7 +++++++ tests/urltest-fuzz.cpp | 25 +++++++++++++++++++++++++ 3 files changed, 43 insertions(+) create mode 100644 tests/fuzz.sh create mode 100644 tests/urltest-fuzz.cpp diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 1d10dc8d..b05cdab9 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -17,3 +17,14 @@ if(UNIX) endif(LINUX) endif(UNIX) + +if(FUZZ) + add_executable(urltest-fuzz urltest-fuzz.cpp) + # TODO add_executable(pcm-sensor-server-fuzz pcm-sensor-server-fuzz.cpp) + set(FUZZER_OPTIONS "-fsanitize=fuzzer,address -fprofile-instr-generate -fcoverage-mapping") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUZZER_OPTIONS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FUZZER_OPTIONS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FUZZER_OPTIONS}") + target_link_libraries(urltest-fuzz Threads::Threads PCM_STATIC) + # TODO: target_link_libraries(pcm-sensor-server-fuzz Threads::Threads PCM_STATIC) +endif() diff --git a/tests/fuzz.sh b/tests/fuzz.sh new file mode 100644 index 00000000..6eab531d --- /dev/null +++ b/tests/fuzz.sh @@ -0,0 +1,7 @@ + +CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir corpus && +make urltest-fuzz && +bin/tests/urltest-fuzz -max_total_time=30 corpus && +llvm-profdata merge -sparse default.profraw -o default.profdata && +llvm-cov report --summary-only ./bin/tests/urltest-fuzz -instr-profile=default.profdata + diff --git a/tests/urltest-fuzz.cpp b/tests/urltest-fuzz.cpp new file mode 100644 index 00000000..6cd5d835 --- /dev/null +++ b/tests/urltest-fuzz.cpp @@ -0,0 +1,25 @@ +#define UNIT_TEST 1 + +#include "../src/pcm-sensor-server.cpp" + +#undef UNIT_TEST + + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + try + { + std::string buf(reinterpret_cast(data), size); + buf.push_back('\0'); + URL x = URL::parse(buf.c_str()); + } + catch (std::runtime_error & ) + { + // catch recognized malformed input (thrown as runtime_error in the URL::parse) + // do not catch any other errors or exceptions to let them be reported + // by libFuzzer + } + + return 0; +} + From 1db51bd9ed4fe99fe4f92ae44a113c6f859f9d0e Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 7 Jun 2024 15:20:36 +0200 Subject: [PATCH 05/25] add github action for fuzzing Change-Id: I1f6784df878b4739d0a88c611712afee7422c71d --- .github/workflows/ci-fuzz.yml | 36 +++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflows/ci-fuzz.yml diff --git a/.github/workflows/ci-fuzz.yml b/.github/workflows/ci-fuzz.yml new file mode 100644 index 00000000..b5aeeb1c --- /dev/null +++ b/.github/workflows/ci-fuzz.yml @@ -0,0 +1,36 @@ +name: fuzz-job + +on: + schedule: + # every sunday at midnight + - cron: '0 0 * * 0' + # also allow manual triggering + workflow_dispatch: + + +permissions: + contents: read + +jobs: + fuzz: + runs-on: ci-test + if: ${{ github.repository != 'intel/pcm' }} + + steps: + - name: Harden Runner + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 + with: + egress-policy: audit + + - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0 + with: + submodules: recursive + + - name: Build and test + run: | + cmake --version + set -o pipefail + mkdir build + cd build + sh ${{ github.workspace }}/tests/fuzz.sh 2>&1 | tee fuzz-log.txt + From 15552b0249b73c8ec2711c914a2aa2717152a043 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 13 Jun 2024 11:42:24 +0200 Subject: [PATCH 06/25] refactor tests/fuzz.sh --- tests/fuzz.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/fuzz.sh b/tests/fuzz.sh index 6eab531d..fafde379 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,7 +1,7 @@ -CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir corpus && +CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && make urltest-fuzz && -bin/tests/urltest-fuzz -max_total_time=30 corpus && -llvm-profdata merge -sparse default.profraw -o default.profdata && -llvm-cov report --summary-only ./bin/tests/urltest-fuzz -instr-profile=default.profdata +LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=30 corpus && +llvm-profdata merge -sparse urltest.profraw -o all.profdata && +llvm-cov report --summary-only ./bin/tests/urltest-fuzz -instr-profile=all.profdata From 10bdec752ed7316609d0c82e785f44d42e19c1fa Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 13 Jun 2024 11:49:07 +0200 Subject: [PATCH 07/25] compile all modules with fuzzing --- CMakeLists.txt | 7 +++++++ tests/CMakeLists.txt | 4 ---- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3de9d970..559aee9a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -96,6 +96,13 @@ if(UNIX) # APPLE, LINUX, FREE_BSD endif(UNIX) +if(FUZZ) + set(FUZZER_OPTIONS "-fsanitize=fuzzer,address -fprofile-instr-generate -fcoverage-mapping") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUZZER_OPTIONS}") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FUZZER_OPTIONS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FUZZER_OPTIONS}") +endif(FUZZ) + ####################### # Install ####################### diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index b05cdab9..c9c07313 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,10 +21,6 @@ endif(UNIX) if(FUZZ) add_executable(urltest-fuzz urltest-fuzz.cpp) # TODO add_executable(pcm-sensor-server-fuzz pcm-sensor-server-fuzz.cpp) - set(FUZZER_OPTIONS "-fsanitize=fuzzer,address -fprofile-instr-generate -fcoverage-mapping") - set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUZZER_OPTIONS}") - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${FUZZER_OPTIONS}") - set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${FUZZER_OPTIONS}") target_link_libraries(urltest-fuzz Threads::Threads PCM_STATIC) # TODO: target_link_libraries(pcm-sensor-server-fuzz Threads::Threads PCM_STATIC) endif() From 916f3e28a646ad43979d3e53f413729c70be0daf Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Thu, 13 Jun 2024 12:54:40 +0200 Subject: [PATCH 08/25] increase fuzzing coverage --- src/pcm.cpp | 4 +++ tests/CMakeLists.txt | 2 ++ tests/fuzz.sh | 7 ++-- tests/pcm-fuzz.cpp | 77 ++++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 tests/pcm-fuzz.cpp diff --git a/src/pcm.cpp b/src/pcm.cpp index 5eb28782..60ed261a 100644 --- a/src/pcm.cpp +++ b/src/pcm.cpp @@ -1340,6 +1340,8 @@ void print_csv(PCM * m, } } +#ifndef UNIT_TEST + PCM_MAIN_NOTHROW; int mainThrows(int argc, char * argv[]) @@ -1632,3 +1634,5 @@ int mainThrows(int argc, char * argv[]) exit(EXIT_SUCCESS); } + +#endif // UNIT_TEST \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c9c07313..3fd1548c 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -21,6 +21,8 @@ endif(UNIX) if(FUZZ) add_executable(urltest-fuzz urltest-fuzz.cpp) # TODO add_executable(pcm-sensor-server-fuzz pcm-sensor-server-fuzz.cpp) + add_executable(pcm-fuzz pcm-fuzz.cpp) target_link_libraries(urltest-fuzz Threads::Threads PCM_STATIC) # TODO: target_link_libraries(pcm-sensor-server-fuzz Threads::Threads PCM_STATIC) + target_link_libraries(pcm-fuzz Threads::Threads PCM_STATIC) endif() diff --git a/tests/fuzz.sh b/tests/fuzz.sh index fafde379..5179356f 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,7 +1,8 @@ CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && -make urltest-fuzz && +make urltest-fuzz pcm-fuzz -j && LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=30 corpus && -llvm-profdata merge -sparse urltest.profraw -o all.profdata && -llvm-cov report --summary-only ./bin/tests/urltest-fuzz -instr-profile=all.profdata +LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=30 corpus && +llvm-profdata merge -sparse urltest.profraw pcm.profraw -o all.profdata && +llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -instr-profile=all.profdata diff --git a/tests/pcm-fuzz.cpp b/tests/pcm-fuzz.cpp new file mode 100644 index 00000000..52facf20 --- /dev/null +++ b/tests/pcm-fuzz.cpp @@ -0,0 +1,77 @@ +#define UNIT_TEST 1 + +#include "../src/pcm.cpp" + +#undef UNIT_TEST + + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + auto m = PCM::getInstance(); + const int *data_int = reinterpret_cast(data); + size_t size_int = size / sizeof(int); + + if (size_int < 1) { + return 0; + } + + int pid = data_int[0]; + + m->resetPMU(); + + m->disableJKTWorkaround(); + + const PCM::ErrorCode status = m->program(PCM::DEFAULT_EVENTS, nullptr, false, pid); + + switch (status) + { + case PCM::Success: + break; + case PCM::UnknownError: // expected for invalid pid + return 0; + case PCM::MSRAccessDenied: + cerr << "Access to Intel(r) Performance Counter Monitor has denied (no MSR or PCI CFG space access).\n"; + exit(EXIT_FAILURE); + case PCM::PMUBusy: + cerr << "Access to Intel(r) Performance Counter Monitor has denied (Performance Monitoring Unit is occupied by other application). Try to stop the application that uses PMU.\n"; + cerr << "Alternatively you can try running PCM with option -r to reset PMU.\n"; + exit(EXIT_FAILURE); + default: + cerr << "Access to Intel(r) Performance Counter Monitor has denied (Unknown error).\n"; + exit(EXIT_FAILURE); + } + + if (size_int < 6) + { + return 0; + } + + print_cpu_details(); + + std::vector cstates1, cstates2; + std::vector sktstate1, sktstate2; + SystemCounterState sstate1, sstate2; + bitset ycores; + const auto cpu_model = m->getCPUModel(); + + print_pid_collection_message(pid); + bool show_partial_core_output = false; // TODO: add support for partial core output + bool csv_output = data_int[1] % 2; + int metricVersion = data_int[2]; + bool show_socket_output = data_int[3] % 2; + bool show_system_output = data_int[4] % 2; + bool show_core_output = data_int[5] % 2; + + m->getAllCounterStates(sstate1, sktstate1, cstates1); + m->getAllCounterStates(sstate2, sktstate2, cstates2); + if (csv_output) + print_csv(m, cstates1, cstates2, sktstate1, sktstate2, ycores, sstate1, sstate2, + cpu_model, show_core_output, show_partial_core_output, show_socket_output, show_system_output); + else + print_output(m, cstates1, cstates2, sktstate1, sktstate2, ycores, sstate1, sstate2, + cpu_model, show_core_output, show_partial_core_output, show_socket_output, show_system_output, + metricVersion); + + return 0; +} + From 2d60379644a01e5f8fa3a8c932808bb87da46b34 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Thu, 13 Jun 2024 13:58:41 +0200 Subject: [PATCH 09/25] increase fuzzing duration Change-Id: If94ecaf7dc79afc8c7d4a5b9bab596557099073d --- tests/fuzz.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/fuzz.sh b/tests/fuzz.sh index 5179356f..c6d41e66 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,8 +1,8 @@ CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && make urltest-fuzz pcm-fuzz -j && -LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=30 corpus && -LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=30 corpus && +LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=2000 corpus && +LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=2000 corpus && llvm-profdata merge -sparse urltest.profraw pcm.profraw -o all.profdata && llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -instr-profile=all.profdata From cbeadc05dcbb8580575e554292b467149814d68e Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Thu, 13 Jun 2024 15:27:54 +0200 Subject: [PATCH 10/25] increase fuzzing coverage (II) Change-Id: I19d632dbf025bbeca4f19b0ff987d9618ce202a2 --- tests/fuzz.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/fuzz.sh b/tests/fuzz.sh index c6d41e66..187ab3d2 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,8 +1,11 @@ CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && make urltest-fuzz pcm-fuzz -j && -LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=2000 corpus && -LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=2000 corpus && -llvm-profdata merge -sparse urltest.profraw pcm.profraw -o all.profdata && +LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=1000 corpus && +LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=500 corpus && +LLVM_PROFILE_FILE="pcm.no_perf.profraw" PCM_NO_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && +LLVM_PROFILE_FILE="pcm.uncore_perf.profraw" PCM_USE_UNCORE_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && +LLVM_PROFILE_FILE="pcm.nmi_watchdog.profraw" PCM_KEEP_NMI_WATCHDOG=1 bin/tests/pcm-fuzz -max_total_time=100 corpus && +llvm-profdata merge -sparse urltest.profraw pcm.profraw pcm.no_perf.profraw pcm.uncore_perf.profraw pcm.nmi_watchdog.profraw -o all.profdata && llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -instr-profile=all.profdata From e3a8814b3db0230e319068d0913680e04326f708 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 14 Jun 2024 09:15:13 +0200 Subject: [PATCH 11/25] increase fuzz coverage --- tests/pcm-fuzz.cpp | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/tests/pcm-fuzz.cpp b/tests/pcm-fuzz.cpp index 52facf20..2f5a3e4f 100644 --- a/tests/pcm-fuzz.cpp +++ b/tests/pcm-fuzz.cpp @@ -7,15 +7,22 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { - auto m = PCM::getInstance(); - const int *data_int = reinterpret_cast(data); size_t size_int = size / sizeof(int); - - if (size_int < 1) { + const auto ints_used = 7; + if (size_int < ints_used) + { return 0; } - int pid = data_int[0]; + auto m = PCM::getInstance(); + const int *data_int = reinterpret_cast(data); + int pos = 0; + int pid = data_int[pos++]; + bool use_pid = data_int[pos++] % 2; + if (!use_pid) + { + pid = -1; + } m->resetPMU(); @@ -41,11 +48,6 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) exit(EXIT_FAILURE); } - if (size_int < 6) - { - return 0; - } - print_cpu_details(); std::vector cstates1, cstates2; @@ -56,11 +58,12 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) print_pid_collection_message(pid); bool show_partial_core_output = false; // TODO: add support for partial core output - bool csv_output = data_int[1] % 2; - int metricVersion = data_int[2]; - bool show_socket_output = data_int[3] % 2; - bool show_system_output = data_int[4] % 2; - bool show_core_output = data_int[5] % 2; + bool csv_output = data_int[pos++] % 2; + int metricVersion = data_int[pos++]; + bool show_socket_output = data_int[pos++] % 2; + bool show_system_output = data_int[pos++] % 2; + bool show_core_output = data_int[pos++] % 2; + assert(pos == ints_used); m->getAllCounterStates(sstate1, sktstate1, cstates1); m->getAllCounterStates(sstate2, sktstate2, cstates2); From 649b16ad12eaec47404ef4ac49afc3ac44292ca7 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Fri, 14 Jun 2024 11:58:49 +0200 Subject: [PATCH 12/25] fuzz pcm-memory --- src/pcm-memory.cpp | 3 + tests/CMakeLists.txt | 2 + tests/fuzz.sh | 7 ++- tests/pcm-fuzz.cpp | 2 + tests/pcm-memory-fuzz.cpp | 116 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 127 insertions(+), 3 deletions(-) create mode 100644 tests/pcm-memory-fuzz.cpp diff --git a/src/pcm-memory.cpp b/src/pcm-memory.cpp index 745d7baf..ba08e513 100644 --- a/src/pcm-memory.cpp +++ b/src/pcm-memory.cpp @@ -1326,6 +1326,7 @@ class CHAEventCollector } }; +#ifndef UNIT_TEST PCM_MAIN_NOTHROW; @@ -1641,3 +1642,5 @@ int mainThrows(int argc, char * argv[]) exit(EXIT_SUCCESS); } + +#endif // UNIT_TEST \ No newline at end of file diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 3fd1548c..a55a15c5 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -22,7 +22,9 @@ if(FUZZ) add_executable(urltest-fuzz urltest-fuzz.cpp) # TODO add_executable(pcm-sensor-server-fuzz pcm-sensor-server-fuzz.cpp) add_executable(pcm-fuzz pcm-fuzz.cpp) + add_executable(pcm-memory-fuzz pcm-memory-fuzz.cpp) target_link_libraries(urltest-fuzz Threads::Threads PCM_STATIC) # TODO: target_link_libraries(pcm-sensor-server-fuzz Threads::Threads PCM_STATIC) target_link_libraries(pcm-fuzz Threads::Threads PCM_STATIC) + target_link_libraries(pcm-memory-fuzz Threads::Threads PCM_STATIC) endif() diff --git a/tests/fuzz.sh b/tests/fuzz.sh index 187ab3d2..c9ccd1e0 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,11 +1,12 @@ CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && -make urltest-fuzz pcm-fuzz -j && +make urltest-fuzz pcm-fuzz pcm-memory-fuzz -j && LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=1000 corpus && LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=500 corpus && LLVM_PROFILE_FILE="pcm.no_perf.profraw" PCM_NO_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && LLVM_PROFILE_FILE="pcm.uncore_perf.profraw" PCM_USE_UNCORE_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && LLVM_PROFILE_FILE="pcm.nmi_watchdog.profraw" PCM_KEEP_NMI_WATCHDOG=1 bin/tests/pcm-fuzz -max_total_time=100 corpus && -llvm-profdata merge -sparse urltest.profraw pcm.profraw pcm.no_perf.profraw pcm.uncore_perf.profraw pcm.nmi_watchdog.profraw -o all.profdata && -llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -instr-profile=all.profdata +LLVM_PROFILE_FILE="pcm-memory.profraw" bin/tests/pcm-memory-fuzz -max_total_time=1000 corpus && +llvm-profdata merge -sparse urltest.profraw pcm.profraw pcm.no_perf.profraw pcm.uncore_perf.profraw pcm.nmi_watchdog.profraw pcm-memory.profraw -o all.profdata && +llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -object ./bin/tests/pcm-memory-fuzz -instr-profile=all.profdata diff --git a/tests/pcm-fuzz.cpp b/tests/pcm-fuzz.cpp index 2f5a3e4f..047d0a8a 100644 --- a/tests/pcm-fuzz.cpp +++ b/tests/pcm-fuzz.cpp @@ -24,6 +24,8 @@ extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) pid = -1; } + print_help(""); + m->resetPMU(); m->disableJKTWorkaround(); diff --git a/tests/pcm-memory-fuzz.cpp b/tests/pcm-memory-fuzz.cpp new file mode 100644 index 00000000..f91841c1 --- /dev/null +++ b/tests/pcm-memory-fuzz.cpp @@ -0,0 +1,116 @@ +#define UNIT_TEST 1 + +#include "../src/pcm-memory.cpp" + +#undef UNIT_TEST + + +extern "C" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) +{ + size_t size_int = size / sizeof(int); + const auto ints_used = 9; + if (size_int < ints_used) + { + return 0; + } + print_help(""); + + auto m = PCM::getInstance(); + const int *data_int = reinterpret_cast(data); + int pos = 0; + + bool csv = data_int[pos++] % 2; + bool csvheader = data_int[pos++] % 2; + bool show_channel_output = data_int[pos++] % 2; + bool print_update = data_int[pos++] % 2; + uint32 no_columns = DEFAULT_DISPLAY_COLUMNS; // Default number of columns is 2 + int delay = data_int[pos++] % 4; + int rankA = data_int[pos++]; + int rankB = data_int[pos++]; + bool use_rank = data_int[pos++] % 2; + if (!use_rank) + { + rankA = -1; + rankB = -1; + } + + + ServerUncoreMemoryMetrics metrics; + switch (data_int[pos++] % 4) + { + case 0: + metrics = PartialWrites; + break; + case 1: + metrics = Pmem; + break; + case 2: + metrics = PmemMemoryMode; + break; + case 3: + metrics = PmemMixedMode; + break; + } + + assert(pos == ints_used); + + m->resetPMU(); + m->disableJKTWorkaround(); + + const auto cpu_model = m->getCPUModel(); + if (!m->hasPCICFGUncore()) + { + cerr << "Unsupported processor model (" << cpu_model << ").\n"; + if (m->memoryTrafficMetricsAvailable()) + cerr << "For processor-level memory bandwidth statistics please use 'pcm' utility\n"; + return 0; + } + if (anyPmem(metrics) && (m->PMMTrafficMetricsAvailable() == false)) + { + cerr << "PMM/Pmem traffic metrics are not available on your processor.\n"; + return 0; + } + if (metrics == PmemMemoryMode && m->PMMMemoryModeMetricsAvailable() == false) + { + cerr << "PMM Memory Mode metrics are not available on your processor.\n"; + return 0; + } + if (metrics == PmemMixedMode && m->PMMMixedModeMetricsAvailable() == false) + { + cerr << "PMM Mixed Mode metrics are not available on your processor.\n"; + return 0; + } + if((rankA >= 0 || rankB >= 0) && anyPmem(metrics)) + { + cerr << "PMM/Pmem traffic metrics are not available on rank level\n"; + return 0; + } + if((rankA >= 0 || rankB >= 0) && !show_channel_output) + { + cerr << "Rank level output requires channel output\n"; + return 0; + } + PCM::ErrorCode status = m->programServerUncoreMemoryMetrics(metrics, rankA, rankB); + m->checkError(status); + + max_imc_channels = (pcm::uint32)m->getMCChannelsPerSocket(); + + std::vector BeforeState(m->getNumSockets()); + std::vector AfterState(m->getNumSockets()); + uint64 BeforeTime = 0, AfterTime = 0; + + readState(BeforeState); + BeforeTime = m->getTickCount(); + MySleepMs(delay); + AfterTime = m->getTickCount(); + readState(AfterState); + + if(rankA >= 0 || rankB >= 0) + calculate_bandwidth_rank(m,BeforeState, AfterState, AfterTime - BeforeTime, csv, csvheader, no_columns, rankA, rankB); + else + calculate_bandwidth(m,BeforeState,AfterState,AfterTime-BeforeTime,csv,csvheader, no_columns, metrics, + show_channel_output, print_update, 0); + + return 0; +} + From ac461f1da2d479997007f98dc4b5f6716f92796e Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Fri, 14 Jun 2024 15:46:59 +0200 Subject: [PATCH 13/25] reduce output Change-Id: I9877f5115e3c31ec989804eb5f95e0f4de5220d7 --- tests/fuzz.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/fuzz.sh b/tests/fuzz.sh index c9ccd1e0..3d21b211 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,12 +1,12 @@ CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && make urltest-fuzz pcm-fuzz pcm-memory-fuzz -j && -LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=1000 corpus && -LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=500 corpus && -LLVM_PROFILE_FILE="pcm.no_perf.profraw" PCM_NO_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && -LLVM_PROFILE_FILE="pcm.uncore_perf.profraw" PCM_USE_UNCORE_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus && -LLVM_PROFILE_FILE="pcm.nmi_watchdog.profraw" PCM_KEEP_NMI_WATCHDOG=1 bin/tests/pcm-fuzz -max_total_time=100 corpus && -LLVM_PROFILE_FILE="pcm-memory.profraw" bin/tests/pcm-memory-fuzz -max_total_time=1000 corpus && +LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=1000 corpus > /dev/null && +LLVM_PROFILE_FILE="pcm.profraw" bin/tests/pcm-fuzz -max_total_time=500 corpus > /dev/null && +LLVM_PROFILE_FILE="pcm.no_perf.profraw" PCM_NO_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus > /dev/null && +LLVM_PROFILE_FILE="pcm.uncore_perf.profraw" PCM_USE_UNCORE_PERF=1 bin/tests/pcm-fuzz -max_total_time=500 corpus > /dev/null && +LLVM_PROFILE_FILE="pcm.nmi_watchdog.profraw" PCM_KEEP_NMI_WATCHDOG=1 bin/tests/pcm-fuzz -max_total_time=100 corpus > /dev/null && +LLVM_PROFILE_FILE="pcm-memory.profraw" bin/tests/pcm-memory-fuzz -max_total_time=1000 corpus > /dev/null && llvm-profdata merge -sparse urltest.profraw pcm.profraw pcm.no_perf.profraw pcm.uncore_perf.profraw pcm.nmi_watchdog.profraw pcm-memory.profraw -o all.profdata && llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -object ./bin/tests/pcm-memory-fuzz -instr-profile=all.profdata From 93cb71ee44eb6ed99c8099e7e5c79c773aaf07ef Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sun, 16 Jun 2024 16:48:02 +0200 Subject: [PATCH 14/25] show fuzzing report and upload the log Change-Id: I2587c00f35fca169a002527872361c949d8fb13b --- .github/workflows/ci-fuzz.yml | 13 +++++++++++++ tests/fuzz.sh | 2 +- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci-fuzz.yml b/.github/workflows/ci-fuzz.yml index b5aeeb1c..30b182fd 100644 --- a/.github/workflows/ci-fuzz.yml +++ b/.github/workflows/ci-fuzz.yml @@ -33,4 +33,17 @@ jobs: mkdir build cd build sh ${{ github.workspace }}/tests/fuzz.sh 2>&1 | tee fuzz-log.txt + cd .. + + - name: Show report + run: | + cat build/report.txt + echo "Fuzzing completed" + + - name: upload-artifact + uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: fuzz-log-${{ github.sha }} + path: "build/fuzz-log.txt" + diff --git a/tests/fuzz.sh b/tests/fuzz.sh index 3d21b211..f64df9f2 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -8,5 +8,5 @@ LLVM_PROFILE_FILE="pcm.uncore_perf.profraw" PCM_USE_UNCORE_PERF=1 bin/tests/pcm- LLVM_PROFILE_FILE="pcm.nmi_watchdog.profraw" PCM_KEEP_NMI_WATCHDOG=1 bin/tests/pcm-fuzz -max_total_time=100 corpus > /dev/null && LLVM_PROFILE_FILE="pcm-memory.profraw" bin/tests/pcm-memory-fuzz -max_total_time=1000 corpus > /dev/null && llvm-profdata merge -sparse urltest.profraw pcm.profraw pcm.no_perf.profraw pcm.uncore_perf.profraw pcm.nmi_watchdog.profraw pcm-memory.profraw -o all.profdata && -llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -object ./bin/tests/pcm-memory-fuzz -instr-profile=all.profdata +llvm-cov report --summary-only -object ./bin/tests/pcm-fuzz -object ./bin/tests/urltest-fuzz -object ./bin/tests/pcm-memory-fuzz -instr-profile=all.profdata | tee report.txt From 9fa93c4d1e77311e5019a7d830968db645b9b0a9 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Sun, 16 Jun 2024 16:53:21 +0200 Subject: [PATCH 15/25] Revert "Bump step-security/harden-runner from 2.6.1 to 2.8.1" This reverts commit 816e2eef50a8306b64fb2a974a5111a800796a90. --- .github/workflows/ci-clang-scan.yml | 2 +- .github/workflows/ci-cmake-options.yml | 2 +- .github/workflows/ci-cov-linux-report.yml | 2 +- .github/workflows/ci-cov-linux.yml | 2 +- .github/workflows/ci-cov-python.yml | 2 +- .github/workflows/ci-cov-windows-report.yml | 2 +- .github/workflows/ci-cov-windows.yml | 2 +- .github/workflows/ci-cpack.yml | 4 ++-- .github/workflows/ci-cppcheck.yml | 2 +- .github/workflows/ci-gcc10.yml | 2 +- .github/workflows/ci-gcc48.yml | 2 +- .github/workflows/ci-gcc5.yml | 2 +- .github/workflows/ci-gcc7.yml | 2 +- .github/workflows/ci-gcc9.yml | 2 +- .github/workflows/ci-test.yml | 2 +- .github/workflows/ci-windows-ip.yml | 2 +- .github/workflows/ci-windows.yml | 2 +- .github/workflows/clang_scan.yml | 2 +- .github/workflows/codeql.yml | 2 +- .github/workflows/cppcheck.yml | 2 +- .github/workflows/dependency-review.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/freebsd_build.yml | 2 +- .github/workflows/freebsd_scan_build.yml | 2 +- .github/workflows/linux_make.yml | 8 ++++---- .github/workflows/macos-scan-build.yml | 2 +- .github/workflows/macosx_build.yml | 2 +- .github/workflows/scorecard.yml | 2 +- .github/workflows/stats-cron-job.yml | 2 +- 29 files changed, 33 insertions(+), 33 deletions(-) diff --git a/.github/workflows/ci-clang-scan.yml b/.github/workflows/ci-clang-scan.yml index 1fd28b5a..ce5fce19 100644 --- a/.github/workflows/ci-clang-scan.yml +++ b/.github/workflows/ci-clang-scan.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cmake-options.yml b/.github/workflows/ci-cmake-options.yml index 6fa316ec..1805aff9 100644 --- a/.github/workflows/ci-cmake-options.yml +++ b/.github/workflows/ci-cmake-options.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cov-linux-report.yml b/.github/workflows/ci-cov-linux-report.yml index 9f299859..1d15abac 100644 --- a/.github/workflows/ci-cov-linux-report.yml +++ b/.github/workflows/ci-cov-linux-report.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cov-linux.yml b/.github/workflows/ci-cov-linux.yml index 61179e13..c8c5c271 100644 --- a/.github/workflows/ci-cov-linux.yml +++ b/.github/workflows/ci-cov-linux.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cov-python.yml b/.github/workflows/ci-cov-python.yml index fa5693a5..e2e3d16a 100644 --- a/.github/workflows/ci-cov-python.yml +++ b/.github/workflows/ci-cov-python.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cov-windows-report.yml b/.github/workflows/ci-cov-windows-report.yml index a74dc2fe..460bfb91 100644 --- a/.github/workflows/ci-cov-windows-report.yml +++ b/.github/workflows/ci-cov-windows-report.yml @@ -13,7 +13,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cov-windows.yml b/.github/workflows/ci-cov-windows.yml index c4d4b779..147f55d5 100644 --- a/.github/workflows/ci-cov-windows.yml +++ b/.github/workflows/ci-cov-windows.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cpack.yml b/.github/workflows/ci-cpack.yml index 35e619a8..6cad4653 100644 --- a/.github/workflows/ci-cpack.yml +++ b/.github/workflows/ci-cpack.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit @@ -44,7 +44,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-cppcheck.yml b/.github/workflows/ci-cppcheck.yml index 5dffe6a1..59e2bca8 100644 --- a/.github/workflows/ci-cppcheck.yml +++ b/.github/workflows/ci-cppcheck.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-gcc10.yml b/.github/workflows/ci-gcc10.yml index c311ebb5..3875ce0d 100644 --- a/.github/workflows/ci-gcc10.yml +++ b/.github/workflows/ci-gcc10.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-gcc48.yml b/.github/workflows/ci-gcc48.yml index a7995729..8f117fff 100644 --- a/.github/workflows/ci-gcc48.yml +++ b/.github/workflows/ci-gcc48.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-gcc5.yml b/.github/workflows/ci-gcc5.yml index 6ff1b31d..109d620c 100644 --- a/.github/workflows/ci-gcc5.yml +++ b/.github/workflows/ci-gcc5.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-gcc7.yml b/.github/workflows/ci-gcc7.yml index 6ef2975c..b58b097a 100644 --- a/.github/workflows/ci-gcc7.yml +++ b/.github/workflows/ci-gcc7.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-gcc9.yml b/.github/workflows/ci-gcc9.yml index 565999ef..9747ae22 100644 --- a/.github/workflows/ci-gcc9.yml +++ b/.github/workflows/ci-gcc9.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-test.yml b/.github/workflows/ci-test.yml index b51b39ab..b18a7289 100644 --- a/.github/workflows/ci-test.yml +++ b/.github/workflows/ci-test.yml @@ -17,7 +17,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-windows-ip.yml b/.github/workflows/ci-windows-ip.yml index d9c40cc7..5a68ae23 100644 --- a/.github/workflows/ci-windows-ip.yml +++ b/.github/workflows/ci-windows-ip.yml @@ -14,7 +14,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/ci-windows.yml b/.github/workflows/ci-windows.yml index aaa08d2b..d83c3d54 100644 --- a/.github/workflows/ci-windows.yml +++ b/.github/workflows/ci-windows.yml @@ -20,7 +20,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/clang_scan.yml b/.github/workflows/clang_scan.yml index 40d8a1be..0dbae717 100644 --- a/.github/workflows/clang_scan.yml +++ b/.github/workflows/clang_scan.yml @@ -28,7 +28,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 707ed1cb..57d12c32 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -41,7 +41,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/cppcheck.yml b/.github/workflows/cppcheck.yml index c61d916e..16fe9192 100644 --- a/.github/workflows/cppcheck.yml +++ b/.github/workflows/cppcheck.yml @@ -27,7 +27,7 @@ jobs: steps: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/dependency-review.yml b/.github/workflows/dependency-review.yml index ff74766f..e82880c5 100644 --- a/.github/workflows/dependency-review.yml +++ b/.github/workflows/dependency-review.yml @@ -19,7 +19,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 1650ea9a..2fd02081 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/freebsd_build.yml b/.github/workflows/freebsd_build.yml index f6ab4c35..e94468c0 100644 --- a/.github/workflows/freebsd_build.yml +++ b/.github/workflows/freebsd_build.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/freebsd_scan_build.yml b/.github/workflows/freebsd_scan_build.yml index 7d135788..72f625a2 100644 --- a/.github/workflows/freebsd_scan_build.yml +++ b/.github/workflows/freebsd_scan_build.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/linux_make.yml b/.github/workflows/linux_make.yml index f15644b9..83400193 100644 --- a/.github/workflows/linux_make.yml +++ b/.github/workflows/linux_make.yml @@ -14,7 +14,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit @@ -31,7 +31,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit @@ -49,7 +49,7 @@ jobs: container: ubuntu:22.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit @@ -70,7 +70,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/macos-scan-build.yml b/.github/workflows/macos-scan-build.yml index 2fac9293..f91f545a 100644 --- a/.github/workflows/macos-scan-build.yml +++ b/.github/workflows/macos-scan-build.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/macosx_build.yml b/.github/workflows/macosx_build.yml index 17273652..c8b78061 100644 --- a/.github/workflows/macosx_build.yml +++ b/.github/workflows/macosx_build.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml index 1038b766..f8ef1970 100644 --- a/.github/workflows/scorecard.yml +++ b/.github/workflows/scorecard.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit diff --git a/.github/workflows/stats-cron-job.yml b/.github/workflows/stats-cron-job.yml index 71de62c4..99b06fda 100644 --- a/.github/workflows/stats-cron-job.yml +++ b/.github/workflows/stats-cron-job.yml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-20.04 steps: - name: Harden Runner - uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1 + uses: step-security/harden-runner@eb238b55efaa70779f274895e782ed17c84f2895 # v2.6.1 with: egress-policy: audit From 54dc987ad9bf9d6617ec8872343df1e146d13fc6 Mon Sep 17 00:00:00 2001 From: Thomas Willhalm Date: Tue, 18 Jun 2024 09:55:40 +0200 Subject: [PATCH 16/25] Update .bdsignore.all Ignore more files in scan --- .bdsignore.all | 1 + 1 file changed, 1 insertion(+) diff --git a/.bdsignore.all b/.bdsignore.all index a5ff86ea..86272d9a 100644 --- a/.bdsignore.all +++ b/.bdsignore.all @@ -35,5 +35,6 @@ My Inspector XE Results - pcm .codedocs .cproject .project +.dockerignore makefile sources From d072217cb5434598393942371b24b8cdd4378fb5 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Tue, 18 Jun 2024 15:49:07 +0200 Subject: [PATCH 17/25] erase unneeded environment variables Change-Id: I3a6653f8ec39ecc0ef2471bb54a385b1eed0387b --- src/utils.cpp | 64 +++++++++++++++++++++++++++++++++++++++++++++++++++ src/utils.h | 15 ++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/utils.cpp b/src/utils.cpp index 42dc1cd6..cf25efc6 100644 --- a/src/utils.cpp +++ b/src/utils.cpp @@ -22,6 +22,7 @@ #include #ifndef _MSC_VER #include +extern char ** environ; #endif #ifdef __linux__ #include @@ -29,6 +30,69 @@ namespace pcm { + +bool startsWithPCM(const StringType& varName) { + const StringType prefix = PCM_STRING("PCM_"); + return varName.compare(0, prefix.size(), prefix) == 0; +} + +bool isInKeepList(const StringType& varName, const std::vector& keepList) { + for (const auto& keepVar : keepList) { + if (varName == keepVar) { + return true; + } + } + return false; +} + +#if defined(_MSC_VER) + +void eraseEnvironmentVariables(const std::vector& keepList) { + // Get a snapshot of the current environment block + LPWCH envBlock = GetEnvironmentStrings(); + if (!envBlock) { + std::cerr << "Error getting environment strings." << std::endl; + return; + } + + // Iterate over the environment block + for (LPWCH var = envBlock; *var != 0; var += std::wcslen(var) + 1) { + std::wstring varName(var); + size_t pos = varName.find('='); + if (pos != std::string::npos) { + varName = varName.substr(0, pos); + if (!startsWithPCM(varName) && !isInKeepList(varName, keepList)) { + SetEnvironmentVariable(varName.c_str(), NULL); + } + } + } + + // Free the environment block + FreeEnvironmentStrings(envBlock); +} +#else +void eraseEnvironmentVariables(const std::vector& keepList) { + std::vector varsToDelete; + + // Collect all the variables that need to be deleted + for (char **env = environ; *env != nullptr; ++env) { + std::string envEntry(*env); + size_t pos = envEntry.find('='); + if (pos != std::string::npos) { + std::string varName = envEntry.substr(0, pos); + if (!startsWithPCM(varName) && !isInKeepList(varName, keepList)) { + varsToDelete.push_back(varName); + } + } + } + + // Delete the collected variables + for (const auto& varName : varsToDelete) { + unsetenv(varName.c_str()); + } +} +#endif + void (*post_cleanup_callback)(void) = NULL; //! \brief handler of exit() call diff --git a/src/utils.h b/src/utils.h index b8c56de9..f4775d90 100644 --- a/src/utils.h +++ b/src/utils.h @@ -42,6 +42,14 @@ #endif namespace pcm { std::string safe_getenv(const char* env); +#ifdef _MSC_VER + typedef std::wstring StringType; + #define PCM_STRING(x) (L ## x) +#else + typedef std::string StringType; + #define PCM_STRING(x) (x) +#endif + void eraseEnvironmentVariables(const std::vector& keepList); } #ifdef _MSC_VER @@ -54,6 +62,13 @@ namespace pcm { int mainThrows(int argc, char * argv[]); \ int main(int argc, char * argv[]) \ { \ + try { \ + eraseEnvironmentVariables({PCM_STRING("POSIXLY_CORRECT")}); \ + } catch(const std::exception & e) \ + { \ + std::cerr << "PCM ERROR. Exception in eraseEnvironmentVariables: " << e.what() << "\n"; \ + return -1; \ + } \ PCM_SET_DLL_DIR \ if (pcm::safe_getenv("PCM_NO_MAIN_EXCEPTION_HANDLER") == std::string("1")) return mainThrows(argc, argv); \ try { \ From 8b8a216c6d3f389b4ef7784dcad4726db605f8cb Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 19 Jun 2024 08:12:58 +0200 Subject: [PATCH 18/25] add bhs-power-mode.sh script Change-Id: Ica3870a7bd97ca83957db9d9ef069bd85f72b7e0 --- scripts/bhs-power-mode.sh | 132 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 132 insertions(+) create mode 100644 scripts/bhs-power-mode.sh diff --git a/scripts/bhs-power-mode.sh b/scripts/bhs-power-mode.sh new file mode 100644 index 00000000..c842d1f8 --- /dev/null +++ b/scripts/bhs-power-mode.sh @@ -0,0 +1,132 @@ +#!/bin/bash + +echo "Intel(r) Performance Counter Monitor" +echo "Birch Stream Power Mode Utility" +echo "" + +echo " Options:" +echo " --default : set default power mode" +echo " --latency-optimized-mode : set latency optimized mode" +echo + +# Run the pcm-tpmi command to determine I/O and compute dies +output=$(pcm-tpmi 2 0x10 -d -b 26:26) + +# Parse the output to build lists of I/O and compute dies +io_dies=() +compute_dies=() +declare -A die_types +while read -r line; do + if [[ $line == *"instance 0"* ]]; then + die=$(echo "$line" | grep -oP 'entry \K[0-9]+') + if [[ $line == *"value 1"* ]]; then + die_types[$die]="IO" + io_dies+=("$die") + elif [[ $line == *"value 0"* ]]; then + die_types[$die]="Compute" + compute_dies+=("$die") + fi + fi +done <<< "$output" + +if [ "$1" == "--default" ]; then + echo "Setting default mode..." + + for die in "${io_dies[@]}"; do + # EFFICIENCY_LATENCY_CTRL_RATIO (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 28:22 -w 8 + + #EFFICIENCY_LATENCY_CTRL_LOW_THRESHOLD (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 38:32 -w 13 + + #EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD(Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 46:40 -w 120 + + #EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD_ENABLE (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 39:39 -w 1 + done + + for die in "${compute_dies[@]}"; do + # EFFICIENCY_LATENCY_CTRL_RATIO (Uncore Compute) + pcm-tpmi 2 0x18 -d -e $die -b 28:22 -w 12 + done + +fi + +if [ "$1" == "--latency-optimized-mode" ]; then + echo "Setting latency optimized mode..." + + for die in "${io_dies[@]}"; do + # EFFICIENCY_LATENCY_CTRL_RATIO (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 28:22 -w 0 + + #EFFICIENCY_LATENCY_CTRL_LOW_THRESHOLD (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 38:32 -w 0 + + #EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD(Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 46:40 -w 0 + + #EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD_ENABLE (Uncore IO) + pcm-tpmi 2 0x18 -d -e $die -b 39:39 -w 1 + done + + for die in "${compute_dies[@]}"; do + # EFFICIENCY_LATENCY_CTRL_RATIO (Uncore Compute) + pcm-tpmi 2 0x18 -d -e $die -b 28:22 -w 0 + done + +fi + + +echo "Dumping TPMI Power control register states..." +echo "" + +# Function to extract and calculate metrics from the value +extract_and_print_metrics() { + local value=$1 + local socket_id=$2 + local die=$3 + local die_type=${die_types[$die]} + + # Extract bits and calculate metrics + local min_ratio=$(( (value >> 15) & 0x7F )) + local max_ratio=$(( (value >> 8) & 0x7F )) + local eff_latency_ctrl_ratio=$(( (value >> 22) & 0x7F )) + local eff_latency_ctrl_low_threshold=$(( (value >> 32) & 0x7F )) + local eff_latency_ctrl_high_threshold=$(( (value >> 40) & 0x7F )) + local eff_latency_ctrl_high_threshold_enable=$(( (value >> 39) & 0x1 )) + + # Convert to MHz or percentage + min_ratio=$(( min_ratio * 100 )) + max_ratio=$(( max_ratio * 100 )) + eff_latency_ctrl_ratio=$(( eff_latency_ctrl_ratio * 100 )) + eff_latency_ctrl_low_threshold=$(( (eff_latency_ctrl_low_threshold * 100) / 127 )) + eff_latency_ctrl_high_threshold=$(( (eff_latency_ctrl_high_threshold * 100) / 127 )) + + # Print metrics + echo "Socket ID: $socket_id, Die: $die, Type: $die_type" + echo "MIN_RATIO: $min_ratio MHz" + echo "MAX_RATIO: $max_ratio MHz" + echo "EFFICIENCY_LATENCY_CTRL_RATIO: $eff_latency_ctrl_ratio MHz" + if [ $die_type == "IO" ] ; then + echo "EFFICIENCY_LATENCY_CTRL_LOW_THRESHOLD: $eff_latency_ctrl_low_threshold%" + echo "EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD: $eff_latency_ctrl_high_threshold%" + echo "EFFICIENCY_LATENCY_CTRL_HIGH_THRESHOLD_ENABLE: $eff_latency_ctrl_high_threshold_enable" + fi + echo +} + +# Iterate over all dies and run pcm-tpmi for each to get the metrics +for die in "${!die_types[@]}"; do + output=$(pcm-tpmi 2 0x18 -d -e "$die") + + # Parse the output and extract metrics for each socket + while read -r line; do + if [[ $line == *"Read value"* ]]; then + value=$(echo "$line" | grep -oP 'value \K[0-9]+') + socket_id=$(echo "$line" | grep -oP 'instance \K[0-9]+') + extract_and_print_metrics "$value" "$socket_id" "$die" + fi + done <<< "$output" +done + From 7a98558700756010b06b632d12febf960bf8d413 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Wed, 19 Jun 2024 08:29:35 +0200 Subject: [PATCH 19/25] add PCM_ENFORCE_MBM option (env variable) addresses https://github.com/intel/pcm/issues/761 Change-Id: Ib3e231e733800533b9d4998a86a745c9432c738e --- doc/ENVVAR_README.md | 2 ++ src/cpucounters.cpp | 15 +++++++++++++-- tests/fuzz.sh | 2 ++ tests/test.sh | 1 + 4 files changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/ENVVAR_README.md b/doc/ENVVAR_README.md index 58acfa7a..af6791fe 100644 --- a/doc/ENVVAR_README.md +++ b/doc/ENVVAR_README.md @@ -11,3 +11,5 @@ `PCM_KEEP_NMI_WATCHDOG=1` : don't disable NMI watchdog (reducing the core metrics set) `PCM_NO_MAIN_EXCEPTION_HANDLER=1` : don't catch exceptions in the main function of pcm tools (a debugging option) + +`PCM_ENFORCE_MBM=1` : force-enable Memory Bandwidth Monitoring (MBM) metrics (LocalMemoryBW = LMB) and (RemoteMemoryBW = RMB) on processors with RDT/MBM errata diff --git a/src/cpucounters.cpp b/src/cpucounters.cpp index d17e7d1a..cbae950f 100644 --- a/src/cpucounters.cpp +++ b/src/cpucounters.cpp @@ -550,9 +550,20 @@ bool PCM::L3CacheOccupancyMetricAvailable() const return (cpuinfo.reg.edx & 1)?true:false; } +bool isMBMEnforced() +{ + static int flag = -1; + if (flag < 0) + { + // flag not yet initialized + flag = pcm::safe_getenv("PCM_ENFORCE_MBM") == std::string("1") ? 1 : 0; + } + return flag > 0; +} + bool PCM::CoreLocalMemoryBWMetricAvailable() const { - if (cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata + if (isMBMEnforced() == false && cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata PCM_CPUID_INFO cpuinfo; if (!(QOSMetricAvailable() && L3QOSMetricAvailable())) return false; @@ -562,7 +573,7 @@ bool PCM::CoreLocalMemoryBWMetricAvailable() const bool PCM::CoreRemoteMemoryBWMetricAvailable() const { - if (cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata + if (isMBMEnforced() == false && cpu_model == SKX && cpu_stepping < 5) return false; // SKZ4 errata PCM_CPUID_INFO cpuinfo; if (!(QOSMetricAvailable() && L3QOSMetricAvailable())) return false; diff --git a/tests/fuzz.sh b/tests/fuzz.sh index f64df9f2..2a94779d 100644 --- a/tests/fuzz.sh +++ b/tests/fuzz.sh @@ -1,4 +1,6 @@ +export PCM_ENFORCE_MBM="1" + CC=`which clang` CXX=`which clang++` cmake .. -DCMAKE_BUILD_TYPE=Debug -DFUZZ=1 && mkdir -p corpus && make urltest-fuzz pcm-fuzz pcm-memory-fuzz -j && LLVM_PROFILE_FILE="urltest.profraw" bin/tests/urltest-fuzz -max_total_time=1000 corpus > /dev/null && diff --git a/tests/test.sh b/tests/test.sh index e5124075..ba1c8935 100755 --- a/tests/test.sh +++ b/tests/test.sh @@ -1,5 +1,6 @@ modprobe msr +export PCM_ENFORCE_MBM="1" export BIN_DIR="build/bin" pushd $BIN_DIR From 7be1b3bbe8ace8134c38639f27e8534ba1d57bea Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Wed, 19 Jun 2024 14:42:59 +0200 Subject: [PATCH 20/25] catch exceptions inside startHTTPSServer and startHTTPServer also print the cause of the exception to simplify debugging Change-Id: Ib1125056abfff77de978da53027e26e689861d69 --- src/pcm-sensor-server.cpp | 34 +++++++++++++++++++++++----------- 1 file changed, 23 insertions(+), 11 deletions(-) diff --git a/src/pcm-sensor-server.cpp b/src/pcm-sensor-server.cpp index 26eccff6..e422a1ca 100644 --- a/src/pcm-sensor-server.cpp +++ b/src/pcm-sensor-server.cpp @@ -3181,23 +3181,35 @@ void my_get_callback( HTTPServer* hs, HTTPRequest const & req, HTTPResponse & re int startHTTPServer( unsigned short port ) { HTTPServer server( "", port ); - // HEAD is GET without body, we will remove the body in execute() - server.registerCallback( HTTPRequestMethod::GET, my_get_callback ); - server.registerCallback( HTTPRequestMethod::HEAD, my_get_callback ); - server.run(); + try { + // HEAD is GET without body, we will remove the body in execute() + server.registerCallback( HTTPRequestMethod::GET, my_get_callback ); + server.registerCallback( HTTPRequestMethod::HEAD, my_get_callback ); + server.run(); + } catch (std::exception & e) + { + std::cerr << "Exception caught: " << e.what() << "\n"; + return -1; + } return 0; } #if defined (USE_SSL) int startHTTPSServer( unsigned short port, std::string const & cFile, std::string const & pkFile) { HTTPSServer server( "", port ); - server.setPrivateKeyFile ( pkFile ); - server.setCertificateFile( cFile ); - server.initialiseSSL(); - // HEAD is GET without body, we will remove the body in execute() - server.registerCallback( HTTPRequestMethod::GET, my_get_callback ); - server.registerCallback( HTTPRequestMethod::HEAD, my_get_callback ); - server.run(); + try { + server.setPrivateKeyFile ( pkFile ); + server.setCertificateFile( cFile ); + server.initialiseSSL(); + // HEAD is GET without body, we will remove the body in execute() + server.registerCallback( HTTPRequestMethod::GET, my_get_callback ); + server.registerCallback( HTTPRequestMethod::HEAD, my_get_callback ); + server.run(); + } catch (std::exception & e) + { + std::cerr << "Exception caught: " << e.what() << "\n"; + return -1; + } return 0; } #endif From 0a176eea54c977eac45a44a5ee19e09b99896808 Mon Sep 17 00:00:00 2001 From: Otto Bruggeman Date: Wed, 19 Jun 2024 17:41:11 +0200 Subject: [PATCH 21/25] Fix ssl accept issues (#629) * Fix ssl accept issues Rearranged the code to catch the exceptions locally in order to be able to continue Added proper code paths for when the ssl error is 1 and ability to recover Hard errors now close the connection instead of crashing --- src/pcm-sensor-server.cpp | 40 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 37 insertions(+), 3 deletions(-) diff --git a/src/pcm-sensor-server.cpp b/src/pcm-sensor-server.cpp index e422a1ca..bb70cfd8 100644 --- a/src/pcm-sensor-server.cpp +++ b/src/pcm-sensor-server.cpp @@ -2915,9 +2915,43 @@ void HTTPSServer::run() { SSL* ssl = SSL_new( sslCTX_ ); SSL_set_fd( ssl, clientSocketFD ); - // Check if the SSL handshake worked - if ( SSL_accept( ssl ) <= 0 ) - throw std::runtime_error( "SSL handshake failure" ); + try { + while (1) { + bool leaveLoop = false; + // Check if the SSL handshake worked + int accept = SSL_accept( ssl ); + switch (accept) { + case 0: + throw std::runtime_error( "accept == 0 is a hard error." ); + case -1: + { + int errorCode = SSL_get_error( ssl, accept ); + switch ( errorCode ) { + case SSL_ERROR_WANT_READ: + case SSL_ERROR_WANT_WRITE: + // All good, just try again + leaveLoop = false; // Unnecessary but for easier understanding + break; + case SSL_ERROR_ZERO_RETURN: + case SSL_ERROR_SYSCALL: + case SSL_ERROR_SSL: + default: + throw std::runtime_error( "Error not read or write is a hard error." ); + } + } + break; + default: + // all good, continue + leaveLoop = true; + } + if ( leaveLoop ) + break; + } + } catch( std::exception& e ) { + DBG( 3, "SSL Accept: error accepting incoming connection, closing the FD and continuing: ", e.what() ); + ::close( clientSocketFD ); + continue; + } // Client connected, let's determine the client ip as string. char ipbuf[INET_ADDRSTRLEN]; From 0b64d5ede8950761927ad97190d821f3c26b6da5 Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Wed, 19 Jun 2024 17:39:13 +0200 Subject: [PATCH 22/25] enable compilation with SSL Change-Id: Icefd89e6dbf88aff3e5fe9137c74b0680edf8fc5 --- src/CMakeLists.txt | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 0af9ad1c..cf3745de 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -140,6 +140,15 @@ foreach(PROJECT_NAME ${PROJECT_NAMES}) endif(${PROJECT_NAME} STREQUAL pcm-raw) if(${PROJECT_NAME} STREQUAL pcm-sensor-server) + if(NO_SSL) + message(STATUS "SSL is disabled") + else() + message(STATUS "Compiling with SSL support, requires libssl-dev or libopenssl-devel or libopenssl-dev package installed") + message(STATUS "To disable SSL support, use -DNO_SSL=1 option") + find_package(OpenSSL REQUIRED) + target_compile_options(${PROJECT_NAME} PRIVATE "-DUSE_SSL") + set(LIBS ${LIBS} OpenSSL::SSL OpenSSL::Crypto) + endif() file(READ pcm-sensor-server.service.in SENSOR_SERVICE_IN) string(REPLACE "@@CMAKE_INSTALL_SBINDIR@@" "${CMAKE_INSTALL_PREFIX}/${CMAKE_INSTALL_SBINDIR}" SENSOR_SERVICE "${SENSOR_SERVICE_IN}") file(WRITE "${CMAKE_BINARY_DIR}/pcm-sensor-server.service" "${SENSOR_SERVICE}") From c11a46276b0432e481452c33d40ea1e91077df9a Mon Sep 17 00:00:00 2001 From: Roman Dementiev Date: Wed, 19 Jun 2024 18:56:41 +0200 Subject: [PATCH 23/25] reduce warnings Change-Id: Ida2c2931d005409adfbcf4d2a282a0996171d5ed --- src/pcm-sensor-server.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/pcm-sensor-server.cpp b/src/pcm-sensor-server.cpp index bb70cfd8..57d7e42f 100644 --- a/src/pcm-sensor-server.cpp +++ b/src/pcm-sensor-server.cpp @@ -849,6 +849,8 @@ void PrometheusPrinter::iterateVectorAndCallAccept(Vector const& v) { template > class basic_socketbuf : public std::basic_streambuf { public: + basic_socketbuf(const basic_socketbuf&) = delete; + basic_socketbuf & operator = (const basic_socketbuf&) = delete; using Base = std::basic_streambuf; using char_type = typename Base::char_type; using int_type = typename Base::int_type; @@ -1060,6 +1062,8 @@ class basic_socketstream : public std::basic_iostream { using traits_type = typename Base::traits_type; public: + basic_socketstream(const basic_socketstream &) = delete; + basic_socketstream & operator = (const basic_socketstream &) = delete; basic_socketstream() : stream_type( &socketBuffer_ ) {} #if defined (USE_SSL) basic_socketstream( int socketFD, SSL* ssl ) : stream_type( &socketBuffer_ ) { @@ -2856,6 +2860,7 @@ class HTTPSServer : public HTTPServer { HTTPSServer() : HTTPServer( "", 443 ) {} HTTPSServer( std::string const & ip, uint16_t port ) : HTTPServer( ip, port ), sslCTX_( nullptr ) {} HTTPSServer( HTTPSServer const & ) = delete; + HTTPSServer & operator = ( HTTPSServer const & ) = delete; virtual ~HTTPSServer() = default; public: From b7ecb55c6838426c31f274748bc16b588416ff3a Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Thu, 20 Jun 2024 07:45:40 +0200 Subject: [PATCH 24/25] add openssl-devel Change-Id: Ie2881bafd6b0dbcd17be977659ac484b1e1d4ac0 --- Dockerfile | 2 +- src/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 2480e704..4b0350c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM fedora:40@sha256:4e007f288dce23966216be81ef62ba05d139b9338f327c1d1c73b7167dd47312 as builder -RUN dnf -y install gcc-c++ git findutils make cmake +RUN dnf -y install gcc-c++ git findutils make cmake openssl openssl-devel COPY . /tmp/pcm RUN cd /tmp/pcm && mkdir build && cd build && cmake .. && make -j diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index cf3745de..6e1a82cc 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -143,7 +143,7 @@ foreach(PROJECT_NAME ${PROJECT_NAMES}) if(NO_SSL) message(STATUS "SSL is disabled") else() - message(STATUS "Compiling with SSL support, requires libssl-dev or libopenssl-devel or libopenssl-dev package installed") + message(STATUS "Compiling with SSL support, requires libssl-dev or openssl-devel or libopenssl-devel or libopenssl-dev package installed") message(STATUS "To disable SSL support, use -DNO_SSL=1 option") find_package(OpenSSL REQUIRED) target_compile_options(${PROJECT_NAME} PRIVATE "-DUSE_SSL") From 2c65bfd780cd3711964f3a21e62ba568fc04d1b0 Mon Sep 17 00:00:00 2001 From: "Dementiev, Roman" Date: Thu, 20 Jun 2024 15:47:27 +0200 Subject: [PATCH 25/25] fix linux make GH action Change-Id: Id892b7aef08514b885c52f1303a969396ec67a3d --- .github/workflows/linux_make.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux_make.yml b/.github/workflows/linux_make.yml index 83400193..17089091 100644 --- a/.github/workflows/linux_make.yml +++ b/.github/workflows/linux_make.yml @@ -57,7 +57,7 @@ jobs: - name: install dependencies run: | apt update - apt -y --no-install-recommends install build-essential cmake libsimdjson-dev + apt -y --no-install-recommends install build-essential cmake libsimdjson-dev libssl-dev - name: cmake run: | rm -rf ${{ github.workspace }}/build