Merge branch 'kokkos:develop' into use-probability-sampling

.github/workflows/build-with-kokkos.yml

@@ -71,9 +71,10 @@ jobs:
                   exit -1
           esac
 
-      - name: Install CMake, OpenMPI, PAPI and dtrace
+      - name: Install git, CMake, OpenMPI, PAPI and dtrace
         run: |
           apt --yes --no-install-recommends install \
+            git ca-certificates \
             cmake make \
             libopenmpi-dev \
             systemtap-sdt-dev \

CMakeLists.txt

@@ -272,6 +272,7 @@ endif()
 # Tests
 if(KokkosTools_ENABLE_TESTS)
   enable_testing()
+  include(cmake/BuildGTest.cmake)
   add_subdirectory(tests)
 endif()
 

build-all.sh

@@ -12,7 +12,6 @@ make -f $ROOT_DIR/profiling/memory-usage/Makefile
 make -f $ROOT_DIR/profiling/nvtx-connector/Makefile
 make -f $ROOT_DIR/profiling/nvtx-focused-connector/Makefile
 make -f $ROOT_DIR/profiling/papi-connector/Makefile
-make -f $ROOT_DIR/profiling/simple-kernel-timer-json/Makefile
 make -f $ROOT_DIR/profiling/simple-kernel-timer/Makefile
 make -f $ROOT_DIR/profiling/space-time-stack/Makefile
 make -f $ROOT_DIR/profiling/systemtap-connector/Makefile

cmake/BuildGTest.cmake

@@ -0,0 +1,35 @@
+# Look for Google Test and enable it as a target.
+#
+# The main targets that will be available are:
+#       * GTest::gtest
+#       * GTest::gmock
+#
+# References:
+#   * https://github.com/google/googletest
+#   * https://matgomes.com/integrate-google-test-into-cmake/
+#   * https://google.github.io/googletest/quickstart-cmake.html
+#   * https://jeremimucha.com/2021/04/cmake-fetchcontent/
+
+include(FetchContent)
+
+# Declare the Google Test dependency
+FetchContent_Declare(
+    googletest
+    GIT_REPOSITORY https://github.com/google/googletest.git
+    GIT_TAG        v1.14.0
+)
+
+# If not yet populated, add Google Test to the build with the following options:
+#       * disable installation of Google Test
+#       * enable GMock
+# Note that we could have used FetchContent_MakeAvailable instead, but it would then
+# use the default configuration that would install Google Test.
+FetchContent_GetProperties(googletest)
+if (NOT googletest_POPULATED)
+    FetchContent_Populate(googletest)
+
+    set(BUILD_GMOCK   ON)
+    set(INSTALL_GTEST OFF)
+
+    add_subdirectory(${googletest_SOURCE_DIR} ${googletest_BINARY_DIR} EXCLUDE_FROM_ALL)
+endif()

example/CMakeLists.txt

@@ -21,7 +21,6 @@ endmacro()
 #       and exported output in expected format, fail the test otherwise.
 if(NOT WIN32)
   add_kp_test(kernel_timer "kernel-timer")
-  add_kp_test(kernel_timer_json "kernel-timer-json")
   add_kp_test(memory_events "memory-events")
   add_kp_test(memory_usage "memory-usage")
   add_kp_test(chrome_tracing "chrome-tracing")

profiling/all/kp_all.cpp

@@ -30,7 +30,6 @@
 
 #ifndef WIN32
 KOKKOSTOOLS_EXTERN_EVENT_SET(KernelTimer)
-KOKKOSTOOLS_EXTERN_EVENT_SET(KernelTimerJSON)
 KOKKOSTOOLS_EXTERN_EVENT_SET(MemoryEvents)
 KOKKOSTOOLS_EXTERN_EVENT_SET(MemoryUsage)
 KOKKOSTOOLS_EXTERN_EVENT_SET(HighwaterMark)
@@ -69,10 +68,9 @@ namespace KokkosTools {
 EventSet get_event_set(const char* profiler, const char* config_str) {
   std::map<std::string, EventSet> handlers;
 #ifndef WIN32
-  handlers["kernel-timer"]      = KernelTimer::get_event_set();
-  handlers["kernel-timer-json"] = KernelTimerJSON::get_event_set();
-  handlers["memory-events"]     = MemoryEvents::get_event_set();
-  handlers["memory-usage"]      = MemoryUsage::get_event_set();
+  handlers["kernel-timer"]  = KernelTimer::get_event_set();
+  handlers["memory-events"] = MemoryEvents::get_event_set();
+  handlers["memory-usage"]  = MemoryUsage::get_event_set();
 #if USE_MPI
   handlers["highwater-mark-mpi"] = HighwaterMarkMPI::get_event_set();
 #endif

profiling/simple-kernel-timer/CMakeLists.txt

@@ -7,10 +7,6 @@ if(NOT MSVC)
   set_property(TARGET kp_kernel_shared PROPERTY POSITION_INDEPENDENT_CODE ON)
 endif()
 
-# Add JSON kernel-timer
-kp_add_library(kp_kernel_timer_json kp_kernel_timer_json.cpp)
-target_link_libraries(kp_kernel_timer_json PRIVATE kp_kernel_shared)
-
 # Add binary kernel-timer
 kp_add_library(kp_kernel_timer kp_kernel_timer.cpp)
 target_link_libraries(kp_kernel_timer PRIVATE kp_kernel_shared)

profiling/simple-kernel-timer/Makefile

@@ -1,12 +1,12 @@
 CXX=g++
-CXXFLAGS=-O3 -std=c++11 -g
+CXXFLAGS=-O3 -std=c++17 -g
 SHARED_CXXFLAGS=-shared -fPIC
 
 all: kp_kernel_timer.so kp_reader kp_json_writer
 
 MAKEFILE_PATH := $(subst Makefile,,$(abspath $(lastword $(MAKEFILE_LIST))))
 
-CXXFLAGS+=-I${MAKEFILE_PATH} -I${MAKEFILE_PATH}/../../common/makefile-only -I${MAKEFILE_PATH}../all
+CXXFLAGS+=-I${MAKEFILE_PATH} -I${MAKEFILE_PATH}/../../common/makefile-only -I${MAKEFILE_PATH}../all -I${MAKEFILE_PATH}../../common
 
 kp_reader: ${MAKEFILE_PATH}kp_reader.cpp kp_kernel_timer.so
 	$(CXX) $(CXXFLAGS) -o kp_reader ${MAKEFILE_PATH}kp_reader.cpp ${MAKEFILE_PATH}kp_shared.cpp 

profiling/simple-kernel-timer/kp_kernel_timer.cpp

@@ -15,6 +15,7 @@
 //@HEADER
 
 #include <vector>
+#include <algorithm>
 #include <string>
 #include <iostream>
 #include <unistd.h>
@@ -25,6 +26,10 @@
 namespace KokkosTools {
 namespace KernelTimer {
 
+bool is_region(KernelPerformanceInfo const& kp) {
+  return kp.getKernelType() == REGION;
+}
+
 void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
                           const uint32_t /*devInfoCount*/,
                           Kokkos_Profiling_KokkosPDeviceInfo* /*deviceInfo*/) {
@@ -52,23 +57,90 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
 void kokkosp_finalize_library() {
   double finishTime = seconds();
 
+  const char* kokkos_tools_timer_json_raw = getenv("KOKKOS_TOOLS_TIMER_JSON");
+  const bool kokkos_tools_timer_json =
+      kokkos_tools_timer_json_raw == NULL
+          ? false
+          : strcmp(kokkos_tools_timer_json_raw, "1") == 0 ||
+                strcmp(kokkos_tools_timer_json_raw, "true") == 0 ||
+                strcmp(kokkos_tools_timer_json_raw, "True") == 0;
+
+  double kernelTimes = 0;
+
   char* hostname = (char*)malloc(sizeof(char) * 256);
   gethostname(hostname, 256);
 
   char* fileOutput = (char*)malloc(sizeof(char) * 256);
-  snprintf(fileOutput, 256, "%s-%d.dat", hostname, (int)getpid());
+  snprintf(fileOutput, 256, "%s-%d.%s", hostname, (int)getpid(),
+           kokkos_tools_timer_json ? "json" : "dat");
 
   free(hostname);
   FILE* output_data = fopen(fileOutput, "wb");
 
   const double totalExecuteTime = (finishTime - initTime);
-  fwrite(&totalExecuteTime, sizeof(totalExecuteTime), 1, output_data);
+  if (!kokkos_tools_timer_json) {
+    fwrite(&totalExecuteTime, sizeof(totalExecuteTime), 1, output_data);
 
-  std::vector<KernelPerformanceInfo*> kernelList;
+    for (auto kernel_itr = count_map.begin(); kernel_itr != count_map.end();
+         kernel_itr++) {
+      kernel_itr->second->writeToBinaryFile(output_data);
+    }
+  } else {
+    std::vector<KernelPerformanceInfo*> kernelList;
+
+    for (auto kernel_itr = count_map.begin(); kernel_itr != count_map.end();
+         kernel_itr++) {
+      kernelList.push_back(kernel_itr->second);
+      kernelTimes += kernel_itr->second->getTime();
+    }
+
+    std::sort(kernelList.begin(), kernelList.end(),
+              compareKernelPerformanceInfo);
+
+    fprintf(output_data, "{\n\"kokkos-kernel-data\" : {\n");
+    fprintf(output_data, "    \"total-app-time\"         : %10.3f,\n",
+            totalExecuteTime);
+    fprintf(output_data, "    \"total-kernel-times\"     : %10.3f,\n",
+            kernelTimes);
+    fprintf(output_data, "    \"total-non-kernel-times\" : %10.3f,\n",
+            (totalExecuteTime - kernelTimes));
+
+    const double percentKokkos = (kernelTimes / totalExecuteTime) * 100.0;
+    fprintf(output_data, "    \"percent-in-kernels\"     : %6.2f,\n",
+            percentKokkos);
+    fprintf(output_data, "    \"unique-kernel-calls\"    : %22llu,\n",
+            (unsigned long long)count_map.size());
+    fprintf(output_data, "\n");
+
+    fprintf(output_data, "    \"region-perf-info\"       : [\n");
+
+#define KERNEL_INFO_INDENT "       "
+
+    bool print_comma = false;
+    for (auto const& kernel : count_map) {
+      if (!is_region(*std::get<1>(kernel))) continue;
+      if (print_comma) fprintf(output_data, ",\n");
+      kernel.second->writeToJSONFile(output_data, KERNEL_INFO_INDENT);
+      print_comma = true;
+    }
+
+    fprintf(output_data, "\n");
+    fprintf(output_data, "    ],\n");
+
+    fprintf(output_data, "    \"kernel-perf-info\"       : [\n");
+
+    print_comma = false;
+    for (auto const& kernel : count_map) {
+      if (is_region(*std::get<1>(kernel))) continue;
+      if (print_comma) fprintf(output_data, ",\n");
+      kernel.second->writeToJSONFile(output_data, KERNEL_INFO_INDENT);
+      print_comma = true;
+    }
+
+    fprintf(output_data, "\n");
+    fprintf(output_data, "    ]\n");
 
-  for (auto kernel_itr = count_map.begin(); kernel_itr != count_map.end();
-       kernel_itr++) {
-    kernel_itr->second->writeToBinaryFile(output_data);
+    fprintf(output_data, "}\n}");
   }
 
   fclose(output_data);