Rename from nvprof focused connector to nvtx focused connector

CMakeLists.txt

@@ -95,7 +95,7 @@ endif()
 include(cmake/configure_variorum.cmake)
 
 set(KOKKOSTOOLS_HAS_CALIPER ${KokkosTools_ENABLE_CALIPER})
-set(KOKKOSTOOLS_HAS_NVPROF ${Kokkos_ENABLE_CUDA}) # we assume that enabling CUDA for Kokkos program means nvprof should be available
+set(KOKKOSTOOLS_HAS_NVTX ${Kokkos_ENABLE_CUDA}) # we assume that enabling CUDA for Kokkos program means nvtx should be available
 
 if(DEFINED ENV{VTUNE_HOME})
   set(VTune_ROOT $ENV{VTUNE_HOME})
@@ -180,7 +180,7 @@ endif()
 # GPU profilers
 if(Kokkos_ENABLE_CUDA)
   add_subdirectory(profiling/nvprof-connector)
-  add_subdirectory(profiling/nvprof-focused-connector)
+  add_subdirectory(profiling/nvtx-focused-connector)
 endif()
 if(Kokkos_ENABLE_HIP)
   add_subdirectory(profiling/roctx-connector)

build-all.sh

@@ -10,7 +10,7 @@ make -f $ROOT_DIR/profiling/memory-hwm/Makefile
 make -f $ROOT_DIR/profiling/memory-hwm-mpi/Makefile
 make -f $ROOT_DIR/profiling/memory-usage/Makefile
 make -f $ROOT_DIR/profiling/nvprof-connector/Makefile
-make -f $ROOT_DIR/profiling/nvprof-focused-connector/Makefile
+make -f $ROOT_DIR/profiling/nvtx-focused-connector/Makefile
 make -f $ROOT_DIR/profiling/papi-connector/Makefile
 make -f $ROOT_DIR/profiling/simple-kernel-timer-json/Makefile
 make -f $ROOT_DIR/profiling/simple-kernel-timer/Makefile

common/kernel-filter/CMakeLists.txt

@@ -1 +1 @@
-add_library(kp_kernel_filter ${KOKKOSTOOLS_LIBRARY_MODE} kp_kernel_filter.cpp)
+kp_add_library(kp_kernel_filter ${KOKKOSTOOLS_LIBRARY_MODE} kp_kernel_filter.cpp)

common/kernel-filter/kp_kernel_filter.cpp

@@ -101,22 +101,17 @@ extern "C" void kokkosp_init_library(const int loadSeq,
     printf("============================================================\n");
     printf("KokkosP: Filter File: %s\n", kernelFilterPath);
     printf("============================================================\n");
-
     FILE* kernelFilterFile = fopen(kernelFilterPath, "rt");
-
     if (NULL == kernelFilterFile) {
       fprintf(stderr, "Unable to open kernel filter: %s\n", kernelFilterPath);
       exit(-1);
     } else {
       char* lineBuffer = (char*)malloc(sizeof(char) * 65536);
-
       while (kokkospReadLine(kernelFilterFile, lineBuffer)) {
         printf("KokkosP: Filter [%s]\n", lineBuffer);
-
         std::regex nextRegEx(lineBuffer, std::regex::optimize);
         kernelNames.push_back(nextRegEx);
       }
-
       free(lineBuffer);
     }
 
@@ -126,63 +121,73 @@ extern "C" void kokkosp_init_library(const int loadSeq,
            (filterKernels ? "enabled" : "disabled"));
 
     if (filterKernels) {
-      char* profileLibrary = getenv("KOKKOS_PROFILE_LIBRARY");
-      char* envBuffer =
-          (char*)malloc(sizeof(char) * (strlen(profileLibrary) + 1));
-      strcpy(envBuffer, profileLibrary);
-
-      char* nextLibrary = strtok(envBuffer, ";");
+      char* profileLibrary = getenv("KOKKOS_TOOLS_LIBS");
+      // check deprecated environment variable.
+      if (NULL == profileLibrary) {
+        printf(
+            "Checking KOKKOS_PROFILE_LIBRARY. WARNING: This is a deprecated "
+            "variable. Please use KOKKOS_TOOLS_LIBS.\n");
+        profileLibrary = getenv("KOKKOS_PROFILE_LIBRARY");
+
+        if (NULL == profileLibrary) {
+          printf("KokkosP: No library to call in %s\n", profileLibrary);
+          exit(-1);
+        }
 
-      for (int i = 0; i < loadSeq; i++) {
-        nextLibrary = strtok(NULL, ";");
-      }
+        char* envBuffer =
+            (char*)malloc(sizeof(char) * (strlen(profileLibrary) + 1));
+        strcpy(envBuffer, profileLibrary);
 
-      nextLibrary = strtok(NULL, ";");
+        char* nextLibrary = strtok(envBuffer, ";");
 
-      if (NULL == nextLibrary) {
-        printf("KokkosP: No child library to call in %s\n", profileLibrary);
-      } else {
-        printf("KokkosP: Next library to call: %s\n", nextLibrary);
-        printf("KokkosP: Loading child library ..\n");
+        for (int i = 0; i < loadSeq; i++) {
+          nextLibrary = strtok(NULL, ";");
+        }
 
-        void* childLibrary = dlopen(nextLibrary, RTLD_NOW | RTLD_GLOBAL);
+        nextLibrary = strtok(NULL, ";");
 
-        if (NULL == childLibrary) {
-          fprintf(stderr, "KokkosP: Error: Unable to load: %s (Error=%s)\n",
-                  nextLibrary, dlerror());
+        if (NULL == nextLibrary) {
+          printf("KokkosP: No child library to call in %s\n", profileLibrary);
         } else {
-          beginForCallee =
-              (beginFunction)dlsym(childLibrary, "kokkosp_begin_parallel_for");
-          beginScanCallee =
-              (beginFunction)dlsym(childLibrary, "kokkosp_begin_parallel_scan");
-          beginReduceCallee = (beginFunction)dlsym(
-              childLibrary, "kokkosp_begin_parallel_reduce");
-
-          endScanCallee =
-              (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_scan");
-          endForCallee =
-              (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_for");
-          endReduceCallee =
-              (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_reduce");
-
-          initProfileLibrary =
-              (initFunction)dlsym(childLibrary, "kokkosp_init_library");
-          finalizeProfileLibrary =
-              (finalizeFunction)dlsym(childLibrary, "kokkosp_finalize_library");
-
-          if (NULL != initProfileLibrary) {
-            (*initProfileLibrary)(loadSeq + 1, interfaceVer, devInfoCount,
-                                  deviceInfo);
+          printf("KokkosP: Next library to call: %s\n", nextLibrary);
+          printf("KokkosP: Loading child library ..\n");
+
+          void* childLibrary = dlopen(nextLibrary, RTLD_NOW | RTLD_GLOBAL);
+
+          if (NULL == childLibrary) {
+            fprintf(stderr, "KokkosP: Error: Unable to load: %s (Error=%s)\n",
+                    nextLibrary, dlerror());
+          } else {
+            beginForCallee  = (beginFunction)dlsym(childLibrary,
+                                                  "kokkosp_begin_parallel_for");
+            beginScanCallee = (beginFunction)dlsym(
+                childLibrary, "kokkosp_begin_parallel_scan");
+            beginReduceCallee = (beginFunction)dlsym(
+                childLibrary, "kokkosp_begin_parallel_reduce");
+            endScanCallee =
+                (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_scan");
+            endForCallee =
+                (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_for");
+            endReduceCallee =
+                (endFunction)dlsym(childLibrary, "kokkosp_end_parallel_reduce");
+            initProfileLibrary =
+                (initFunction)dlsym(childLibrary, "kokkosp_init_library");
+            finalizeProfileLibrary = (finalizeFunction)dlsym(
+                childLibrary, "kokkosp_finalize_library");
+
+            if (NULL != initProfileLibrary) {
+              (*initProfileLibrary)(loadSeq + 1, interfaceVer, devInfoCount,
+                                    deviceInfo);
+            }
           }
+          free(envBuffer);
         }
       }
-
-      free(envBuffer);
     }
-
     printf("============================================================\n");
   }
-}
+
+}  // end kokkosp_init_library
 
 extern "C" void kokkosp_finalize_library() {
   if (NULL != finalizeProfileLibrary) {

common/kokkos-sampler/kp_sampler_skip.cpp

@@ -133,14 +133,11 @@ void kokkosp_init_library(const int loadSeq, const uint64_t interfaceVer,
   }
 
   free(envBuffer);
-
-  uniqID = 1;
-
+  uniqID                  = 1;
   const char* tool_sample = getenv("KOKKOS_TOOLS_SAMPLER_SKIP");
   if (NULL != tool_sample) {
     kernelSampleSkip = atoi(tool_sample) + 1;
   }
-
   if (tool_verbosity > 0) {
     printf("KokkosP: Sampling rate set to: %s\n", tool_sample);
   }
@@ -152,14 +149,11 @@ void kokkosp_finalize_library() {
 
 void kokkosp_begin_parallel_for(const char* name, const uint32_t devID,
                                 uint64_t* kID) {
-  *kID = uniqID++;
-
   if (((*kID) % kernelSampleSkip) == 0) {
     if (tool_verbosity > 0) {
       printf("KokkosP: sample %llu calling child-begin function...\n",
              (unsigned long long)(*kID));
     }
-
     if (NULL != beginForCallee) {
       (*beginForCallee)(name, devID, kID);
     }
@@ -181,8 +175,6 @@ void kokkosp_end_parallel_for(const uint64_t kID) {
 
 void kokkosp_begin_parallel_scan(const char* name, const uint32_t devID,
                                  uint64_t* kID) {
-  *kID = uniqID++;
-
   if (((*kID) % kernelSampleSkip) == 0) {
     if (tool_verbosity > 0) {
       printf("KokkosP: sample %llu calling child-begin function...\n",

profiling/all/CMakeLists.txt

@@ -19,4 +19,4 @@ endif()
 file(GLOB_RECURSE HEADER_FILES CONFIGURE_DEPENDS kp_all.hpp "${COMMON_HEADERS_PATH}/*.hpp")
 
 install(FILES ${HEADER_FILES} DESTINATION ${EXPORT_INCLUDE_DIR})
-install(TARGETS ${LIBNAME} EXPORT ${EXPORT_NAME})
+install(TARGETS ${LIBNAME} EXPORT ${EXPORT_NAME})

profiling/all/kp_all.cpp

@@ -50,7 +50,7 @@ KOKKOSTOOLS_EXTERN_EVENT_SET(VariorumConnector)
 #endif
 #ifdef KOKKOSTOOLS_HAS_NVPROF
 KOKKOSTOOLS_EXTERN_EVENT_SET(NVProfConnector)
-KOKKOSTOOLS_EXTERN_EVENT_SET(NVProfFocusedConnector)
+KOKKOSTOOLS_EXTERN_EVENT_SET(NVTXFocusedConnector)
 #endif
 #ifdef KOKKOSTOOLS_HAS_CALIPER
 namespace cali {
@@ -91,9 +91,8 @@ EventSet get_event_set(const char* profiler, const char* config_str) {
   handlers["caliper"] = cali::get_kokkos_event_set(config_str);
 #endif
 #ifdef KOKKOSTOOLS_HAS_NVPROF
-  handlers["nvprof-connector"] = NVProfConnector::get_event_set();
-  handlers["nvprof-focused-connector"] =
-      NVProfFocusedConnector::get_event_set();
+  handlers["nvprof-connector"]       = NVProfConnector::get_event_set();
+  handlers["nvtx-focused-connector"] = NVTXFocusedConnector::get_event_set();
 #endif
   auto e = handlers.find(profiler);
   if (e != handlers.end()) return e->second;

profiling/nvtx-focused-connector/CMakeLists.txt

@@ -0,0 +1,4 @@
+find_package(CUDAToolkit REQUIRED)
+kp_add_library(kp_nvtx_focused_connector kp_nvtx_focused_connector.cpp)
+
+target_link_libraries(kp_nvtx_focused_connector CUDA::nvToolsExt)

profiling/nvtx-focused-connector/Makefile

@@ -0,0 +1,18 @@
+CXX=g++
+CXXFLAGS=-O3 -std=c++11 -g -I$(CUDA_ROOT)/include -I./
+LDFLAGS=-L$(CUDA_ROOT)/lib64
+LIBS=
+SHARED_CXXFLAGS=-shared -fPIC
+
+all: kp_nvtx_focused_connector.so
+
+MAKEFILE_PATH := $(subst Makefile,,$(abspath $(lastword $(MAKEFILE_LIST))))
+
+CXXFLAGS+=-I${MAKEFILE_PATH} -I${MAKEFILE_PATH}../../common/makefile-only -I${MAKEFILE_PATH}../all
+
+kp_nvtx_focused_connector.so: ${MAKEFILE_PATH}kp_nvtx_focused_connector.cpp
+	$(CXX) $(SHARED_CXXFLAGS) $(CXXFLAGS) $(LDFLAGS) \
+		-o $@ ${MAKEFILE_PATH}kp_nvtx_focused_connector.cpp $(LIBS)
+
+clean:
+	rm *.so