ROCm environment fixes + workflow updates (#117)

* Improve dlopen of ROCm libraries + rocprofiler test - Use PROJECT_BINARY_DIR in tests - Added rocprofiler test * Revert OMNITRACE_FORCE_ROCPROFILER_INIT * omnitrace-avail --all test * Fix ROCP_METRICS for ROCm 5.2.0 * Fix ROCP_METRICS for ROCm 5.2.0 * Restrict containers workflow to AMDResearch/omnitrace * Bump version to 1.3.1 * Update cpack workflow - generate release draft - upload installers as release assets * Test rocprofiler w/o roctracer enabled * Fix formatting * verbose message
ROCm · Jul 27, 2022 · 7e31d9f · 7e31d9f
1 parent 4dd144a
commit 7e31d9f
Show file tree

Hide file tree

Showing 12 changed files with 253 additions and 65 deletions.
diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml
@@ -14,6 +14,8 @@ env:
 
 jobs:
   omnitrace-ci:
+    if: github.repository == 'AMDResearch/omnitrace'
+
     runs-on: ubuntu-20.04
 
     strategy:
@@ -57,6 +59,8 @@ jobs:
           popd
 
   omnitrace-release-ubuntu:
+    if: github.repository == 'AMDResearch/omnitrace'
+
     runs-on: ubuntu-20.04
 
     strategy:
@@ -98,6 +102,8 @@ jobs:
           popd
 
   omnitrace-release-opensuse:
+    if: github.repository == 'AMDResearch/omnitrace'
+
     runs-on: ubuntu-20.04
 
     strategy:

diff --git a/.github/workflows/cpack.yml b/.github/workflows/cpack.yml
@@ -13,6 +13,22 @@ env:
   GIT_DISCOVERY_ACROSS_FILESYSTEM: 1
 
 jobs:
+  release:
+    if: github.repository == 'AMDResearch/omnitrace'
+    runs-on: ubuntu-20.04
+    permissions:
+      contents: write
+      packages: write
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Generate Release
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/')
+        with:
+          draft: True
+          generate_release_notes: True
+
   ubuntu:
     runs-on: ubuntu-20.04
     strategy:
@@ -77,7 +93,6 @@ jobs:
       # before testing remove any artifacts of the build
       - name: Remove Build
         timeout-minutes: 10
-        continue-on-error: true
         run: |
           mv build-release/stgz/*.sh ./
           mv build-release/deb/*.deb ./
@@ -86,7 +101,6 @@ jobs:
 
       - name: Test STGZ Install
         timeout-minutes: 20
-        continue-on-error: true
         run: |
           set -v
           for i in omnitrace-*.sh
@@ -96,14 +110,21 @@ jobs:
 
       - name: Test DEB Install
         timeout-minutes: 20
-        continue-on-error: true
         run: |
           set -v
           for i in omnitrace_*.deb
           do
             ./docker/test-docker-release.sh --distro ubuntu --versions ${{ matrix.os }} --rocm-versions ${CI_ROCM_VERSION} -- --deb ${i}
           done
 
+      - name: Upload Release Assets
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/') && github.repository == 'AMDResearch/omnitrace'
+        with:
+          files: |
+            build-release/stgz/*.sh
+            build-release/deb/*.deb
+
   opensuse:
     runs-on: ubuntu-20.04
     strategy:
@@ -173,7 +194,6 @@ jobs:
       # before testing remove any artifacts of the build
       - name: Remove Build
         timeout-minutes: 10
-        continue-on-error: true
         run: |
           mv build-release/stgz/*.sh ./
           mv build-release/rpm/*.rpm ./
@@ -182,7 +202,6 @@ jobs:
 
       - name: Test STGZ Install
         timeout-minutes: 20
-        continue-on-error: true
         run: |
           set -v
           for i in omnitrace-*.sh
@@ -192,10 +211,17 @@ jobs:
 
       - name: Test RPM Install
         timeout-minutes: 20
-        continue-on-error: true
         run: |
           set -v
           for i in omnitrace-*.rpm
           do
             ./docker/test-docker-release.sh --distro opensuse --versions ${{ matrix.os }} --rocm-versions ${CI_ROCM_VERSION} -- --rpm ${i}
           done
+
+      - name: Upload Release Assets
+        uses: softprops/action-gh-release@v1
+        if: startsWith(github.ref, 'refs/tags/') && github.repository == 'AMDResearch/omnitrace'
+        with:
+          files: |
+            build-release/stgz/*.sh
+            build-release/rpm/*.rpm
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-1.3.0
+1.3.1
diff --git a/cmake/MacroUtilities.cmake b/cmake/MacroUtilities.cmake
@@ -124,7 +124,8 @@ function(OMNITRACE_STRIP_TARGET _TARGET)
                 --keep-symbol="omnitrace_pop_region" --keep-symbol="omnitrace_set_env"
                 --keep-symbol="omnitrace_set_mpi" --keep-symbol="omnitrace_user_*"
                 --keep-symbol="ompt_start_tool" --keep-symbol="kokkosp_*"
-                --keep-symbol="OnLoad" --keep-symbol="OnUnload" ${ARGN}
+                --keep-symbol="OnLoad" --keep-symbol="OnUnload"
+                --keep-symbol="OnLoadToolProp" --keep-symbol="OnUnloadTool" ${ARGN}
                 $<TARGET_FILE:${_TARGET}>
             WORKING_DIRECTORY ${CMAKE_BINARY_DIR}
             COMMENT "Stripping ${_TARGET}...")

diff --git a/source/bin/tests/CMakeLists.txt b/source/bin/tests/CMakeLists.txt
@@ -201,6 +201,13 @@ omnitrace_add_bin_test(
         ".*\\\[omnitrace-avail\\\] Usage:.*\\\[CATEGORIES\\\].*\\\[VIEW OPTIONS\\\].*\\\[COLUMN OPTIONS\\\].*\\\[WIDTH OPTIONS\\\].*\\\[OUTPUT OPTIONS\\\].*"
     )
 
+omnitrace_add_bin_test(
+    NAME omnitrace-avail-all
+    TARGET omnitrace-avail
+    ARGS --all
+    LABELS "omnitrace-avail"
+    TIMEOUT 45)
+
 omnitrace_add_bin_test(
     NAME omnitrace-avail-all-csv
     TARGET omnitrace-avail

diff --git a/source/lib/common/setup.hpp b/source/lib/common/setup.hpp
@@ -132,6 +132,11 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
 #endif
 
 #if defined(OMNITRACE_USE_ROCPROFILER) && OMNITRACE_USE_ROCPROFILER > 0
+#    if OMNITRACE_HIP_VERSION >= 50200
+#        define ROCPROFILER_METRICS_DIR "lib/rocprofiler"
+#    else
+#        define ROCPROFILER_METRICS_DIR "rocprofiler/lib"
+#    endif
     setenv("HSA_TOOLS_LIB", _omnilib.c_str(), 0);
     setenv("ROCP_TOOL_LIB", _omnilib.c_str(), 0);
     setenv("ROCPROFILER_LOG", "1", 0);
@@ -142,7 +147,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
         if(getenv(itr))
         {
             setenv("ROCP_METRICS",
-                   common::join('/', getenv(itr), "rocprofiler/lib/metrics.xml").c_str(),
+                   common::join('/', getenv(itr), ROCPROFILER_METRICS_DIR, "metrics.xml")
+                       .c_str(),
                    0);
             setenv("OMNITRACE_ROCPROFILER_LIBRARY",
                    common::join('/', getenv(itr), "rocprofiler/lib/librocprofiler64.so")
@@ -153,7 +159,8 @@ setup_environ(int _verbose, const std::string& _search_paths = {},
     }
     // default path
     setenv("ROCP_METRICS",
-           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, "rocprofiler/lib/metrics.xml")
+           common::join('/', OMNITRACE_DEFAULT_ROCM_PATH, ROCPROFILER_METRICS_DIR,
+                        "metrics.xml")
                .c_str(),
            0);
 #endif

diff --git a/source/lib/omnitrace/library/components/roctracer.cpp b/source/lib/omnitrace/library/components/roctracer.cpp
@@ -123,11 +123,18 @@ roctracer::setup()
 
     OMNITRACE_VERBOSE_F(1, "setting up roctracer...\n");
 
-    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64", "libamdhip64.so" };
+    dynamic_library _amdhip64{ "OMNITRACE_ROCTRACER_LIBAMDHIP64",
+                               find_library_path("libamdhip64.so",
+                                                 { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                                                 { OMNITRACE_DEFAULT_ROCM_PATH }) };
 
 #if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR < 4
-    dynamic_library _kfdwrapper{ "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
-                                 OMNITRACE_ROCTRACER_LIBKFDWRAPPER };
+    dynamic_library _kfdwrapper{
+        "OMNITRACE_ROCTRACER_LIBKFDWRAPPER",
+        find_library_path("libkfdwrapper64.so", { "OMNITRACE_ROCM_PATH", "ROCM_PATH" },
+                          { OMNITRACE_DEFAULT_ROCM_PATH },
+                          { "roctracer/lib", "roctracer/lib64", "lib", "lib64" })
+    };
 #endif
 
     ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));

diff --git a/source/lib/omnitrace/library/defines.hpp.in b/source/lib/omnitrace/library/defines.hpp.in
@@ -24,14 +24,6 @@
 
 #include "common/defines.h"
 
-// clang-format off
-#if defined(OMNITRACE_USE_ROCTRACER)
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "@roctracer_kfdwrapper_LIBRARY@"
-#else
-#    define OMNITRACE_ROCTRACER_LIBKFDWRAPPER "/opt/rocm/roctracer/lib/libkfdwrapper64.so"
-#endif
-// clang-format on
-
 #define TIMEMORY_USER_COMPONENT_ENUM                                                     \
     OMNITRACE_COMPONENT_idx, OMNITRACE_USER_REGION_idx, OMNITRACE_ROCTRACER_idx,         \
         OMNITRACE_ROCPROFILER_idx, OMNITRACE_SAMPLING_WALL_CLOCK_idx,                    \

diff --git a/source/lib/omnitrace/library/dynamic_library.cpp b/source/lib/omnitrace/library/dynamic_library.cpp
@@ -21,20 +21,90 @@
 // SOFTWARE.
 
 #include "library/dynamic_library.hpp"
+#include "common/defines.h"
+#include "library/common.hpp"
 #include "library/debug.hpp"
 #include "library/defines.hpp"
 
 #include <timemory/environment.hpp>
+#include <timemory/utility/delimit.hpp>
+#include <timemory/utility/filepath.hpp>
+
+#include <string>
+#include <utility>
 
 namespace omnitrace
 {
-dynamic_library::dynamic_library(const char* _env, const char* _fname, int _flags,
-                                 bool _store)
-: envname{ _env }
-, filename{ tim::get_env<std::string>(_env, _fname, _store) }
+std::string
+find_library_path(const std::string& _name, const std::vector<std::string>& _env_vars,
+                  const std::vector<std::string>& _hints,
+                  const std::vector<std::string>& _path_suffixes)
+{
+    if(_name.find('/') == 0) return _name;
+
+    auto _paths = std::vector<std::string>{};
+    for(const std::string& itr : _env_vars)
+    {
+        auto _env_val = get_env(itr, std::string{});
+        for(auto vitr : tim::delimit(_env_val, ":"))
+            if(!vitr.empty()) _paths.emplace_back(vitr);
+    }
+
+    for(const std::string& itr : _hints)
+    {
+        if(!itr.empty()) _paths.emplace_back(itr);
+    }
+
+    for(auto& itr : _paths)
+    {
+        auto _v = JOIN('/', itr, _name);
+        if(filepath::exists(_v)) return _v;
+        for(const auto& litr : _path_suffixes)
+        {
+            _v = JOIN('/', itr, litr, _name);
+            if(filepath::exists(_v)) return _v;
+        }
+    }
+
+    return _name;
+}
+
+dynamic_library::dynamic_library(std::string _env, std::string _fname, int _flags,
+                                 bool _open, bool _query_env, bool _store)
+: envname{ std::move(_env) }
+, filename{ std::move(_fname) }
 , flags{ _flags }
 {
-    open();
+    if(_query_env)
+    {
+        auto _env_val = get_env(envname, std::string{}, _store);
+        // if the environment variable is set to an absolute path that exists,
+        // override with value
+        if(!_env_val.empty())
+        {
+            if(_env_val.find('/') == 0 && filepath::exists(_env_val))
+            {
+                filename = _env_val;
+            }
+            else if(_env_val.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(1,
+                                    "Ignoring environment variable %s=\"%s\" because the "
+                                    "filepath does not exist. Using \"%s\" instead...\n",
+                                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+            else if(_env_val.find('/') != 0 && filename.find('/') == 0)
+            {
+                OMNITRACE_VERBOSE_F(
+                    1,
+                    "Ignoring environment variable %s=\"%s\" because the "
+                    "filepath is relative. Using absolute path \"%s\" instead...\n",
+                    envname.c_str(), _env_val.c_str(), filename.c_str())
+            }
+        }
+    }
+
+    if(_open) open();
 }
 
 dynamic_library::~dynamic_library() { close(); }
@@ -47,8 +117,8 @@ dynamic_library::open()
         handle = dlopen(filename.c_str(), flags);
         if(!handle)
         {
-            OMNITRACE_VERBOSE(2, "[dynamic_library][%s][%s] %s\n", envname.c_str(),
-                              filename.c_str(), dlerror());
+            OMNITRACE_VERBOSE(2, "[dynamic_library] Error opening %s=\"%s\" :: %s.\n",
+                              envname.c_str(), filename.c_str(), dlerror());
         }
         dlerror();  // Clear any existing error
     }
@@ -61,4 +131,10 @@ dynamic_library::close() const
     if(handle) return dlclose(handle);
     return -1;
 }
+
+bool
+dynamic_library::is_open() const
+{
+    return (handle != nullptr);
+}
 }  // namespace omnitrace