Merge branch 'main' into switch-perfetto-source-to-gh

ROCm · Sep 13, 2024 · cc96c90 · cc96c90
2 parents 053e8ee + 96d7b8f
commit cc96c90
Show file tree

Hide file tree

Showing 21 changed files with 188 additions and 78 deletions.
diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS
@@ -1,4 +1,4 @@
-* @jrmadsen
+* @jrmadsen @dgaliffiAMD
 
 # Documentation files
 docs/* @ROCm/rocm-documentation

diff --git a/.github/workflows/cpack.yml b/.github/workflows/cpack.yml
@@ -130,6 +130,12 @@ jobs:
           - os-distro: "opensuse"
             os-version: "15.5"
             rocm-version: "0.0"
+          - os-distro: "opensuse"
+            os-version: "15.5"
+            rocm-version: "5.6"
+          - os-distro: "opensuse"
+            os-version: "15.5"
+            rocm-version: "5.7"
           - os-distro: "opensuse"
             os-version: "15.5"
             rocm-version: "6.0"

diff --git a/.github/workflows/opensuse.yml b/.github/workflows/opensuse.yml
@@ -58,6 +58,11 @@ jobs:
         timeout_minutes: 25
         max_attempts: 5
         command: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Configure Env
@@ -87,6 +92,7 @@ jobs:
           -DOMNITRACE_USE_HIP=OFF
           -DOMNITRACE_USE_OMPT=OFF
           -DOMNITRACE_USE_PYTHON=ON
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"

diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml
@@ -68,6 +68,16 @@ jobs:
         env
 
     - name: Install Packages
+      shell: bash
+      run: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+
+    - name: Install ROCm Packages
       if: ${{ matrix.rocm-version > 0 }}
       timeout-minutes: 30
       shell: bash
@@ -109,6 +119,7 @@ jobs:
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON
           -DOMNITRACE_MAX_THREADS=64
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"

diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml
@@ -94,9 +94,10 @@ jobs:
           apt-get update &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -246,20 +247,12 @@ jobs:
         mpi-headers: ['OFF']
         build-jobs: ['3']
         ctest-exclude: ['-LE "mpi-example|transpose"']
-        perfetto-tools: ['OFF']
         include:
-          - compiler: 'g++'
-            rocm-version: '4.5'
-            mpi-headers: 'OFF'
-            build-jobs: '2'
-            ctest-exclude: '-LE "mpi-example|transpose"'
-            perfetto-tools: 'ON'
           - compiler: 'g++'
             rocm-version: 'debian'
             mpi-headers: 'ON'
             build-jobs: '2'
             ctest-exclude: '-LE transpose'
-            perfetto-tools: 'OFF'
 
     env:
       BUILD_TYPE: MinSizeRel
@@ -283,7 +276,10 @@ jobs:
           echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -345,7 +341,6 @@ jobs:
           -DOMNITRACE_USE_PYTHON=ON
           -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }}
           -DOMNITRACE_USE_SANITIZER=OFF
-          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${{ matrix.perfetto-tools }}
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=${{ matrix.mpi-headers }}
@@ -444,9 +439,10 @@ jobs:
           sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
           if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
           if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           sudo apt-get -y --purge autoremove &&
           sudo apt-get -y clean
@@ -596,9 +592,10 @@ jobs:
         command: |
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.24.1' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&

diff --git a/.github/workflows/ubuntu-jammy.yml b/.github/workflows/ubuntu-jammy.yml
@@ -110,8 +110,10 @@ jobs:
           apt-get install -y software-properties-common &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -395,6 +395,11 @@ install(
     DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/modulefiles/${PROJECT_NAME}
     COMPONENT setup)
 
+install(
+    FILES ${PROJECT_SOURCE_DIR}/LICENSE
+    DESTINATION ${CMAKE_INSTALL_DOCDIR}
+    COMPONENT setup)
+
 # ------------------------------------------------------------------------------#
 #
 # install

diff --git a/cmake/ConfigCPack.cmake b/cmake/ConfigCPack.cmake
@@ -214,6 +214,7 @@ set(CPACK_RPM_PACKAGE_PROVIDES
     "${CPACK_RPM_PACKAGE_PROVIDES}"
     CACHE STRING "RPM package provides" FORCE)
 
+set(CPACK_RPM_PACKAGE_LICENSE "MIT")
 set(CPACK_RPM_FILE_NAME "RPM-DEFAULT")
 set(CPACK_RPM_PACKAGE_RELEASE_DIST ON)
 set(CPACK_RPM_PACKAGE_AUTOREQPROV ON)

diff --git a/cmake/Packages.cmake b/cmake/Packages.cmake
@@ -483,7 +483,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/perfetto
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://github.com/google/perfetto.git
-    REPO_BRANCH v28.0
+    REPO_BRANCH v46.0
     TEST_FILE sdk/perfetto.cc)
 
 include(Perfetto)

diff --git a/cmake/Perfetto.cmake b/cmake/Perfetto.cmake
@@ -67,10 +67,10 @@ if(CMAKE_CXX_COMPILER_IS_CLANG)
 else()
     set(PERFETTO_IS_CLANG false)
     set(OMNITRACE_PERFETTO_C_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow"
         CACHE STRING "Perfetto C flags")
     set(OMNITRACE_PERFETTO_CXX_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow -Wno-mismatched-new-delete"
         CACHE STRING "Perfetto C++ flags")
 endif()
 

diff --git a/cmake/Templates/args.gn.in b/cmake/Templates/args.gn.in
@@ -8,19 +8,19 @@ is_clang = @PERFETTO_IS_CLANG@
 is_hermetic_clang = false
 
 enable_perfetto_benchmarks = false
-enable_perfetto_integration_tests = false
+enable_perfetto_integration_tests = true
 enable_perfetto_unittests = false
 enable_perfetto_fuzzers = false
 
 # enable_perfetto_stderr_crash_dump = false
 enable_perfetto_heapprofd = false
 enable_perfetto_tools = false
-enable_perfetto_trace_processor = false
-enable_perfetto_trace_processor_httpd = false
+enable_perfetto_trace_processor = true
+enable_perfetto_trace_processor_httpd = true
 enable_perfetto_trace_processor_json = false
 enable_perfetto_trace_processor_linenoise = false
 enable_perfetto_trace_processor_percentile = false
-enable_perfetto_trace_processor_sqlite = false
+enable_perfetto_trace_processor_sqlite = true
 enable_perfetto_ui = false
 
 extra_cflags = "@OMNITRACE_PERFETTO_C_FLAGS@"

diff --git a/docs/sphinx/requirements.txt b/docs/sphinx/requirements.txt
@@ -31,7 +31,7 @@ click==8.1.7
     #   sphinx-external-toc
 click-log==0.4.0
     # via doxysphinx
-cryptography==42.0.8
+cryptography==43.0.1
     # via pyjwt
 deprecated==1.2.14
     # via pygithub

diff --git a/examples/python/source-numpy.py b/examples/python/source-numpy.py
@@ -0,0 +1,83 @@
+#!@PYTHON_EXECUTABLE@
+
+import os
+import sys
+import time
+import omnitrace
+from omnitrace.user import region as omni_user_region
+
+_prefix = ""
+
+
+def fib(n):
+    return n if n < 2 else (fib(n - 1) + fib(n - 2))
+
+
+try:
+    import numpy as np
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n}) (1)")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _ret = np.random.rand(_len).sum()
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+except ImportError as e:
+    print(f"ImportError: {e}")
+    import random
+
+    def _sum(arr):
+        print(f"----  in _sum")
+        return sum(arr)
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n})")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _arr = [random.random() for _ in range(_len)]
+        _ret = _sum(_arr)
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+
+@omnitrace.profile()
+def run(n):
+    _ret = 0
+    _ret += fib(n)
+    _ret += inefficient(n)
+    return _ret
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--num-iterations", help="Number", type=int, default=3)
+    parser.add_argument("-v", "--value", help="Starting value", type=int, default=20)
+    parser.add_argument(
+        "-s",
+        "--stop-profile",
+        help="Stop tracing after given iterations",
+        type=int,
+        default=0,
+    )
+    args = parser.parse_args()
+
+    _prefix = os.path.basename(__file__)
+    print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n")
+    for i in range(args.num_iterations):
+        with omni_user_region(f"main_loop"):
+            if args.stop_profile > 0 and i == args.stop_profile:
+                omnitrace.user.stop_trace()
+            ans = run(args.value)
+            print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n")
diff --git a/examples/python/source.py b/examples/python/source.py
@@ -5,6 +5,7 @@
 import time
 import omnitrace
 from omnitrace.user import region as omni_user_region
+import random
 
 _prefix = ""
 
@@ -13,39 +14,23 @@ def fib(n):
     return n if n < 2 else (fib(n - 1) + fib(n - 2))
 
 
-try:
-    import numpy as np
+def _sum(arr):
+    print(f"----  in _sum")
+    return sum(arr)
 
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _ret = np.random.rand(_len).sum()
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
 
-except ImportError:
-    import random
-
-    def _sum(arr):
-        return sum(arr)
-
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _arr = [random.random() for _ in range(_len)]
-        _ret = _sum(_arr)
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
+def inefficient(n):
+    print(f"[{_prefix}] ... running inefficient({n})")
+    a = 0
+    for i in range(n):
+        a += i
+        for j in range(n):
+            a += j
+    _len = a * n * n
+    _arr = [random.random() for _ in range(_len)]
+    _ret = _sum(_arr)
+    print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+    return _ret
 
 
 @omnitrace.profile()

diff --git a/external/perfetto b/external/perfetto
diff --git a/source/lib/core/perfetto.hpp b/source/lib/core/perfetto.hpp
@@ -109,11 +109,12 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
     auto        _index     = _track_data.size();
     auto&       _name      = _name_data.emplace_back(std::make_unique<std::string>(_v));
     const char* _unit_name = (_units && strlen(_units) > 0) ? _units : nullptr;
-    _track_data.emplace_back(::perfetto::CounterTrack{ _name->c_str() }
-                                 .set_unit_name(_unit_name)
-                                 .set_category(_category)
-                                 .set_unit_multiplier(_mult)
-                                 .set_is_incremental(_incr));
+    _track_data.emplace_back(
+        ::perfetto::CounterTrack{ ::perfetto::DynamicString{ _name->c_str() } }
+            .set_unit_name(_unit_name)
+            .set_category(_category)
+            .set_unit_multiplier(_mult)
+            .set_is_incremental(_incr));
     if(config::get_is_continuous_integration())
     {
         for(auto& itr : _missing)