Update Perfetto and fix tests (#378)

Fix for "SWDEV-479652" - Perfetto-based tests are failing. Updated version of perfetto submodule to v46.0. Modified Omnitrace code that uses Perfetto, so it can compile. Modified the testing code, so it can run the version of trace_processor_shell provided (v46.0). --------- Signed-off-by: Aleksandar Janicijevic <[email protected]> Co-authored-by: David Galiffi <[email protected]>
ROCm · Sep 13, 2024 · 96d7b8f · 96d7b8f
1 parent 1413ab6
commit 96d7b8f
Show file tree

Hide file tree

Showing 15 changed files with 173 additions and 70 deletions.
diff --git a/.github/workflows/opensuse.yml b/.github/workflows/opensuse.yml
@@ -58,6 +58,11 @@ jobs:
         timeout_minutes: 25
         max_attempts: 5
         command: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Configure Env
@@ -87,6 +92,7 @@ jobs:
           -DOMNITRACE_USE_HIP=OFF
           -DOMNITRACE_USE_OMPT=OFF
           -DOMNITRACE_USE_PYTHON=ON
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"

diff --git a/.github/workflows/redhat.yml b/.github/workflows/redhat.yml
@@ -68,6 +68,16 @@ jobs:
         env
 
     - name: Install Packages
+      shell: bash
+      run: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+
+    - name: Install ROCm Packages
       if: ${{ matrix.rocm-version > 0 }}
       timeout-minutes: 30
       shell: bash
@@ -109,6 +119,7 @@ jobs:
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON
           -DOMNITRACE_MAX_THREADS=64
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"

diff --git a/.github/workflows/ubuntu-focal.yml b/.github/workflows/ubuntu-focal.yml
@@ -94,9 +94,10 @@ jobs:
           apt-get update &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -246,20 +247,12 @@ jobs:
         mpi-headers: ['OFF']
         build-jobs: ['3']
         ctest-exclude: ['-LE "mpi-example|transpose"']
-        perfetto-tools: ['OFF']
         include:
-          - compiler: 'g++'
-            rocm-version: '4.5'
-            mpi-headers: 'OFF'
-            build-jobs: '2'
-            ctest-exclude: '-LE "mpi-example|transpose"'
-            perfetto-tools: 'ON'
           - compiler: 'g++'
             rocm-version: 'debian'
             mpi-headers: 'ON'
             build-jobs: '2'
             ctest-exclude: '-LE transpose'
-            perfetto-tools: 'OFF'
 
     env:
       BUILD_TYPE: MinSizeRel
@@ -283,7 +276,10 @@ jobs:
           echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -345,7 +341,6 @@ jobs:
           -DOMNITRACE_USE_PYTHON=ON
           -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }}
           -DOMNITRACE_USE_SANITIZER=OFF
-          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${{ matrix.perfetto-tools }}
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=${{ matrix.mpi-headers }}
@@ -444,9 +439,10 @@ jobs:
           sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
           if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
           if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           sudo apt-get -y --purge autoremove &&
           sudo apt-get -y clean
@@ -596,9 +592,10 @@ jobs:
         command: |
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.24.1' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&

diff --git a/.github/workflows/ubuntu-jammy.yml b/.github/workflows/ubuntu-jammy.yml
@@ -110,8 +110,10 @@ jobs:
           apt-get install -y software-properties-common &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 

diff --git a/cmake/Packages.cmake b/cmake/Packages.cmake
@@ -483,7 +483,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/perfetto
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://android.googlesource.com/platform/external/perfetto
-    REPO_BRANCH v28.0
+    REPO_BRANCH v46.0
     TEST_FILE sdk/perfetto.cc)
 
 include(Perfetto)

diff --git a/cmake/Perfetto.cmake b/cmake/Perfetto.cmake
@@ -67,10 +67,10 @@ if(CMAKE_CXX_COMPILER_IS_CLANG)
 else()
     set(PERFETTO_IS_CLANG false)
     set(OMNITRACE_PERFETTO_C_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow"
         CACHE STRING "Perfetto C flags")
     set(OMNITRACE_PERFETTO_CXX_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow -Wno-mismatched-new-delete"
         CACHE STRING "Perfetto C++ flags")
 endif()
 

diff --git a/cmake/Templates/args.gn.in b/cmake/Templates/args.gn.in
@@ -8,19 +8,19 @@ is_clang = @PERFETTO_IS_CLANG@
 is_hermetic_clang = false
 
 enable_perfetto_benchmarks = false
-enable_perfetto_integration_tests = false
+enable_perfetto_integration_tests = true
 enable_perfetto_unittests = false
 enable_perfetto_fuzzers = false
 
 # enable_perfetto_stderr_crash_dump = false
 enable_perfetto_heapprofd = false
 enable_perfetto_tools = false
-enable_perfetto_trace_processor = false
-enable_perfetto_trace_processor_httpd = false
+enable_perfetto_trace_processor = true
+enable_perfetto_trace_processor_httpd = true
 enable_perfetto_trace_processor_json = false
 enable_perfetto_trace_processor_linenoise = false
 enable_perfetto_trace_processor_percentile = false
-enable_perfetto_trace_processor_sqlite = false
+enable_perfetto_trace_processor_sqlite = true
 enable_perfetto_ui = false
 
 extra_cflags = "@OMNITRACE_PERFETTO_C_FLAGS@"

diff --git a/examples/python/source-numpy.py b/examples/python/source-numpy.py
@@ -0,0 +1,83 @@
+#!@PYTHON_EXECUTABLE@
+
+import os
+import sys
+import time
+import omnitrace
+from omnitrace.user import region as omni_user_region
+
+_prefix = ""
+
+
+def fib(n):
+    return n if n < 2 else (fib(n - 1) + fib(n - 2))
+
+
+try:
+    import numpy as np
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n}) (1)")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _ret = np.random.rand(_len).sum()
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+except ImportError as e:
+    print(f"ImportError: {e}")
+    import random
+
+    def _sum(arr):
+        print(f"----  in _sum")
+        return sum(arr)
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n})")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _arr = [random.random() for _ in range(_len)]
+        _ret = _sum(_arr)
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+
+@omnitrace.profile()
+def run(n):
+    _ret = 0
+    _ret += fib(n)
+    _ret += inefficient(n)
+    return _ret
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--num-iterations", help="Number", type=int, default=3)
+    parser.add_argument("-v", "--value", help="Starting value", type=int, default=20)
+    parser.add_argument(
+        "-s",
+        "--stop-profile",
+        help="Stop tracing after given iterations",
+        type=int,
+        default=0,
+    )
+    args = parser.parse_args()
+
+    _prefix = os.path.basename(__file__)
+    print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n")
+    for i in range(args.num_iterations):
+        with omni_user_region(f"main_loop"):
+            if args.stop_profile > 0 and i == args.stop_profile:
+                omnitrace.user.stop_trace()
+            ans = run(args.value)
+            print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n")
diff --git a/examples/python/source.py b/examples/python/source.py
@@ -5,6 +5,7 @@
 import time
 import omnitrace
 from omnitrace.user import region as omni_user_region
+import random
 
 _prefix = ""
 
@@ -13,39 +14,23 @@ def fib(n):
     return n if n < 2 else (fib(n - 1) + fib(n - 2))
 
 
-try:
-    import numpy as np
+def _sum(arr):
+    print(f"----  in _sum")
+    return sum(arr)
 
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _ret = np.random.rand(_len).sum()
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
 
-except ImportError:
-    import random
-
-    def _sum(arr):
-        return sum(arr)
-
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _arr = [random.random() for _ in range(_len)]
-        _ret = _sum(_arr)
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
+def inefficient(n):
+    print(f"[{_prefix}] ... running inefficient({n})")
+    a = 0
+    for i in range(n):
+        a += i
+        for j in range(n):
+            a += j
+    _len = a * n * n
+    _arr = [random.random() for _ in range(_len)]
+    _ret = _sum(_arr)
+    print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+    return _ret
 
 
 @omnitrace.profile()

diff --git a/external/perfetto b/external/perfetto
diff --git a/source/lib/core/perfetto.hpp b/source/lib/core/perfetto.hpp
@@ -109,11 +109,12 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
     auto        _index     = _track_data.size();
     auto&       _name      = _name_data.emplace_back(std::make_unique<std::string>(_v));
     const char* _unit_name = (_units && strlen(_units) > 0) ? _units : nullptr;
-    _track_data.emplace_back(::perfetto::CounterTrack{ _name->c_str() }
-                                 .set_unit_name(_unit_name)
-                                 .set_category(_category)
-                                 .set_unit_multiplier(_mult)
-                                 .set_is_incremental(_incr));
+    _track_data.emplace_back(
+        ::perfetto::CounterTrack{ ::perfetto::DynamicString{ _name->c_str() } }
+            .set_unit_name(_unit_name)
+            .set_category(_category)
+            .set_unit_multiplier(_mult)
+            .set_is_incremental(_incr));
     if(config::get_is_continuous_integration())
     {
         for(auto& itr : _missing)

diff --git a/source/lib/omnitrace/library/tracing.hpp b/source/lib/omnitrace/library/tracing.hpp
@@ -642,14 +642,14 @@ mark_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
 
 template <typename CategoryT, typename... Args>
 inline void
-mark_perfetto_track(CategoryT, const char*, ::perfetto::Track _track, uint64_t _ts,
+mark_perfetto_track(CategoryT, const char* name, ::perfetto::Track _track, uint64_t _ts,
                     Args&&... args)
 {
     // skip if category is disabled
     if(category_mark_disabled<CategoryT>()) return;
 
-    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, _track, _ts,
-                        std::forward<Args>(args)...);
+    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, ::perfetto::DynamicString{ name },
+                        _track, _ts, std::forward<Args>(args)...);
 }
 }  // namespace tracing
 }  // namespace omnitrace
diff --git a/tests/omnitrace-python-tests.cmake b/tests/omnitrace-python-tests.cmake
@@ -166,7 +166,8 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             NAME ${TEST_NAME}-validate-perfetto
             COMMAND
                 ${_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py
-                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -i
+                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -t
+                /opt/trace_processor/bin/trace_processor_shell -i
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/${TEST_NAME}/${_VERSION}/${TEST_PERFETTO_FILE}
             DEPENDS ${TEST_NAME}-${_VERSION}

diff --git a/tests/omnitrace-testing.cmake b/tests/omnitrace-testing.cmake
@@ -985,6 +985,7 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
                 ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py -m
                 "${TEST_PERFETTO_METRIC}" ${TEST_ARGS} -i
                 ${PROJECT_BINARY_DIR}/omnitrace-tests-output/${TEST_NAME}/${TEST_PERFETTO_FILE}
+                -t /opt/trace_processor/bin/trace_processor_shell
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()