Update Perfetto and fix tests

.github/workflows/opensuse.yml

@@ -58,6 +58,11 @@ jobs:
         timeout_minutes: 25
         max_attempts: 5
         command: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 
     - name: Configure Env
@@ -87,6 +92,7 @@ jobs:
           -DOMNITRACE_USE_HIP=OFF
           -DOMNITRACE_USE_OMPT=OFF
           -DOMNITRACE_USE_PYTHON=ON
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"

.github/workflows/redhat.yml

@@ -68,6 +68,16 @@ jobs:
         env
 
     - name: Install Packages
+      shell: bash
+      run: |
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell
+          python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
+          python3 -m pip install 'cmake==3.21.4' &&
+          for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
+
+    - name: Install ROCm Packages
       if: ${{ matrix.rocm-version > 0 }}
       timeout-minutes: 30
       shell: bash
@@ -109,6 +119,7 @@ jobs:
           -DOMNITRACE_USE_MPI_HEADERS=ON
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=ON
           -DOMNITRACE_MAX_THREADS=64
+          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=OFF
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_DISABLE_EXAMPLES="transpose;rccl"

.github/workflows/ubuntu-focal.yml

@@ -94,9 +94,10 @@ jobs:
           apt-get update &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libmpich-dev mpich environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -246,20 +247,12 @@ jobs:
         mpi-headers: ['OFF']
         build-jobs: ['3']
         ctest-exclude: ['-LE "mpi-example|transpose"']
-        perfetto-tools: ['OFF']
         include:
-          - compiler: 'g++'
-            rocm-version: '4.5'
-            mpi-headers: 'OFF'
-            build-jobs: '2'
-            ctest-exclude: '-LE "mpi-example|transpose"'
-            perfetto-tools: 'ON'
           - compiler: 'g++'
             rocm-version: 'debian'
             mpi-headers: 'ON'
             build-jobs: '2'
             ctest-exclude: '-LE transpose'
-            perfetto-tools: 'OFF'
 
     env:
       BUILD_TYPE: MinSizeRel
@@ -283,7 +276,10 @@ jobs:
           echo "deb [arch=amd64] https://repo.radeon.com/rocm/apt/${{ matrix.rocm-version }}/ ubuntu main" | tee /etc/apt/sources.list.d/rocm.list &&
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev ${{ matrix.compiler }} libudev1 libnuma1 rocm-dev rocm-utils rocm-smi-lib roctracer-dev rocprofiler-dev rccl-dev hip-base hsa-amd-aqlprofile hsa-rocr-dev hsakmt-roct-dev libpapi-dev curl libopenmpi-dev openmpi-bin libfabric-dev &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&
@@ -345,7 +341,6 @@ jobs:
           -DOMNITRACE_USE_PYTHON=ON
           -DOMNITRACE_USE_MPI_HEADERS=${{ matrix.mpi-headers }}
           -DOMNITRACE_USE_SANITIZER=OFF
-          -DOMNITRACE_INSTALL_PERFETTO_TOOLS=${{ matrix.perfetto-tools }}
           -DOMNITRACE_PYTHON_PREFIX=/opt/conda/envs
           -DOMNITRACE_PYTHON_ENVS="py3.6;py3.7;py3.8;py3.9;py3.10;py3.11"
           -DOMNITRACE_CI_MPI_RUN_AS_ROOT=${{ matrix.mpi-headers }}
@@ -444,9 +439,10 @@ jobs:
           sudo apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules ${{ matrix.deps }} ${{ matrix.compiler }} &&
           if [ "${{ matrix.mpi }}" = "mpich" ]; then sudo apt-get install -y libmpich-dev mpich; fi &&
           if [ "${{ matrix.mpi }}" = "openmpi" ]; then sudo apt-get install -y libopenmpi-dev openmpi-bin libfabric-dev; fi &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.16.3' &&
           sudo apt-get -y --purge autoremove &&
           sudo apt-get -y clean
@@ -596,9 +592,10 @@ jobs:
         command: |
           apt-get update &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip clang libomp-dev environment-modules gcc g++ mpich libmpich-dev texinfo &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
-          python3 -m pip install perfetto &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.24.1' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done &&
           apt-get -y --purge autoremove &&

.github/workflows/ubuntu-jammy.yml

@@ -110,8 +110,10 @@ jobs:
           apt-get install -y software-properties-common &&
           apt-get upgrade -y &&
           apt-get install -y build-essential m4 autoconf libtool python3-pip libiberty-dev clang libomp-dev libopenmpi-dev libfabric-dev openmpi-bin environment-modules ${{ matrix.compiler }} &&
+          wget https://commondatastorage.googleapis.com/perfetto-luci-artifacts/v46.0/linux-amd64/trace_processor_shell -P /opt/trace_processor/bin &&
+          chmod +x /opt/trace_processor/bin/trace_processor_shell &&
           python3 -m pip install --upgrade pip &&
-          python3 -m pip install numpy &&
+          python3 -m pip install numpy perfetto dataclasses &&
           python3 -m pip install 'cmake==3.21.4' &&
           for i in 6 7 8 9 10; do /opt/conda/envs/py3.${i}/bin/python -m pip install numpy perfetto dataclasses; done
 

cmake/Packages.cmake

@@ -483,7 +483,7 @@ omnitrace_checkout_git_submodule(
     RELATIVE_PATH external/perfetto
     WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}
     REPO_URL https://android.googlesource.com/platform/external/perfetto
-    REPO_BRANCH v28.0
+    REPO_BRANCH v46.0
     TEST_FILE sdk/perfetto.cc)
 
 include(Perfetto)

cmake/Perfetto.cmake

@@ -67,10 +67,10 @@ if(CMAKE_CXX_COMPILER_IS_CLANG)
 else()
     set(PERFETTO_IS_CLANG false)
     set(OMNITRACE_PERFETTO_C_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow"
         CACHE STRING "Perfetto C flags")
     set(OMNITRACE_PERFETTO_CXX_FLAGS
-        "-static-libgcc -Wno-maybe-uninitialized"
+        "-static-libgcc -Wno-maybe-uninitialized -Wno-stringop-overflow -Wno-mismatched-new-delete"
         CACHE STRING "Perfetto C++ flags")
 endif()
 

cmake/Templates/args.gn.in

@@ -8,19 +8,19 @@ is_clang = @PERFETTO_IS_CLANG@
 is_hermetic_clang = false
 
 enable_perfetto_benchmarks = false
-enable_perfetto_integration_tests = false
+enable_perfetto_integration_tests = true
 enable_perfetto_unittests = false
 enable_perfetto_fuzzers = false
 
 # enable_perfetto_stderr_crash_dump = false
 enable_perfetto_heapprofd = false
 enable_perfetto_tools = false
-enable_perfetto_trace_processor = false
-enable_perfetto_trace_processor_httpd = false
+enable_perfetto_trace_processor = true
+enable_perfetto_trace_processor_httpd = true
 enable_perfetto_trace_processor_json = false
 enable_perfetto_trace_processor_linenoise = false
 enable_perfetto_trace_processor_percentile = false
-enable_perfetto_trace_processor_sqlite = false
+enable_perfetto_trace_processor_sqlite = true
 enable_perfetto_ui = false
 
 extra_cflags = "@OMNITRACE_PERFETTO_C_FLAGS@"

examples/python/source-numpy.py

@@ -0,0 +1,83 @@
+#!@PYTHON_EXECUTABLE@
+
+import os
+import sys
+import time
+import omnitrace
+from omnitrace.user import region as omni_user_region
+
+_prefix = ""
+
+
+def fib(n):
+    return n if n < 2 else (fib(n - 1) + fib(n - 2))
+
+
+try:
+    import numpy as np
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n}) (1)")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _ret = np.random.rand(_len).sum()
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+except ImportError as e:
+    print(f"ImportError: {e}")
+    import random
+
+    def _sum(arr):
+        print(f"----  in _sum")
+        return sum(arr)
+
+    def inefficient(n):
+        print(f"[{_prefix}] ... running inefficient({n})")
+        a = 0
+        for i in range(n):
+            a += i
+            for j in range(n):
+                a += j
+        _len = a * n * n
+        _arr = [random.random() for _ in range(_len)]
+        _ret = _sum(_arr)
+        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+        return _ret
+
+
+@omnitrace.profile()
+def run(n):
+    _ret = 0
+    _ret += fib(n)
+    _ret += inefficient(n)
+    return _ret
+
+
+if __name__ == "__main__":
+    import argparse
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-n", "--num-iterations", help="Number", type=int, default=3)
+    parser.add_argument("-v", "--value", help="Starting value", type=int, default=20)
+    parser.add_argument(
+        "-s",
+        "--stop-profile",
+        help="Stop tracing after given iterations",
+        type=int,
+        default=0,
+    )
+    args = parser.parse_args()
+
+    _prefix = os.path.basename(__file__)
+    print(f"[{_prefix}] Executing {args.num_iterations} iterations...\n")
+    for i in range(args.num_iterations):
+        with omni_user_region(f"main_loop"):
+            if args.stop_profile > 0 and i == args.stop_profile:
+                omnitrace.user.stop_trace()
+            ans = run(args.value)
+            print(f"[{_prefix}] [{i}] result of run({args.value}) = {ans}\n")

examples/python/source.py

@@ -5,6 +5,7 @@
 import time
 import omnitrace
 from omnitrace.user import region as omni_user_region
+import random
 
 _prefix = ""
 
@@ -13,39 +14,23 @@ def fib(n):
     return n if n < 2 else (fib(n - 1) + fib(n - 2))
 
 
-try:
-    import numpy as np
+def _sum(arr):
+    print(f"----  in _sum")
+    return sum(arr)
 
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _ret = np.random.rand(_len).sum()
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
 
-except ImportError:
-    import random
-
-    def _sum(arr):
-        return sum(arr)
-
-    def inefficient(n):
-        print(f"[{_prefix}] ... running inefficient({n})")
-        a = 0
-        for i in range(n):
-            a += i
-            for j in range(n):
-                a += j
-        _len = a * n * n
-        _arr = [random.random() for _ in range(_len)]
-        _ret = _sum(_arr)
-        print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
-        return _ret
+def inefficient(n):
+    print(f"[{_prefix}] ... running inefficient({n})")
+    a = 0
+    for i in range(n):
+        a += i
+        for j in range(n):
+            a += j
+    _len = a * n * n
+    _arr = [random.random() for _ in range(_len)]
+    _ret = _sum(_arr)
+    print(f"[{_prefix}] ... sum of {_len} random elements: {_ret}")
+    return _ret
 
 
 @omnitrace.profile()

source/lib/core/perfetto.hpp

@@ -109,11 +109,12 @@ perfetto_counter_track<Tp>::emplace(size_t _idx, const std::string& _v,
     auto        _index     = _track_data.size();
     auto&       _name      = _name_data.emplace_back(std::make_unique<std::string>(_v));
     const char* _unit_name = (_units && strlen(_units) > 0) ? _units : nullptr;
-    _track_data.emplace_back(::perfetto::CounterTrack{ _name->c_str() }
-                                 .set_unit_name(_unit_name)
-                                 .set_category(_category)
-                                 .set_unit_multiplier(_mult)
-                                 .set_is_incremental(_incr));
+    _track_data.emplace_back(
+        ::perfetto::CounterTrack{ ::perfetto::DynamicString{ _name->c_str() } }
+            .set_unit_name(_unit_name)
+            .set_category(_category)
+            .set_unit_multiplier(_mult)
+            .set_is_incremental(_incr));
     if(config::get_is_continuous_integration())
     {
         for(auto& itr : _missing)

source/lib/omnitrace/library/tracing.hpp

@@ -642,14 +642,14 @@ mark_perfetto_ts(CategoryT, const char* name, uint64_t _ts, Args&&... args)
 
 template <typename CategoryT, typename... Args>
 inline void
-mark_perfetto_track(CategoryT, const char*, ::perfetto::Track _track, uint64_t _ts,
+mark_perfetto_track(CategoryT, const char* name, ::perfetto::Track _track, uint64_t _ts,
                     Args&&... args)
 {
     // skip if category is disabled
     if(category_mark_disabled<CategoryT>()) return;
 
-    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, _track, _ts,
-                        std::forward<Args>(args)...);
+    TRACE_EVENT_INSTANT(trait::name<CategoryT>::value, ::perfetto::DynamicString{ name },
+                        _track, _ts, std::forward<Args>(args)...);
 }
 }  // namespace tracing
 }  // namespace omnitrace

tests/omnitrace-python-tests.cmake

@@ -166,7 +166,8 @@ foreach(_VERSION ${OMNITRACE_PYTHON_VERSIONS})
             NAME ${TEST_NAME}-validate-perfetto
             COMMAND
                 ${_PYTHON_EXECUTABLE} ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py
-                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -i
+                -m ${TEST_PERFETTO_METRIC} ${TEST_ARGS} -p -t
+                /opt/trace_processor/bin/trace_processor_shell -i
             PYTHON_VERSION ${_VERSION}
             FILE omnitrace-tests-output/${TEST_NAME}/${_VERSION}/${TEST_PERFETTO_FILE}
             DEPENDS ${TEST_NAME}-${_VERSION}

tests/omnitrace-testing.cmake

@@ -985,6 +985,7 @@ function(OMNITRACE_ADD_VALIDATION_TEST)
                 ${CMAKE_CURRENT_LIST_DIR}/validate-perfetto-proto.py -m
                 "${TEST_PERFETTO_METRIC}" ${TEST_ARGS} -i
                 ${PROJECT_BINARY_DIR}/omnitrace-tests-output/${TEST_NAME}/${TEST_PERFETTO_FILE}
+                -t /opt/trace_processor/bin/trace_processor_shell
             WORKING_DIRECTORY ${PROJECT_BINARY_DIR})
     endif()