Rewrite Caliper integration

* replace its Cython interface by a Python interface
* fix bug in the begin/end markers script interface
* use the original Caliper annotation macros
* build Caliper as a CMake subproject
* add integration tests

.gitlab-ci.yml

@@ -68,7 +68,7 @@ style_doxygen:
     - mkdir build
     - cd build
     - cp ../maintainer/configs/maxset.hpp myconfig.hpp
-    - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_FFT=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON
+    - cmake .. -D ESPRESSO_BUILD_WITH_CUDA=ON -D ESPRESSO_BUILD_WITH_GSL=ON -D ESPRESSO_BUILD_WITH_HDF5=ON -D ESPRESSO_BUILD_WITH_SCAFACOS=ON -D ESPRESSO_BUILD_WITH_WALBERLA=ON -D ESPRESSO_BUILD_WITH_WALBERLA_FFT=ON -D ESPRESSO_BUILD_WITH_STOKESIAN_DYNAMICS=ON -D ESPRESSO_BUILD_WITH_CALIPER=ON
     - sh ../maintainer/CI/dox_warnings.sh
   tags:
     - espresso
@@ -111,6 +111,7 @@ maxset:
      with_scafacos: 'true'
      with_walberla: 'true'
      with_stokesian_dynamics: 'true'
+     with_caliper: 'true'
      check_skip_long: 'true'
      cmake_params: '-D ESPRESSO_TEST_NP=8'
   script:
@@ -241,6 +242,7 @@ cuda11-maxset:
      with_walberla: 'true'
      with_walberla_avx: 'true'
      with_stokesian_dynamics: 'true'
+     with_caliper: 'true'
   script:
     - bash maintainer/CI/build_cmake.sh
   artifacts:

CMakeLists.txt

@@ -100,6 +100,7 @@ option(ESPRESSO_BUILD_WITH_WALBERLA_AVX
 option(ESPRESSO_BUILD_WITH_WALBERLA_FFT "Build waLBerla with FFT support" OFF)
 option(ESPRESSO_BUILD_BENCHMARKS "Enable benchmarks" OFF)
 option(ESPRESSO_BUILD_WITH_VALGRIND "Build with Valgrind instrumentation" OFF)
+option(ESPRESSO_BUILD_WITH_CALIPER "Build with Caliper instrumentation" OFF)
 option(ESPRESSO_BUILD_WITH_CPPCHECK "Run Cppcheck during compilation" OFF)
 if(CMAKE_CXX_COMPILER_ID MATCHES "Clang")
   option(ESPRESSO_BUILD_WITH_CLANG_TIDY "Run Clang-Tidy during compilation" OFF)
@@ -127,8 +128,6 @@ option(
 option(ESPRESSO_WARNINGS_ARE_ERRORS
        "Treat warnings as errors during compilation" OFF)
 option(ESPRESSO_BUILD_WITH_CCACHE "Use ccache compiler invocation." OFF)
-option(ESPRESSO_BUILD_WITH_PROFILER "Build with Caliper profiler annotations."
-       OFF)
 option(ESPRESSO_INSIDE_DOCKER "Set this to ON when running inside Docker." OFF)
 mark_as_advanced(ESPRESSO_INSIDE_DOCKER)
 set(ESPRESSO_TEST_TIMEOUT "300"
@@ -655,6 +654,33 @@ if(ESPRESSO_BUILD_WITH_WALBERLA)
   endif()
 endif()
 
+if(ESPRESSO_BUILD_WITH_CALIPER)
+  # cmake-format: off
+  include(FetchContent)
+  FetchContent_Declare(
+    caliper
+    GIT_REPOSITORY https://github.com/LLNL/Caliper.git
+    GIT_TAG        v2.9.1
+  )
+  if(NOT DEFINED caliper_SOURCE_DIR OR NOT EXISTS "${caliper_SOURCE_DIR}")
+    FetchContent_Populate(caliper)
+  endif()
+  # cmake-format: on
+  set(CALIPER_OPTION_PREFIX on CACHE BOOL "")
+  set(CALIPER_WITH_MPI on CACHE BOOL "")
+  if(ESPRESSO_BUILD_WITH_CUDA)
+    set(CALIPER_WITH_NVTX on CACHE BOOL "")
+    set(CALIPER_WITH_CUPTI on CACHE BOOL "")
+  endif()
+  set(CALIPER_BUILD_SHARED_LIBS on CACHE BOOL "")
+  add_subdirectory("${caliper_SOURCE_DIR}")
+  if(CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL
+                                              "GNU")
+    target_compile_options(caliper-services
+                           PRIVATE -Wno-deprecated-declarations)
+  endif()
+endif()
+
 #
 # Subdirectories
 #

cmake/espresso_cmake_config.cmakein

@@ -19,6 +19,8 @@
 
 #cmakedefine ESPRESSO_BUILD_WITH_VALGRIND
 
+#cmakedefine ESPRESSO_BUILD_WITH_CALIPER
+
 #define PACKAGE_NAME "${PROJECT_NAME}"
 
 /**

doc/sphinx/installation.rst

@@ -770,7 +770,7 @@ The following options control features from external libraries:
 The following options control code instrumentation:
 
 * ``ESPRESSO_BUILD_WITH_VALGRIND``: Build with Valgrind instrumentation
-* ``ESPRESSO_BUILD_WITH_PROFILER``: Build with Caliper profiler annotations
+* ``ESPRESSO_BUILD_WITH_CALIPER``: Build with Caliper instrumentation
 * ``ESPRESSO_BUILD_WITH_MSAN``: Compile C++ code with memory sanitizer
 * ``ESPRESSO_BUILD_WITH_ASAN``: Compile C++ code with address sanitizer
 * ``ESPRESSO_BUILD_WITH_UBSAN``: Compile C++ code with undefined behavior sanitizer
@@ -902,6 +902,12 @@ the clone. You can automate this task by adapting the following commands:
 
     sed -ri 's|GIT_REPOSITORY +.+stokesian-dynamics.git|GIT_REPOSITORY /work/username/stokesian_dynamics|' CMakeLists.txt
 
+* ``ESPRESSO_BUILD_WITH_CALIPER``
+
+  .. code-block:: bash
+
+    sed -ri 's|GIT_REPOSITORY +.+/Caliper.git|GIT_REPOSITORY /work/username/caliper|' CMakeLists.txt
+
 
 Compiling, testing and installing
 ---------------------------------

maintainer/CI/build_cmake.sh

@@ -105,6 +105,7 @@ set_default_value with_coverage_python ${with_coverage}
 set_default_value with_ubsan false
 set_default_value with_asan false
 set_default_value with_static_analysis false
+set_default_value with_caliper false
 set_default_value myconfig "default"
 set_default_value build_procs ${ci_procs}
 set_default_value check_procs ${build_procs}
@@ -159,6 +160,10 @@ if [ "${with_ccache}" = true ]; then
     cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_CCACHE=ON"
 fi
 
+if [ "${with_caliper}" = true ]; then
+    cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_CALIPER=ON"
+fi
+
 if [ "${with_hdf5}" = true ]; then
     cmake_params="${cmake_params} -D ESPRESSO_BUILD_WITH_HDF5=ON"
 else

src/config/features.def

@@ -117,3 +117,4 @@ STOKESIAN_DYNAMICS external
 WALBERLA external
 WALBERLA_FFT external
 VALGRIND external
+CALIPER external

src/core/forces.cpp

@@ -53,7 +53,9 @@
 
 #include <boost/variant.hpp>
 
-#include <profiler/profiler.hpp>
+#ifdef CALIPER
+#include <caliper/cali.h>
+#endif
 
 #include <cassert>
 #include <memory>
@@ -112,7 +114,10 @@ inline ParticleForce init_real_particle_force(Particle const &p,
 static void init_forces(const ParticleRange &particles,
                         const ParticleRange &ghost_particles, double time_step,
                         double kT) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
+
   /* The force initialization depends on the used thermostat and the
      thermodynamic ensemble */
 
@@ -143,12 +148,20 @@ void init_forces_ghosts(const ParticleRange &particles) {
 }
 
 void force_calc(CellStructure &cell_structure, double time_step, double kT) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
 
 #ifdef CUDA
+#ifdef CALIPER
+  CALI_MARK_BEGIN("copy_particles_to_GPU");
+#endif
   auto &espresso_system = System::get_system();
   espresso_system.gpu.update();
+#ifdef CALIPER
+  CALI_MARK_END("copy_particles_to_GPU");
 #endif
+#endif // CUDA
 
 #ifdef COLLISION_DETECTION
   prepare_local_collision_queue();
@@ -230,8 +243,14 @@ void force_calc(CellStructure &cell_structure, double time_step, double kT) {
   }
 
 #ifdef CUDA
+#ifdef CALIPER
+  CALI_MARK_BEGIN("copy_forces_from_GPU");
+#endif
   espresso_system.gpu.copy_forces_to_host(particles, this_node);
+#ifdef CALIPER
+  CALI_MARK_END("copy_forces_from_GPU");
 #endif
+#endif // CUDA
 
 // VIRTUAL_SITES distribute forces
 #ifdef VIRTUAL_SITES
@@ -252,7 +271,10 @@ void force_calc(CellStructure &cell_structure, double time_step, double kT) {
 }
 
 void calc_long_range_forces(const ParticleRange &particles) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
+
 #ifdef ELECTROSTATICS
   /* calculate k-space part of electrostatic interaction. */
   Coulomb::calc_long_range_force(particles);

src/core/grid_based_algorithms/lb_particle_coupling.cpp

@@ -30,12 +30,15 @@
 #include "grid_based_algorithms/lb_interpolation.hpp"
 #include "grid_based_algorithms/lb_particle_coupling.hpp"
 
-#include <profiler/profiler.hpp>
 #include <utils/Counter.hpp>
 #include <utils/Vector.hpp>
 
 #include <boost/mpi.hpp>
 
+#ifdef CALIPER
+#include <caliper/cali.h>
+#endif
+
 #include <cmath>
 #include <cstdint>
 #include <initializer_list>
@@ -283,7 +286,9 @@ static void lb_coupling_sanity_checks(Particle const &p) {
 
 void couple_particles(bool couple_virtual, ParticleRange const &real_particles,
                       ParticleRange const &ghost_particles, double time_step) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
   if (lattice_switch == ActiveLB::WALBERLA_LB) {
     if (lb_particle_coupling.couple_to_md) {
       ParticleCoupling coupling{couple_virtual, time_step};

src/core/integrate.cpp

@@ -57,11 +57,13 @@
 #include "thermostat.hpp"
 #include "virtual_sites.hpp"
 
-#include <profiler/profiler.hpp>
-
 #include <boost/mpi/collectives/reduce.hpp>
 #include <boost/range/algorithm/min_element.hpp>
 
+#ifdef CALIPER
+#include <caliper/cali.h>
+#endif
+
 #ifdef VALGRIND
 #include <callgrind.h>
 #endif
@@ -256,7 +258,9 @@ static void integrator_step_2(ParticleRange const &particles, double kT) {
 }
 
 int integrate(int n_steps, int reuse_forces) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
 
   // Prepare particle structure and run sanity checks of all active algorithms
   on_integration_start(time_step);
@@ -268,7 +272,9 @@ int integrate(int n_steps, int reuse_forces) {
   // Additional preparations for the first integration step
   if (reuse_forces == INTEG_REUSE_FORCES_NEVER or
       (recalc_forces and reuse_forces != INTEG_REUSE_FORCES_ALWAYS)) {
-    ESPRESSO_PROFILER_MARK_BEGIN("Initial Force Calculation");
+#ifdef CALIPER
+    CALI_MARK_BEGIN("Initial Force Calculation");
+#endif
     lb_lbcoupling_deactivate();
 
 #ifdef VIRTUAL_SITES
@@ -286,7 +292,9 @@ int integrate(int n_steps, int reuse_forces) {
 #endif
     }
 
-    ESPRESSO_PROFILER_MARK_END("Initial Force Calculation");
+#ifdef CALIPER
+    CALI_MARK_END("Initial Force Calculation");
+#endif
   }
 
   lb_lbcoupling_activate();
@@ -307,10 +315,14 @@ int integrate(int n_steps, int reuse_forces) {
   CALLGRIND_START_INSTRUMENTATION;
 #endif
   // Integration loop
-  ESPRESSO_PROFILER_CXX_MARK_LOOP_BEGIN(integration_loop, "Integration loop");
+#ifdef CALIPER
+  CALI_CXX_MARK_LOOP_BEGIN(integration_loop, "Integration loop");
+#endif
   int integrated_steps = 0;
   for (int step = 0; step < n_steps; step++) {
-    ESPRESSO_PROFILER_CXX_MARK_LOOP_ITERATION(integration_loop, step);
+#ifdef CALIPER
+    CALI_CXX_MARK_LOOP_ITERATION(integration_loop, step);
+#endif
 
     auto particles = cell_structure.local_particles();
 
@@ -440,7 +452,9 @@ int integrate(int n_steps, int reuse_forces) {
 
   } // for-loop over integration steps
   LeesEdwards::update_box_params();
-  ESPRESSO_PROFILER_CXX_MARK_LOOP_END(integration_loop);
+#ifdef CALIPER
+  CALI_CXX_MARK_LOOP_END(integration_loop);
+#endif
 
 #ifdef VALGRIND
   CALLGRIND_STOP_INSTRUMENTATION;

src/core/short_range_loop.hpp

@@ -19,9 +19,13 @@
 #ifndef CORE_SHORT_RANGE_HPP
 #define CORE_SHORT_RANGE_HPP
 
+#include "config/config.hpp"
+
 #include "cells.hpp"
 
-#include <profiler/profiler.hpp>
+#ifdef CALIPER
+#include <caliper/cali.h>
+#endif
 
 #include <cassert>
 
@@ -40,7 +44,9 @@ template <class BondKernel, class PairKernel,
 void short_range_loop(BondKernel bond_kernel, PairKernel pair_kernel,
                       double pair_cutoff, double bond_cutoff,
                       const VerletCriterion &verlet_criterion = {}) {
-  ESPRESSO_PROFILER_CXX_MARK_FUNCTION;
+#ifdef CALIPER
+  CALI_CXX_MARK_FUNCTION;
+#endif
 
   assert(cell_structure.get_resort_particles() == Cells::RESORT_NONE);
 

src/profiler/CMakeLists.txt

@@ -19,18 +19,20 @@
 
 add_library(espresso_profiler INTERFACE)
 add_library(espresso::profiler ALIAS espresso_profiler)
-target_include_directories(espresso_profiler INTERFACE "include")
 
 if(ESPRESSO_BUILD_WITH_VALGRIND)
   target_include_directories(espresso_profiler
                              INTERFACE "${VALGRIND_INCLUDE_DIRS}")
 endif()
 
-if(ESPRESSO_BUILD_WITH_PROFILER)
-  find_package(caliper REQUIRED)
-
-  target_link_libraries(espresso_profiler INTERFACE caliper-mpi)
-  target_compile_definitions(espresso_profiler INTERFACE HAVE_CALIPER)
-  set_target_properties(espresso_profiler
-                        PROPERTIES CXX_CLANG_TIDY "${ESPRESSO_CXX_CLANG_TIDY}")
+if(ESPRESSO_BUILD_WITH_CALIPER)
+  target_link_libraries(espresso_profiler INTERFACE caliper)
+  # workaround to handle the case where the build directory is outside `_deps`
+  cmake_path(GET CMAKE_BINARY_DIR FILENAME CMAKE_BINARY_DIR_BASENAME)
+  set(caliper_BINARY_DIR_ALTERNATIVE
+      "${CMAKE_BINARY_DIR}/${CMAKE_BINARY_DIR_BASENAME}/_deps/caliper-src")
+  target_include_directories(
+    espresso_profiler
+    INTERFACE "${caliper_SOURCE_DIR}/include" "${caliper_BINARY_DIR}/include"
+              "${caliper_BINARY_DIR_ALTERNATIVE}/include")
 endif()