Skip to content

Commit

Permalink
Fix finalization segfaults (#174)
Browse files Browse the repository at this point in the history 
- update timemory submodule with fixes to papi components and signals
update
  • Loading branch information
@jrmadsen
jrmadsen authored Oct 4, 2022
1 parent 7d7a8f2 commit 2a387f9
Show file tree
Hide file tree
Showing 14 changed files with 94 additions and 84 deletions.
2 changes: 1 addition & 1 deletion external/timemory
Submodule timemory updated 40 files
+2 −2 docs/features.md
+1 −1 examples/ex-cxx-basic/ex_cxx_basic.cpp
+1 −1 examples/ex-cxx-tuple/ex_cxx_tuple.cpp
+1 −1 examples/ex-ert/ex_ert.cpp
+1 −1 examples/ex-ert/ex_ert_kokkos.cpp
+1 −1 examples/ex-gpu-roofline/ex_gpu_roofline.cpp
+1 −1 examples/ex-gpu/ex_gpu_event.cu
+1 −1 examples/ex-statistics/ex_statistics.cpp
+1 −1 source/pthread.cpp
+6 −6 source/python/libpytimemory-signals.cpp
+7 −7 source/python/libpytimemory.cpp
+4 −3 source/tests/ptl_tests.cpp
+2 −1 source/tests/test_macros.hpp
+2 −1 source/tests/throttle_tests.cpp
+8 −8 source/tests/warning_tests.cpp
+19 −6 source/timemory/CMakeLists.txt
+18 −9 source/timemory/components/papi/papi_array.hpp
+11 −14 source/timemory/components/papi/papi_rate_tuple.hpp
+4 −4 source/timemory/components/papi/papi_tuple.hpp
+7 −1 source/timemory/components/papi/papi_vector.cpp
+14 −9 source/timemory/components/papi/papi_vector.hpp
+4 −1 source/timemory/config/config.cpp
+2 −2 source/timemory/sampling/allocator.hpp
+4 −4 source/timemory/sampling/sampler.cpp
+4 −4 source/timemory/settings/settings.cpp
+4 −0 source/timemory/signals/CMakeLists.txt
+37 −0 source/timemory/signals/macros.hpp
+58 −65 source/timemory/signals/signal_handlers.cpp
+102 −0 source/timemory/signals/signal_handlers.hpp
+2 −2 source/timemory/signals/signal_mask.hpp
+62 −60 source/timemory/signals/signal_settings.cpp
+120 −0 source/timemory/signals/signal_settings.hpp
+97 −0 source/timemory/signals/types.hpp
+1 −15 source/timemory/utility/declaration.hpp
+5 −224 source/timemory/utility/signals.hpp
+4 −3 source/tools/timemory-compiler-instrument/CMakeLists.txt
+3 −3 source/tools/timemory-compiler-instrument/compiler-instrument.cpp
+1 −1 source/tools/timemory-ert/timemory-ert.cpp
+1 −1 source/tools/timemory-timem/timem.cpp
+5 −5 source/trace.cpp
8 changes: 4 additions & 4 deletions source/bin/omnitrace/omnitrace.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,8 +29,8 @@
#include <timemory/hash.hpp>
#include <timemory/log/macros.hpp>
#include <timemory/manager.hpp>
#include <timemory/sampling/signals.hpp>
#include <timemory/settings.hpp>
#include <timemory/signals/signal_mask.hpp>
#include <timemory/utility/console.hpp>
#include <timemory/utility/demangle.hpp>
#include <timemory/utility/signals.hpp>
Expand Down Expand Up @@ -202,8 +202,8 @@ int
main(int argc, char** argv)
{
{
using signal_settings = tim::signal_settings;
using sys_signal = tim::sys_signal;
using signal_settings = tim::signals::signal_settings;
using sys_signal = tim::signals::sys_signal;

// default signals to catch
for(const auto& itr :
Expand Down Expand Up @@ -239,7 +239,7 @@ main(int argc, char** argv)

signal_settings::set_exit_action(_exit_action);
signal_settings::check_environment();
tim::enable_signal_detection(signal_settings::get_enabled());
tim::signals::enable_signal_detection(signal_settings::get_enabled());
}

argv0 = argv[0];
Expand Down
6 changes: 3 additions & 3 deletions source/lib/omnitrace/library.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,7 @@
#include <timemory/hash/types.hpp>
#include <timemory/manager/manager.hpp>
#include <timemory/operations/types/file_output_message.hpp>
#include <timemory/sampling/signals.hpp>
#include <timemory/signals/signal_mask.hpp>
#include <timemory/settings/types.hpp>
#include <timemory/utility/backtrace.hpp>
#include <timemory/utility/procfs/maps.hpp>
Expand Down Expand Up @@ -585,8 +585,8 @@ omnitrace_finalize_hidden(void)

thread_info::set_stop(comp::wall_clock::record());

tim::sampling::block_signals(get_sampling_signals(),
tim::sampling::sigmask_scope::process);
tim::signals::block_signals(get_sampling_signals(),
tim::signals::sigmask_scope::process);

// some functions called during finalization may alter the push/pop count so we need
// to save them here
Expand Down
6 changes: 3 additions & 3 deletions source/lib/omnitrace/library/components/mpi_gotcha.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@
#include <timemory/backends/mpi.hpp>
#include <timemory/backends/process.hpp>
#include <timemory/mpl/types.hpp>
#include <timemory/sampling/signals.hpp>
#include <timemory/signals/signal_mask.hpp>
#include <timemory/utility/locking.hpp>

#include <cstdint>
Expand Down Expand Up @@ -114,7 +114,7 @@ omnitrace_mpi_set_attr()
OMNITRACE_DEBUG("MPI Comm attribute finalize\n");
auto _blocked = get_sampling_signals();
if(!_blocked.empty())
tim::sampling::block_signals(_blocked, tim::sampling::sigmask_scope::process);
tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);
if(mpip_index != std::numeric_limits<uint64_t>::max())
comp::deactivate_mpip<mpip_bundle_t, project::omnitrace>(mpip_index);
omnitrace_finalize_hidden();
Expand Down Expand Up @@ -249,7 +249,7 @@ mpi_gotcha::audit(const gotcha_data_t& _data, audit::incoming)

auto _blocked = get_sampling_signals();
if(!_blocked.empty())
tim::sampling::block_signals(_blocked, tim::sampling::sigmask_scope::process);
tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);

if(mpip_index != std::numeric_limits<uint64_t>::max())
comp::deactivate_mpip<mpip_bundle_t, project::omnitrace>(mpip_index);
Expand Down
4 changes: 2 additions & 2 deletions source/lib/omnitrace/library/components/pthread_create_gotcha.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -400,7 +400,7 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
if(_enable_sampling && !_blocked.empty())
{
OMNITRACE_DEBUG("blocking signals...\n");
tim::sampling::block_signals(_blocked, tim::sampling::sigmask_scope::process);
tim::signals::block_signals(_blocked, tim::signals::sigmask_scope::process);
}

if(_use_bundle)
Expand All @@ -425,7 +425,7 @@ pthread_create_gotcha::operator()(pthread_t* thread, const pthread_attr_t* attr,
if(_enable_sampling && !_blocked.empty())
{
OMNITRACE_DEBUG("unblocking signals...\n");
tim::sampling::unblock_signals(_blocked, tim::sampling::sigmask_scope::process);
tim::signals::unblock_signals(_blocked, tim::signals::sigmask_scope::process);
}

OMNITRACE_DEBUG("returning success...\n");
Expand Down
14 changes: 8 additions & 6 deletions source/lib/omnitrace/library/components/pthread_mutex_gotcha.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ pthread_mutex_gotcha::operator()(uintptr_t&& _id, int (*_callee)(Args...),
{
using bundle_t = category_region<category::pthread>;

if(is_disabled())
if(is_disabled() || m_protect)
{
if(!_callee)
{
Expand Down Expand Up @@ -225,39 +225,41 @@ int
pthread_mutex_gotcha::operator()(int (*_callee)(pthread_mutex_t*),
pthread_mutex_t* _mutex) const
{
if(m_protect) return (*_callee)(_mutex);
if(get_state() != ::omnitrace::State::Active || m_protect) return (*_callee)(_mutex);
return (*this)(reinterpret_cast<uintptr_t>(_mutex), _callee, _mutex);
}

int
pthread_mutex_gotcha::operator()(int (*_callee)(pthread_spinlock_t*),
pthread_spinlock_t* _lock) const
{
if(m_protect) return (*_callee)(_lock);
if(get_state() != ::omnitrace::State::Active || m_protect) return (*_callee)(_lock);
return (*this)(reinterpret_cast<uintptr_t>(_lock), _callee, _lock);
}

int
pthread_mutex_gotcha::operator()(int (*_callee)(pthread_rwlock_t*),
pthread_rwlock_t* _lock) const
{
if(m_protect) return (*_callee)(_lock);
if(get_state() != ::omnitrace::State::Active || m_protect) return (*_callee)(_lock);
return (*this)(reinterpret_cast<uintptr_t>(_lock), _callee, _lock);
}

int
pthread_mutex_gotcha::operator()(int (*_callee)(pthread_barrier_t*),
pthread_barrier_t* _barrier) const
{
if(m_protect) return (*_callee)(_barrier);
if(get_state() != ::omnitrace::State::Active || m_protect)
return (*_callee)(_barrier);
return (*this)(reinterpret_cast<uintptr_t>(_barrier), _callee, _barrier);
}

int
pthread_mutex_gotcha::operator()(int (*_callee)(pthread_t, void**), pthread_t _thr,
void** _tinfo) const
{
if(m_protect) return (*_callee)(_thr, _tinfo);
if(get_state() != ::omnitrace::State::Active || m_protect)
return (*_callee)(_thr, _tinfo);
return (*this)(static_cast<uintptr_t>(threading::get_id()), _callee, _thr, _tinfo);
}

Expand Down
22 changes: 11 additions & 11 deletions source/lib/omnitrace/library/components/roctracer.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -137,15 +137,15 @@ roctracer::setup()
};
#endif

ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_set_properties(ACTIVITY_DOMAIN_HIP_API, nullptr));

// Allocating tracing pool
roctracer_properties_t properties{};
memset(&properties, 0, sizeof(roctracer_properties_t));
// properties.mode = 0x1000;
properties.buffer_size = 0x100;
properties.buffer_callback_fun = hip_activity_callback;
ROCTRACER_CALL(roctracer_open_pool(&properties));
OMNITRACE_ROCTRACER_CALL(roctracer_open_pool(&properties));

#if OMNITRACE_HIP_VERSION_MAJOR == 4 && OMNITRACE_HIP_VERSION_MINOR >= 4
// HIP 4.5.0 has an invalid warning
Expand All @@ -155,15 +155,15 @@ roctracer::setup()
"invalid domain ID(4) in: roctracer_enable_activity()" };
#endif

ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API,
hip_api_callback, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_HIP_API,
hip_api_callback, nullptr));
if(get_use_roctx())
{
ROCTRACER_CALL(roctracer_enable_domain_callback(ACTIVITY_DOMAIN_ROCTX,
roctx_api_callback, nullptr));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_callback(
ACTIVITY_DOMAIN_ROCTX, roctx_api_callback, nullptr));
}
// Enable HIP activity tracing
ROCTRACER_CALL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
OMNITRACE_ROCTRACER_CALL(roctracer_enable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));

// callback for HSA
for(auto& itr : roctracer_setup_routines())
Expand Down Expand Up @@ -212,19 +212,19 @@ roctracer::shutdown()
};
#endif

// ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_ROCTX));
// OMNITRACE_ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_ROCTX));
OMNITRACE_VERBOSE_F(
2, "executing roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API)...\n");
ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API));
OMNITRACE_ROCTRACER_CALL(roctracer_disable_domain_callback(ACTIVITY_DOMAIN_HIP_API));

OMNITRACE_VERBOSE_F(
2, "executing roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS)...\n");
ROCTRACER_CALL(roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));
OMNITRACE_ROCTRACER_CALL(roctracer_disable_domain_activity(ACTIVITY_DOMAIN_HIP_OPS));

if(roctracer_activity_count() == 0)
{
OMNITRACE_VERBOSE_F(2, "executing roctracer_flush_activity()...\n");
ROCTRACER_CALL(roctracer_flush_activity());
OMNITRACE_ROCTRACER_CALL(roctracer_flush_activity());
}
else
{
Expand Down
42 changes: 20 additions & 22 deletions source/lib/omnitrace/library/config.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ void
finalize()
{
OMNITRACE_DEBUG("[omnitrace_finalize] Disabling signal handling...\n");
tim::disable_signal_detection();
tim::signals::disable_signal_detection();
_settings_are_configured() = false;
}

Expand Down Expand Up @@ -930,12 +930,12 @@ configure_signal_handler()

if(_config->get_enable_signal_handler())
{
using signal_settings = tim::signal_settings;
using sys_signal = tim::sys_signal;
tim::disable_signal_detection();
using signal_settings = tim::signals::signal_settings;
using sys_signal = tim::signals::sys_signal;
tim::signals::disable_signal_detection();
auto _exit_action = [](int nsig) {
tim::sampling::block_signals(get_sampling_signals(),
tim::sampling::sigmask_scope::process);
tim::signals::block_signals(get_sampling_signals(),
tim::signals::sigmask_scope::process);
OMNITRACE_BASIC_PRINT(
"Finalizing afer signal %i :: %s\n", nsig,
signal_settings::str(static_cast<sys_signal>(nsig)).c_str());
Expand All @@ -950,29 +950,27 @@ configure_signal_handler()
if(_ignore_dyninst_trampoline)
signal_settings::disable(static_cast<sys_signal>(_dyninst_trampoline_signal));
auto enabled_signals = signal_settings::get_enabled();
tim::enable_signal_detection(enabled_signals);
tim::signals::enable_signal_detection(enabled_signals);
}

if(_ignore_dyninst_trampoline)
{
using signal_handler_t = void (*)(int);
static signal_handler_t _old_handler = nullptr;
static auto _trampoline_handler = [](int _v) {
if(_v == _dyninst_trampoline_signal)
using signal_handler_t = void (*)(int);
static auto _trampoline_handler = [](int _v) {
if(get_verbose_env() >= 2)
{
auto _info =
::tim::signal_settings::get_info(static_cast<tim::sys_signal>(_v));
OMNITRACE_VERBOSE(
2,
"signal %s (%i) ignored (OMNITRACE_IGNORE_DYNINST_TRAMPOLINE=ON)\n",
std::get<0>(_info).c_str(), _v);
if(get_verbose_env() > 1 || get_debug_env())
timemory_print_demangled_backtrace<64>();
if(_old_handler) _old_handler(_v);
::omnitrace::debug::flush();
::omnitrace::debug::lock _debug_lk{};
OMNITRACE_FPRINTF_STDERR_COLOR(warning);
fprintf(::omnitrace::debug::get_file(),
"signal %i ignored (OMNITRACE_IGNORE_DYNINST_TRAMPOLINE=ON)\n",
_v);
::omnitrace::debug::flush();
timemory_print_demangled_backtrace<64>();
}
};
_old_handler = signal(_dyninst_trampoline_signal,
static_cast<signal_handler_t>(_trampoline_handler));
::signal(_dyninst_trampoline_signal,
static_cast<signal_handler_t>(_trampoline_handler));
}
}

Expand Down
8 changes: 4 additions & 4 deletions source/lib/omnitrace/library/debug.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -353,7 +353,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
fprintf(::omnitrace::debug::get_file(), __VA_ARGS__); \
::omnitrace::debug::flush(); \
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::signals::disable_signal_detection(); \
timemory_print_demangled_backtrace<64>(); \
METHOD; \
}
Expand All @@ -368,7 +368,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
fprintf(::omnitrace::debug::get_file(), __VA_ARGS__); \
::omnitrace::debug::flush(); \
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::signals::disable_signal_detection(); \
timemory_print_demangled_backtrace<64>(); \
METHOD; \
}
Expand All @@ -385,7 +385,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
fprintf(::omnitrace::debug::get_file(), __VA_ARGS__); \
::omnitrace::debug::flush(); \
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::signals::disable_signal_detection(); \
timemory_print_demangled_backtrace<64>(); \
METHOD; \
}
Expand All @@ -400,7 +400,7 @@ get_chars(T&& _c, std::index_sequence<Idx...>)
fprintf(::omnitrace::debug::get_file(), __VA_ARGS__); \
::omnitrace::debug::flush(); \
::omnitrace::set_state(::omnitrace::State::Finalized); \
::tim::disable_signal_detection(); \
::tim::signals::disable_signal_detection(); \
timemory_print_demangled_backtrace<64>(); \
METHOD; \
}
Expand Down
8 changes: 1 addition & 7 deletions source/lib/omnitrace/library/ompt.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,7 @@ ompt_start_tool(unsigned int omp_version, const char* runtime_version)
return 1; // success
};

static auto ompt_finalize = [](ompt_data_t* tool_data) {
if(_use_ompt)
{
TIMEMORY_PRINTF(stderr, "OpenMP-tools finalized\n\n");
tim::consume_parameters(tool_data);
}
};
static auto ompt_finalize = [](ompt_data_t*) {};

static auto data = ompt_start_tool_result_t{ ompt_initialize, ompt_finalize, { 0 } };
return (ompt_start_tool_result_t*) &data;
Expand Down
Loading

0 comments on commit 2a387f9

Please sign in to comment.