From 922067c38ffc3519e8470bbf0374a6b5ebf7e253 Mon Sep 17 00:00:00 2001 From: Shoken Kaneko Date: Tue, 12 May 2020 23:27:16 -0400 Subject: [PATCH 1/4] Merged qos features. The qos features implemented by Lee Savoie are merged. --- .gitignore | 1 + tracer/Makefile | 2 +- tracer/Makefile.common | 7 ++++-- tracer/coll-events.C | 10 +++++++++ tracer/p2p-events.C | 11 ++++++++++ tracer/qos-manager.C | 48 ++++++++++++++++++++++++++++++++++++++++++ tracer/qos-manager.h | 30 ++++++++++++++++++++++++++ tracer/tracer-driver.C | 45 +++++++++++++++++++++++++++++++++------ 8 files changed, 144 insertions(+), 10 deletions(-) create mode 100644 tracer/qos-manager.C create mode 100644 tracer/qos-manager.h diff --git a/.gitignore b/.gitignore index ad85b4c..e827c54 100644 --- a/.gitignore +++ b/.gitignore @@ -27,6 +27,7 @@ *.a *.d tracer/traceR +bin/traceR # hide backups *~ diff --git a/tracer/Makefile b/tracer/Makefile index 45b73e4..b4d5df0 100644 --- a/tracer/Makefile +++ b/tracer/Makefile @@ -24,7 +24,7 @@ all: traceR PREFIX = .. -SRCS = p2p-events.C coll-events.C tracer-driver.C +SRCS = p2p-events.C coll-events.C tracer-driver.C qos-manager.C OBJS = ${SRCS:.C=.o} traceR: ${OBJS} components diff --git a/tracer/Makefile.common b/tracer/Makefile.common index 9e3c5a6..26dad6b 100644 --- a/tracer/Makefile.common +++ b/tracer/Makefile.common @@ -28,10 +28,13 @@ endif SEQ_CXX = ${CXX} # ROSS install directory -ROSS_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/ross-7.0.0 +#ROSS_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/ross-7.0.0 +ROSS_DIR = ${HOME}/ROSS/install + # CODES install directory -CODES_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/codes-1.0.0 +#CODES_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/codes-1.0.0 +CODES_DIR = ${HOME}/codes_github/codes/build CODES_LIBS = -L${CODES_DIR}/lib -Wl,-rpath,${CODES_DIR}/lib CODES_LIBS += -L${ROSS_DIR}/lib -Wl,-rpath,${ROSS_DIR}/lib CODES_LIBS += -lcodes -lROSS -lm diff --git a/tracer/coll-events.C b/tracer/coll-events.C index c9827ba..377ac5b 100644 --- a/tracer/coll-events.C +++ b/tracer/coll-events.C @@ -8,6 +8,7 @@ extern "C" { #include "codes/model-net.h" +#include "codes/model-net-sched.h" #include "codes/lp-io.h" #include "codes/codes.h" #include "codes/codes_mapping.h" @@ -16,6 +17,9 @@ extern "C" { } #include "tracer-driver.h" +#include "qos-manager.h" + +extern QoSManager qosManager; // the indexing should match between the #define and the lookUpTable #define TRACER_A2A 0 @@ -155,6 +159,10 @@ void enqueue_coll_msg( m_local.proc_event_type = lookUpTable[index].local_event; m_local.executed.taskid = ns->my_pe->currentCollTask; + + int prio = qosManager.getServiceLevel(ns->my_job, lpid_to_pe(lp->id), dest); + model_net_set_msg_param(MN_MSG_PARAM_SCHED, MN_SCHED_PARAM_PRIO, (void*)&prio); + model_net_event(net_id, "coll", pe_to_lpid(dest, ns->my_job), size, sendOffset + copyTime*(isEager?1:0), sizeof(proc_msg), (const void*)&m_remote, sizeof(proc_msg), &m_local, lp); @@ -241,6 +249,8 @@ void handle_coll_recv_post_event( size = m->msgId.size; m_remote.msgId.size = size; } + int prio = qosManager.getServiceLevel(ns->my_job, lpid_to_pe(lp->id), m->msgId.pe); + model_net_set_msg_param(MN_MSG_PARAM_SCHED, MN_SCHED_PARAM_PRIO, (void*)&prio); model_net_event(net_id, "coll", pe_to_lpid(m->msgId.pe, ns->my_job), size, nic_delay, sizeof(proc_msg), (const void*)&m_remote, sizeof(proc_msg), &m_local, lp); diff --git a/tracer/p2p-events.C b/tracer/p2p-events.C index 5497107..e6fcde4 100644 --- a/tracer/p2p-events.C +++ b/tracer/p2p-events.C @@ -8,6 +8,7 @@ extern "C" { #include "codes/model-net.h" +#include "codes/model-net-sched.h" #include "codes/lp-io.h" #include "codes/codes.h" #include "codes/codes_mapping.h" @@ -16,6 +17,9 @@ extern "C" { } #include "tracer-driver.h" +#include "qos-manager.h" + +extern QoSManager qosManager; void handle_recv_event( proc_state * ns, @@ -841,6 +845,9 @@ int send_msg( #endif m_remote.iteration = iter; + int prio = qosManager.getServiceLevel(ns->my_job, lpid_to_pe(lp->id), lpid_to_pe(dest_id)); + model_net_set_msg_param(MN_MSG_PARAM_SCHED, MN_SCHED_PARAM_PRIO, (void*)&prio); + /* model_net_event params: int net_id, char* category, tw_lpid final_dest_lp, uint64_t message_size, tw_stime offset, int remote_event_size, @@ -877,6 +884,10 @@ void enqueue_msg( #endif m_remote.iteration = iter; + int prio = qosManager.getServiceLevel(ns->my_job, lpid_to_pe(lp->id), lpid_to_pe(dest_id)); + model_net_set_msg_param(MN_MSG_PARAM_SCHED, MN_SCHED_PARAM_PRIO, (void*)&prio); + + model_net_event(net_id, "p2p", dest_id, size, sendOffset, sizeof(proc_msg), (const void*)&m_remote, sizeof(proc_msg), m_local, lp); diff --git a/tracer/qos-manager.C b/tracer/qos-manager.C new file mode 100644 index 0000000..fd321c2 --- /dev/null +++ b/tracer/qos-manager.C @@ -0,0 +1,48 @@ +#include "qos-manager.h" + +#include +#include + +using namespace std; + +bool QoSManager::readQoSFileForJob(int job, const char filename[]) { + ifstream qosFile(filename); + if (!qosFile) { + fprintf(stderr, "Cannot read QoS file %s\n", filename); + return false; + } + ServiceLevel defaultSL; + qosFile >> defaultSL; + if(qosFile.fail()) { + fprintf(stderr, "Error reading default service level from QoS file %s\n", filename); + return false; + } + jobs[job].defaultSL = defaultSL; + + Rank rank; + ServiceLevel sl; + while(qosFile >> rank >> sl) { + jobs[job].serviceLevels[rank] = sl; + } + return true; +} + +QoSManager::ServiceLevel QoSManager::getServiceLevel(Job j, Rank s, Rank d) { + JobQoSMap::iterator job = jobs.find(j); + if(job != jobs.end()) { + ServiceLevelMap::iterator src = job->second.serviceLevels.find(s); + if (src != job->second.serviceLevels.end()) { + return src->second; + } + ServiceLevelMap::iterator dest = job->second.serviceLevels.find(d); + if (dest != job->second.serviceLevels.end()) { + return dest->second; + } + else { + return job->second.defaultSL; + } + } + else { + return overallDefaultSL; + } +} \ No newline at end of file diff --git a/tracer/qos-manager.h b/tracer/qos-manager.h new file mode 100644 index 0000000..75174ac --- /dev/null +++ b/tracer/qos-manager.h @@ -0,0 +1,30 @@ +#ifndef _QOS_MANAGER_H_ +#define _QOS_MANAGER_H_ + +#include + +class QoSManager { +public: + typedef int Job; + typedef int Rank; + typedef int ServiceLevel; + +private: + typedef std::map ServiceLevelMap; + struct JobQoS { + ServiceLevel defaultSL; + ServiceLevelMap serviceLevels; + }; + typedef std::map JobQoSMap; + + ServiceLevel overallDefaultSL; + JobQoSMap jobs; + +public: + QoSManager(ServiceLevel overallDefaultSL): overallDefaultSL(overallDefaultSL) {} + void setDefaultServiceLevel(ServiceLevel defaultSL) { overallDefaultSL = defaultSL; } + bool readQoSFileForJob(Job job, const char filename[]); + ServiceLevel getServiceLevel(Job job, Rank src, Rank dest); +}; + +#endif \ No newline at end of file diff --git a/tracer/tracer-driver.C b/tracer/tracer-driver.C index 560fa6a..9ed763c 100644 --- a/tracer/tracer-driver.C +++ b/tracer/tracer-driver.C @@ -37,11 +37,13 @@ extern "C" { } #include "tracer-driver.h" +#include "qos-manager.h" char tracer_input[256]; /* filename for tracer input file */ CoreInf *global_rank; /* core to job ID and process ID */ JobInf *jobs; +QoSManager qosManager(0); int default_mapping; int total_ranks; tw_stime *jobTimes; @@ -179,6 +181,14 @@ int main(int argc, char **argv) printf("Eager limit is %f bytes\n", eager_limit); int ret; + int default_sl; + ret = configuration_get_value_int(&config, "PARAMS", "default_sl", NULL, + &default_sl); + if (ret == 0) { + qosManager.setDefaultServiceLevel(default_sl); + printf("Default service level is %d\n", default_sl); + } + /* set up the output directory */ if(lp_io_dir[0]) { ret = lp_io_prepare(lp_io_dir, 0, &handle, MPI_COMM_WORLD); @@ -241,25 +251,44 @@ int main(int argc, char **argv) total_ranks = 0; /* read per job information */ +#define LINE_SIZE 1024 + char line[LINE_SIZE]; + fgets(line, LINE_SIZE, jobIn); // Eat everything to the next newline for(int i = 0; i < num_jobs; i++) { + fgets(line, LINE_SIZE, jobIn); + FILE * lineStream = fmemopen(line, strlen(line), "r"); #if TRACER_BIGSIM_TRACES char tempTrace[200]; - fscanf(jobIn, "%s", tempTrace); + fscanf(lineStream, "%s", tempTrace); sprintf(jobs[i].traceDir, "%s%s", tempTrace, "/bgTrace"); #else - fscanf(jobIn, "%s", jobs[i].traceDir); + fscanf(lineStream, "%s", jobs[i].traceDir); #endif - fscanf(jobIn, "%s", jobs[i].map_file); - fscanf(jobIn, "%d", &jobs[i].numRanks); /* number of processes */ - fscanf(jobIn, "%d", &jobs[i].numIters); /* number of repetitions */ + fscanf(lineStream, "%s", jobs[i].map_file); + fscanf(lineStream, "%d", &jobs[i].numRanks); /* number of processes */ + fscanf(lineStream, "%d", &jobs[i].numIters); /* number of repetitions */ + + // See if there is a QoS file + char qosFile[LINE_SIZE] = ""; + if (fscanf(lineStream, "%s", qosFile) == 1) { + if(!qosManager.readQoSFileForJob(i, qosFile)) { + // The QoS Manager already printed an error message, so just quit + MPI_Abort(MPI_COMM_WORLD, 1); + } + } + total_ranks += jobs[i].numRanks; jobs[i].rankMap = (int*) malloc(jobs[i].numRanks * sizeof(int)); jobs[i].skipMsgId = -1; jobTimes[i] = 0; if(!rank) { - printf("Job %d - ranks %d, trace folder %s, rank file %s, iters %d\n", + printf("Job %d - ranks %d, trace folder %s, rank file %s, iters %d", i, jobs[i].numRanks, jobs[i].traceDir, jobs[i].map_file, jobs[i].numIters); - } + if(qosFile[0] != '\0') { + printf(", QoS file %s", qosFile); + } + printf("\n"); + } } if (total_ranks > num_servers) { @@ -323,6 +352,8 @@ int main(int argc, char **argv) fscanf(jobIn, "%c", &next); } + fclose(jobIn); + int ranks_till_now = 0; for(int i = 0; i < num_jobs && !dump_topo_only; i++) { int num_workers = jobs[i].numRanks; From d4717242e682be55118d3000cb2ff3028d8fe36e Mon Sep 17 00:00:00 2001 From: kaneko60 <56411560+kaneko60@users.noreply.github.com> Date: Fri, 6 Nov 2020 12:08:08 -0500 Subject: [PATCH 2/4] Update Makefile.common Removed modifications --- tracer/Makefile.common | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tracer/Makefile.common b/tracer/Makefile.common index 26dad6b..a040e94 100644 --- a/tracer/Makefile.common +++ b/tracer/Makefile.common @@ -28,13 +28,11 @@ endif SEQ_CXX = ${CXX} # ROSS install directory -#ROSS_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/ross-7.0.0 -ROSS_DIR = ${HOME}/ROSS/install +ROSS_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/ross-7.0.0 # CODES install directory -#CODES_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/codes-1.0.0 -CODES_DIR = ${HOME}/codes_github/codes/build +CODES_DIR = ${HOME}/spack/opt/spack/linux-rhel7-x86_64/gcc-4.9.3/codes-1.0.0 CODES_LIBS = -L${CODES_DIR}/lib -Wl,-rpath,${CODES_DIR}/lib CODES_LIBS += -L${ROSS_DIR}/lib -Wl,-rpath,${ROSS_DIR}/lib CODES_LIBS += -lcodes -lROSS -lm From 9a8dcd2680f9b9c615a788558774a6155717d19c Mon Sep 17 00:00:00 2001 From: kaneko60 <56411560+kaneko60@users.noreply.github.com> Date: Fri, 6 Nov 2020 13:27:31 -0500 Subject: [PATCH 3/4] Add files via upload Applied updates from master branch. --- tracer/reader/otf2_reader.C | 22 ++++++++++++++++++++++ tracer/reader/otf2_reader.h | 1 + 2 files changed, 23 insertions(+) diff --git a/tracer/reader/otf2_reader.C b/tracer/reader/otf2_reader.C index 222c417..f1345e4 100644 --- a/tracer/reader/otf2_reader.C +++ b/tracer/reader/otf2_reader.C @@ -143,6 +143,14 @@ callbackDefRegion(void * userData, } else { new_r.isLoopEvt = false; } + if(strncmp(((AllData*)userData)->strings[name].c_str(), "TRACER_", 7) == 0 && + strncmp(((AllData*)userData)->strings[name].c_str(), "TRACER_Loop", 11) != 0 && + strncmp(((AllData*)userData)->strings[name].c_str(), "TRACER_WallTime", 15) != 0){ + new_r.isExtraTracerEvt = true; + } + else{ + new_r.isExtraTracerEvt = false; + } if(regionRole == OTF2_REGION_ROLE_BARRIER || regionRole == OTF2_REGION_ROLE_IMPLICIT_BARRIER || regionRole == OTF2_REGION_ROLE_COLL_ONE2ALL || @@ -212,6 +220,13 @@ callbackEvtBegin( OTF2_LocationRef location, new_task.loopStartEvent = true; new_task.event_id = TRACER_LOOP_EVT; } + if(globalData->regions[region].isExtraTracerEvt) { + ld->tasks.push_back(Task()); + Task &new_task = ld->tasks[ld->tasks.size() - 1]; + new_task.execTime = 0; + new_task.event_id = region; + new_task.beginEvent = true; + } ld->lastLogTime = time; return OTF2_CALLBACK_SUCCESS; } @@ -241,6 +256,13 @@ callbackEvtEnd( OTF2_LocationRef location, new_task.loopEvent = true; new_task.event_id = TRACER_LOOP_EVT; } + if(globalData->regions[region].isExtraTracerEvt) { + addUserEvt(userData, time); + ld->tasks.push_back(Task()); + Task &new_task = ld->tasks[ld->tasks.size() - 1]; + new_task.execTime = 0; + new_task.event_id = region; + } ld->lastLogTime = time; return OTF2_CALLBACK_SUCCESS; } diff --git a/tracer/reader/otf2_reader.h b/tracer/reader/otf2_reader.h index 1431f56..c09cd44 100644 --- a/tracer/reader/otf2_reader.h +++ b/tracer/reader/otf2_reader.h @@ -59,6 +59,7 @@ struct Region { bool isTracerPrintEvt; bool isLoopEvt; bool isCommunication; + bool isExtraTracerEvt; }; struct Group { From b0c9584d8151f15a2711f9bbf9971a9a59d32aea Mon Sep 17 00:00:00 2001 From: kaneko60 <56411560+kaneko60@users.noreply.github.com> Date: Fri, 6 Nov 2020 13:28:31 -0500 Subject: [PATCH 4/4] Add files via upload --- docs/scorep-5.0.patch | 265 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 265 insertions(+) create mode 100644 docs/scorep-5.0.patch diff --git a/docs/scorep-5.0.patch b/docs/scorep-5.0.patch new file mode 100644 index 0000000..b291ad4 --- /dev/null +++ b/docs/scorep-5.0.patch @@ -0,0 +1,265 @@ +diff -Naur scorep-5.0.original/src/adapters/mpi/SCOREP_Mpi.h scorep-5.0/src/adapters/mpi/SCOREP_Mpi.h +--- scorep-5.0.original/src/adapters/mpi/SCOREP_Mpi.h 2019-03-22 10:41:49.127806703 -0700 ++++ scorep-5.0/src/adapters/mpi/SCOREP_Mpi.h 2019-04-03 01:58:09.441425000 -0700 +@@ -119,6 +119,8 @@ + * Flag which indicates whether event generation is turned on/off. + */ + extern bool scorep_mpi_generate_events; ++extern bool scorep_mpi_reduce_probe_test; ++extern uint64_t scorep_mpi_last_probe_test_evt; + + /** + Flag which indicates whether recording of MPI topologies is enabled. +diff -Naur scorep-5.0.original/src/adapters/mpi/SCOREP_Mpi_P2p.c scorep-5.0/src/adapters/mpi/SCOREP_Mpi_P2p.c +--- scorep-5.0.original/src/adapters/mpi/SCOREP_Mpi_P2p.c 2019-03-22 10:41:49.127806703 -0700 ++++ scorep-5.0/src/adapters/mpi/SCOREP_Mpi_P2p.c 2019-04-24 14:22:41.803509000 -0700 +@@ -74,6 +74,8 @@ + */ + static int scorep_mpi_status_array_size = 0; + ++extern uint64_t SCOREP_get_number_of_events(); ++ + /** + * Get a pointer to a status array of at least 'size' statuses + * @param size minimal requested size +@@ -1444,8 +1446,11 @@ + const int event_gen_active = SCOREP_MPI_IS_EVENT_GEN_ON; + const int event_gen_active_for_group = SCOREP_MPI_IS_EVENT_GEN_ON_FOR( SCOREP_MPI_ENABLED_P2P ); + int return_val; ++ uint64_t num_evts_written = SCOREP_get_number_of_events(); ++ bool gen_region = !scorep_mpi_reduce_probe_test || ++ (num_evts_written != scorep_mpi_last_probe_test_evt); + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { + SCOREP_MPI_EVENT_GEN_OFF(); + if ( event_gen_active_for_group ) +@@ -1462,7 +1467,7 @@ + return_val = PMPI_Iprobe( source, tag, comm, flag, status ); + SCOREP_EXIT_WRAPPED_REGION(); + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { + if ( event_gen_active_for_group ) + { +@@ -1473,6 +1478,7 @@ + SCOREP_ExitWrapper( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_IPROBE ] ); + } + SCOREP_MPI_EVENT_GEN_ON(); ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); + } + + SCOREP_IN_MEASUREMENT_DECREMENT(); +@@ -1918,8 +1924,11 @@ + scorep_mpi_request* orig_req; + MPI_Status mystatus; + uint64_t start_time_stamp; ++ uint64_t num_evts_written = SCOREP_get_number_of_events(); ++ bool gen_region = !scorep_mpi_reduce_probe_test || ++ (num_evts_written != scorep_mpi_last_probe_test_evt); + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { + SCOREP_MPI_EVENT_GEN_OFF(); + +@@ -1963,8 +1972,9 @@ + SCOREP_MpiRequestTested( orig_req->id ); + } + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { ++ bool updateEvt = (scorep_mpi_last_probe_test_evt == SCOREP_get_number_of_events()); + if ( event_gen_active_for_group ) + { + SCOREP_ExitRegion( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TEST ] ); +@@ -1975,6 +1985,9 @@ + } + + SCOREP_MPI_EVENT_GEN_ON(); ++ if(updateEvt) { ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); ++ } + } + + SCOREP_IN_MEASUREMENT_DECREMENT(); +@@ -2006,8 +2019,11 @@ + scorep_mpi_request* orig_req; + MPI_Status mystatus; + uint64_t start_time_stamp; ++ uint64_t num_evts_written = SCOREP_get_number_of_events(); ++ bool gen_region = !scorep_mpi_reduce_probe_test || ++ (num_evts_written != scorep_mpi_last_probe_test_evt); + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { + SCOREP_MPI_EVENT_GEN_OFF(); + +@@ -2019,6 +2035,7 @@ + { + SCOREP_EnterWrapper( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTANY ] ); + } ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); + } + + #if !defined( SCOREP_MPI_NO_HOOKS ) +@@ -2072,8 +2089,9 @@ + #endif + scorep_mpi_check_request( orig_req, status ); + } +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { ++ bool updateEvt = (scorep_mpi_last_probe_test_evt == SCOREP_get_number_of_events()); + if ( event_gen_active_for_group ) + { + SCOREP_ExitRegion( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTANY ] ); +@@ -2084,6 +2102,9 @@ + } + + SCOREP_MPI_EVENT_GEN_ON(); ++ if(updateEvt) { ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); ++ } + } + + SCOREP_IN_MEASUREMENT_DECREMENT(); +@@ -2114,8 +2135,12 @@ + int i; + scorep_mpi_request* orig_req; + uint64_t start_time_stamp; ++ uint64_t num_evts_written = SCOREP_get_number_of_events(); ++ bool gen_region = !scorep_mpi_reduce_probe_test || ++ (num_evts_written != scorep_mpi_last_probe_test_evt); + +- if ( event_gen_active ) ++ ++ if ( event_gen_active && gen_region ) + { + SCOREP_MPI_EVENT_GEN_OFF(); + +@@ -2127,6 +2152,8 @@ + { + SCOREP_EnterWrapper( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTALL ] ); + } ++ ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); + } + + #if !defined( SCOREP_MPI_NO_HOOKS ) +@@ -2177,8 +2204,9 @@ + } + } + } +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { ++ bool updateEvt = (scorep_mpi_last_probe_test_evt == SCOREP_get_number_of_events()); + if ( event_gen_active_for_group ) + { + SCOREP_ExitRegion( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTALL ] ); +@@ -2189,6 +2217,9 @@ + } + + SCOREP_MPI_EVENT_GEN_ON(); ++ if(updateEvt) { ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); ++ } + } + + SCOREP_IN_MEASUREMENT_DECREMENT(); +@@ -2220,8 +2251,11 @@ + int i; + scorep_mpi_request* orig_req; + uint64_t start_time_stamp; ++ uint64_t num_evts_written = SCOREP_get_number_of_events(); ++ bool gen_region = !scorep_mpi_reduce_probe_test || ++ (num_evts_written != scorep_mpi_last_probe_test_evt); + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { + SCOREP_MPI_EVENT_GEN_OFF(); + +@@ -2233,6 +2267,7 @@ + { + SCOREP_EnterWrapper( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTSOME ] ); + } ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); + } + + #if !defined( SCOREP_MPI_NO_HOOKS ) +@@ -2314,8 +2349,10 @@ + } + } + +- if ( event_gen_active ) ++ if ( event_gen_active && gen_region ) + { ++ bool updateEvt = (scorep_mpi_last_probe_test_evt == ++ SCOREP_get_number_of_events()); + if ( event_gen_active_for_group ) + { + SCOREP_ExitRegion( scorep_mpi_regions[ SCOREP_MPI_REGION__MPI_TESTSOME ] ); +@@ -2326,6 +2363,9 @@ + } + + SCOREP_MPI_EVENT_GEN_ON(); ++ if(updateEvt) { ++ scorep_mpi_last_probe_test_evt = SCOREP_get_number_of_events(); ++ } + } + + SCOREP_IN_MEASUREMENT_DECREMENT(); +diff -Naur scorep-5.0.original/src/adapters/mpi/scorep_mpi_init.c scorep-5.0/src/adapters/mpi/scorep_mpi_init.c +--- scorep-5.0.original/src/adapters/mpi/scorep_mpi_init.c 2019-03-22 10:41:49.131806652 -0700 ++++ scorep-5.0/src/adapters/mpi/scorep_mpi_init.c 2019-04-03 01:59:13.540480000 -0700 +@@ -308,6 +308,10 @@ + mpi_subsystem_begin( void ) + { + SCOREP_MPI_EVENT_GEN_ON(); ++ char *reduce_logs = getenv("SCOREP_REDUCE_PROBE_TEST"); ++ if(reduce_logs != NULL) { ++ scorep_mpi_reduce_probe_test = (atoi(reduce_logs) == 1); ++ } + return SCOREP_SUCCESS; + } + +@@ -426,6 +430,8 @@ + events are generated. + */ + bool scorep_mpi_generate_events = false; ++bool scorep_mpi_reduce_probe_test = false; ++uint64_t scorep_mpi_last_probe_test_evt = -1; + + /** + * @internal +diff -Naur scorep-5.0.original/src/measurement/tracing/SCOREP_Tracing.c scorep-5.0/src/measurement/tracing/SCOREP_Tracing.c +--- scorep-5.0.original/src/measurement/tracing/SCOREP_Tracing.c 2019-03-22 10:41:49.391803294 -0700 ++++ scorep-5.0/src/measurement/tracing/SCOREP_Tracing.c 2019-04-03 02:12:25.959430000 -0700 +@@ -580,3 +580,11 @@ + write_properties(); + write_definitions(); + } ++ ++uint64_t SCOREP_get_number_of_events() { ++ OTF2_EvtWriter* writer = scorep_tracing_get_trace_data( SCOREP_Location_GetCurrentCPULocation() )->otf_writer; ++ if(writer == NULL) return 0; ++ uint64_t num_evts_written; ++ OTF2_EvtWriter_GetNumberOfEvents( writer, &num_evts_written); ++ return num_evts_written; ++} +diff -Naur scorep-5.0.original/src/measurement/tracing/SCOREP_Tracing.h scorep-5.0/src/measurement/tracing/SCOREP_Tracing.h +--- scorep-5.0.original/src/measurement/tracing/SCOREP_Tracing.h 2019-03-22 10:41:49.391803294 -0700 ++++ scorep-5.0/src/measurement/tracing/SCOREP_Tracing.h 2019-04-03 02:11:30.732441000 -0700 +@@ -86,5 +86,6 @@ + void + SCOREP_Tracing_Write( void ); + ++uint64_t SCOREP_get_number_of_events(); + + #endif /* SCOREP_TRACING_H */