diff --git a/CMakeLists.txt b/CMakeLists.txt index 7ba0a9ff..469ab75d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -391,12 +391,12 @@ if (WITH_AMS_LIB) list(APPEND AMS_BENCHMARKS_DEFINES "-D__ENABLE_PERFFLOWASPECT__") endif() - # if (WITH_ADIAK) - # find_package(adiak REQUIRED) - # list(APPEND AMS_BENCHMARKS_DEFINES "-D__ENABLE_ADIAK__") - # list(APPEND AMS_BENCHMARKS_INCLUDES ${adiak_INCLUDE_DIR}) - # list(APPEND AMS_BENCHMARKS_LIBRARIES adiak::adiak) - # endif() + if (WITH_ADIAK) + find_package(adiak REQUIRED) + list(APPEND AMS_BENCHMARKS_DEFINES "-D__ENABLE_ADIAK__") + list(APPEND AMS_BENCHMARKS_INCLUDES ${adiak_INCLUDE_DIR}) + list(APPEND AMS_BENCHMARKS_LIBRARIES adiak::adiak) + endif() add_subdirectory(benchmarks) endif() diff --git a/benchmarks/ams_bench_db.cpp b/benchmarks/ams_bench_db.cpp index 24dff972..58f0a7ce 100644 --- a/benchmarks/ams_bench_db.cpp +++ b/benchmarks/ams_bench_db.cpp @@ -1,12 +1,20 @@ #ifdef __AMS_ENABLE_MPI__ #include #endif -#include + +#ifdef __AMS_ENABLE_ADIAK__ +#include +#endif + +#ifdef __AMS_ENABLE_CALIPER__ +#include +#endif #include #include #include #include +#include #include #include @@ -14,36 +22,40 @@ #include #include "AMS.h" -// #include -// #include -// #include -// #include "wf/debug.h" - -// #include - - -void createUmpirePool(std::string parent_name, std::string pool_name) +void createUmpirePool(const std::string& parent_name, const std::string& pool_name) { auto &rm = umpire::ResourceManager::getInstance(); auto alloc_resource = rm.makeAllocator( pool_name, rm.getAllocator(parent_name)); } - -AMSDType getDataType(char *d_type) +AMSDType getDataType(const char* d_type) { AMSDType dType = AMSDType::AMS_DOUBLE; if (std::strcmp(d_type, "float") == 0) { dType = AMSDType::AMS_SINGLE; - } else if (std::strcmp(d_type, "double") == 0) { + } else if (d_type == "double") { dType = AMSDType::AMS_DOUBLE; } else { - assert(false && "Unknown data type"); + assert(false && "Unknown data type (must be 'float' or 'double')"); } return dType; } +AMSDBType getDBType(const char* db_type) +{ + AMSDBType dbType = AMSDBType::AMS_NONE; + if (std::strcmp(db_type, "csv") == 0) { + dbType = AMSDBType::AMS_CSV; + } else if (std::strcmp(db_type, "hdf5") == 0) { + dbType = AMSDBType::AMS_HDF5; + } else if (std::strcmp(db_type, "rmq") == 0) { + dbType = AMSDBType::AMS_RMQ; + } + return dbType; +} + template struct Problem { int num_inputs; @@ -73,14 +85,26 @@ struct Problem { return inputs; } + +/* +To move to CUDA + FPTypeValue *pPtr = + rm.allocate(num_elements, AMSResourceType::AMS_HOST); + rm.copy(outputs[i], AMS_DEVICE, pPtr, AMS_HOST, num_elements); +*/ + void ams_run(AMSExecutor &wf, AMSResourceType resource, int iterations, int num_elements) { + CALIPER(CALI_CXX_MARK_FUNCTION;) auto &rm = umpire::ResourceManager::getInstance(); + CALIPER(CALI_CXX_MARK_LOOP_BEGIN(mainloop_id, "mainloop");) + for (int i = 0; i < iterations; i++) { + CALIPER(CALI_CXX_MARK_LOOP_ITERATION(mainloop_id, i);) int elements = num_elements; // * ((DType)(rand()) / RAND_MAX) + 1; std::vector inputs; std::vector outputs; @@ -108,19 +132,18 @@ struct Problem { delete[] outputs[i]; outputs[i] = nullptr; } - - for (int i = 0; i < num_inputs; i++) { delete[] inputs[i]; inputs[i] = nullptr; } } + CALIPER(CALI_CXX_MARK_LOOP_END(mainloop_id);) } }; void callBackDouble(void *cls, long elements, void **inputs, void **outputs) { - std::cout << "Called the double precision model\n"; + // std::cout << "Called the double precision model\n"; static_cast *>(cls)->run(elements, (double **)(inputs), (double **)(outputs)); @@ -129,7 +152,7 @@ void callBackDouble(void *cls, long elements, void **inputs, void **outputs) void callBackSingle(void *cls, long elements, void **inputs, void **outputs) { - std::cout << "Called the single precision model\n"; + // std::cout << "Called the single precision model\n"; static_cast *>(cls)->run(elements, (float **)(inputs), (float **)(outputs)); @@ -138,118 +161,159 @@ void callBackSingle(void *cls, long elements, void **inputs, void **outputs) int main(int argc, char **argv) { - - if (argc != 7) { - std::cout << "Wrong cli\n"; - std::cout << argv[0] - << " use_device(0|1) num_inputs num_outputs " - "data_type(float|double) " - "num_iterations num_elements" << std::endl; - // return -1; + // Number of ranks in this run + int wS = 1; + // My Local Id + int rId = 0; + // Level of Threading provided by MPI + int provided = 0; + MPI_CALL(MPI_Init_thread(&argc, &argv, MPI_THREAD_SERIALIZED, &provided)); + MPI_CALL(MPI_Comm_size(MPI_COMM_WORLD, &wS)); + MPI_CALL(MPI_Comm_rank(MPI_COMM_WORLD, &rId)); + if (rId != 0) { + std::cout.setstate(std::ios::failbit); } - // // ------------------------------------------------------------------------- - // // setup command line parser - // // ------------------------------------------------------------------------- - // mfem::OptionsParser args(argc, argv); - // args.AddOption(&device_name, "-d", "--device", "Device config string"); - - // // data parameters - // args.AddOption(&num_elems, "-e", "--num-elems", "Number of elements"); - - // // random speed and packing - // args.AddOption(&seed, "-s", "--seed", "Seed for rand"); - - // args.AddOption(&db_config, - // "-db", - // "--dbconfig", - // "Path to directory where applications will store their data " - // "(or Path to JSON configuration if RabbitMQ is chosen)", - // reqDB); - - // args.AddOption(&db_type, - // "-dt", - // "--dbtype", - // "Configuration option of the different DB types:\n" - // "\t 'csv' Use csv as back end\n" - // "\t 'hdf5': use hdf5 as a back end\n" - // "\t 'rmq': use RabbitMQ as a back end\n"); - - // args.AddOption( - // &verbose, "-v", "--verbose", "-qu", "--quiet", "Print extra stuff"); - - // // ------------------------------------------------------------------------- - // // parse arguments - // // ------------------------------------------------------------------------- - // args.Parse(); - // if (!args.Good()) { - // args.PrintUsage(std::cout); - // return -1; - // } - - // if (rId == 0) { - // args.PrintOptions(std::cout); - // std::cout << std::endl; - // } - - - - // int use_device = std::atoi(argv[1]); - // int num_inputs = std::atoi(argv[2]); - // int num_outputs = std::atoi(argv[3]); - // AMSDType data_type = getDataType(argv[4]); - // int num_iterations = std::atoi(argv[5]); - // int avg_elements = std::atoi(argv[6]); - AMSResourceType resource = AMSResourceType::AMS_HOST; - srand(time(NULL)); + const char *device_name = "cpu"; + const char *db_config = ""; + const char *db_type = ""; + const char *precision_opt = "double"; - int num_inputs = 2; - int num_outputs = 4; - AMSDType data_type = getDataType("double"); + int seed = 0; + int num_elems = 1024; + int num_inputs = 8; + int num_outputs = 9; int num_iterations = 1; - int avg_elements = 10; + bool verbose = false; + bool reqDB = false; +#ifdef __ENABLE_DB__ + reqDB = true; +#endif - // Configure DB - auto device_name = "cpu"; - auto db_type = "rmq"; + // ------------------------------------------------------------------------- + // setup command line parser + // ------------------------------------------------------------------------- + mfem::OptionsParser args(argc, argv); + args.AddOption(&device_name, "-d", "--device", "Device config string (cpu or cuda)"); + + // set precision + args.AddOption(&precision_opt, + "-pr", + "--precision", + "Set precision (single or double)"); + + // data parameters + args.AddOption(&num_elems, "-e", "--num-elems", "Number of elements per iteration"); + args.AddOption(&num_inputs, "-di", "--dim-inputs", "Dimension of inputs"); + args.AddOption(&num_outputs, "-do", "--dim-outputs", "Dimension of outputs"); + args.AddOption(&num_iterations, "-i", "--num-iter", "Number of iterations"); + + // random speed and packing + args.AddOption(&seed, "-s", "--seed", "Seed for rand (default 0)"); + + args.AddOption(&db_config, + "-db", + "--dbconfig", + "Path to directory where applications will store their data (for CSV/HDF5)", + reqDB); + + args.AddOption(&db_type, + "-dt", + "--dbtype", + "Configuration option of the different DB types:\n" + "\t 'csv': use CSV as a back end\n" + "\t 'hdf5': use HDF5 as a back end\n" + "\t 'rmq': use RabbitMQ as a back end\n"); + + args.AddOption( + &verbose, "-v", "--verbose", "-qu", "--quiet", "Enable more verbose benchmark"); + + // ------------------------------------------------------------------------- + // parse arguments + // ------------------------------------------------------------------------- + args.Parse(); + if (!args.Good()) { + args.PrintUsage(std::cout); + return -1; + } - const bool use_device = std::strcmp(device_name, "cpu") != 0; - AMSDBType dbType = AMSDBType::AMS_NONE; - if (std::strcmp(db_type, "csv") == 0) { - dbType = AMSDBType::AMS_CSV; - } else if (std::strcmp(db_type, "hdf5") == 0) { - dbType = AMSDBType::AMS_HDF5; - } else if (std::strcmp(db_type, "rmq") == 0) { - dbType = AMSDBType::AMS_RMQ; + if (rId == 0) { + args.PrintOptions(std::cout); + std::cout << std::endl; } + srand(seed + rId); + + // ------------------------------------------------------------------------- + // Adiak + // ------------------------------------------------------------------------- +#ifdef __AMS_ENABLE_ADIAK__ + // add adiak init here + adiak::init(NULL); + + // replace with adiak::collect_all(); once adiak v0.4.0 + adiak::uid(); + adiak::launchdate(); + adiak::launchday(); + adiak::executable(); + adiak::executablepath(); + adiak::workdir(); + adiak::libraries(); + adiak::cmdline(); + adiak::hostname(); + adiak::clustername(); + adiak::walltime(); + adiak::systime(); + adiak::cputime(); + adiak::jobsize(); + adiak::hostlist(); + adiak::numhosts(); + adiak::value("compiler", std::string("@RAJAPERF_COMPILER@")); +#endif - char* db_config = "/g/g92/pottier1/ams/AMS/build_lassen_nompi/rmq-short.json"; - // AMSConfigureRMQDatabase(db_config); + AMSDType data_type = getDataType(precision_opt); + AMSDBType dbType = getDBType(db_type); - if (db_config == nullptr) dbType = AMSDBType::AMS_NONE; - if (dbType != AMSDBType::AMS_RMQ) { - AMSConfigureFSDatabase(dbType, db_config); + if (dbType == AMSDBType::AMS_NONE) { + std::cerr << "Error: no DB backend specified with --dbtype\n"; + return -1; } - // const char *db_path = (strlen(db_config) > 0) ? db_config : nullptr; - // std::ifstream json_file(db_config); - // nlohmann::json data = nlohmann::json::parse(json_file); - // parseDatabase(data); + const char *object_descr = std::getenv("AMS_OBJECTS"); + if (dbType == AMSDBType::AMS_RMQ && !object_descr) { + std::cerr << "Error: RabbitMQ backend required to set env variable AMS_OBJECTS\n"; + return -1; + } + if (dbType != AMSDBType::AMS_RMQ) { + AMSConfigureFSDatabase(dbType, db_config); + } - createUmpirePool("HOST", "TEST_HOST"); - AMSSetAllocator(AMSResourceType::AMS_HOST, "TEST_HOST"); + // ------------------------------------------------------------------------- + // AMS allocators setup + // ------------------------------------------------------------------------- + AMSResourceType resource = AMSResourceType::AMS_HOST; + const bool use_device = std::strcmp(device_name, "cpu") != 0; + if (use_device) { +#ifdef __ENABLE_CUDA__ + resource = AMSResourceType::AMS_DEVICE; +#else + std::cerr << "Error: Benchmark has not been compiled with CUDA support\n"; + return -1; +#endif + } - AMSCAbstrModel ams_model = AMSQueryModel("no_model"); + AMSCAbstrModel ams_model = AMSRegisterAbstractModel("bench_db_no_model", + AMSUQPolicy::AMS_RANDOM, + 0.5, + "", + "", + "bench_db_no_model", + 1); - // AMSCAbstrModel ams_model = AMSRegisterAbstractModel("bench_db_no_model", - // AMSUQPolicy::AMS_RANDOM, - // 0.5, - // "", - // "", - // "bench_db_no_model", - // 1); + std::cout << "Total elements across all " << wS << " ranks: " << wS * num_elems + << " (Weak Scaling)\n"; + std::cout << "Total elements per rank: " << num_elems << "\n"; if (data_type == AMSDType::AMS_SINGLE) { Problem prob(num_inputs, num_outputs); @@ -257,20 +321,24 @@ int main(int argc, char **argv) AMSDType::AMS_SINGLE, resource, (AMSPhysicFn)callBackSingle, - 0, - 1); + rId, + wS); - prob.ams_run(wf, resource, num_iterations, avg_elements); + prob.ams_run(wf, resource, num_iterations, num_elems); } else { Problem prob(num_inputs, num_outputs); AMSExecutor wf = AMSCreateExecutor(ams_model, AMSDType::AMS_DOUBLE, resource, (AMSPhysicFn)callBackDouble, - 0, - 1); - prob.ams_run(wf, resource, num_iterations, avg_elements); + rId, + wS); + prob.ams_run(wf, resource, num_iterations, num_elems); } +#ifdef __AMS_ENABLE_ADIAK__ + adiak::fini() +#endif + return 0; } diff --git a/src/AMSlib/AMS.cpp b/src/AMSlib/AMS.cpp index ef876eca..b3d2c61c 100644 --- a/src/AMSlib/AMS.cpp +++ b/src/AMSlib/AMS.cpp @@ -382,6 +382,11 @@ class AMSWrap getEntry(rmq_entry, "rabbitmq-routing-key"); bool update_surrogate = getEntry(entry, "update_surrogate"); + if(exchange == "") { + DBG(AMS, "Found empty RMQ exchange, deactivating model update") + update_surrogate = false; + } + auto &DB = ams::db::DBManager::getInstance(); DB.instantiate_rmq_db(port, host,