From 6b7f72ed13efc3f271663e991ca3501d3ecc057a Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Wed, 5 Jul 2023 01:05:08 -0700 Subject: [PATCH 1/8] add more PFA annotations --- src/AMSlib/wf/redist_load.hpp | 6 ++++++ src/AMSlib/wf/workflow.hpp | 2 ++ 2 files changed, 8 insertions(+) diff --git a/src/AMSlib/wf/redist_load.hpp b/src/AMSlib/wf/redist_load.hpp index f7e461e8..2ce9b92d 100644 --- a/src/AMSlib/wf/redist_load.hpp +++ b/src/AMSlib/wf/redist_load.hpp @@ -114,6 +114,7 @@ class AMSLoadBalancer * It initializes the 'dataElements', 'displs' on the root node and the localLoad, balancedLoad * across all ranks. */ +PERFFASPECT() void init(int numIn, int numOut, AMSResourceType resource) { // We need to store information @@ -176,6 +177,7 @@ class AMSLoadBalancer /** @brief Computes the number of elements every rank will receive after balancing. * @returns the number of elements computed by this rank. **/ +PERFFASPECT() int computeBalanceLoad() { int rc = MPI_Bcast(&globalLoad, 1, MPI_INT, root, Comm); @@ -208,6 +210,7 @@ class AMSLoadBalancer * * \return void. */ +PERFFASPECT() void distribute(FPTypeValue *src, FPTypeValue *dest, FPTypeValue *buffer, @@ -253,6 +256,7 @@ class AMSLoadBalancer * * \return void. */ +PERFFASPECT() void distributeV(std::vector &src, std::vector &dest, int *gNElems, @@ -352,6 +356,7 @@ class AMSLoadBalancer * @param[out] outputs The vector to store all the output values gathered from their compute (remote) ranks. * @param[in] resource The location of the data (CPU|GPU) */ +PERFFASPECT() void gatherOutputs(std::vector &outputs, AMSResourceType resource) { @@ -381,6 +386,7 @@ class AMSLoadBalancer * @param[out] inputs The vector to load balance across all compute (remote) ranks. * @param[in] resource The location of the data (CPU|GPU) */ +PERFFASPECT() void scatterInputs(std::vector &inputs, AMSResourceType resource) { diff --git a/src/AMSlib/wf/workflow.hpp b/src/AMSlib/wf/workflow.hpp index a20b4aaa..8183722e 100644 --- a/src/AMSlib/wf/workflow.hpp +++ b/src/AMSlib/wf/workflow.hpp @@ -84,6 +84,7 @@ class AMSWorkflow * @param[in] outputs vector to 1-D vectors storing num_elements * items to be stored in the database */ +PERFFASPECT() void Store(size_t num_elements, std::vector &inputs, std::vector &outputs) @@ -241,6 +242,7 @@ class AMSWorkflow * This transformation can exploit the parallel nature of all the required * steps. */ +PERFFASPECT() void evaluate(void *probDescr, const int totalElements, const FPTypeValue **inputs, From 75a02751378fc106a866cbdf36314afbaaa81308 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 31 Jul 2023 12:47:43 -0700 Subject: [PATCH 2/8] update PFA build instructions --- INSTALL.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 02f3b036..58ed090a 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -66,16 +66,25 @@ To built AMS with [PFA](https://github.com/flux-framework/PerfFlowAspect) suppor ``` $ cd $CODE_ROOT/setup +$ source setup_env_with_pfa.sh $ mkdir build; cd build $ cmake \ - -DCMAKE_CXX_COMPILER=clang++ \ - -DCMAKE_C_COMPILER=clang \ + -DCMAKE_CXX_COMPILER=/usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ \ + -DCMAKE_C_COMPILER=/usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang \ + -DWITH_HDF5=On -DHDF5_Dir=$AMS_HDF5_PATH \ + -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ + -DWITH_DB=On \ + -DCMAKE_INSTALL_PREFIX=./install \ + -DCMAKE_BUILD_TYPE=Debug \ + -DWITH_EXAMPLES=On \ -DMFEM_DIR=$AMS_MFEM_PATH \ -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ + -DWITH_MPI=On \ -DWITH_CUDA=On \ -DWITH_CALIPER=On \ -DWITH_TORCH=On -DTorch_DIR=$AMS_TORCH_PATH \ -DWITH_FAISS=On -DFAISS_DIR=$AMS_FAISS_PATH \ + -DAMS_CUDA_ARCH=${AMS_CUDA_ARCH} \ -DWITH_PERFFLOWASPECT=On \ -Dperfflowaspect_DIR=$AMS_PFA_PATH/share \ ../ From 2427287de39a204d7c29cf3751b9513676649044 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Mon, 14 Aug 2023 17:03:17 -0700 Subject: [PATCH 3/8] add missing PFA annotations --- examples/app/utils_mfem.hpp | 2 ++ examples/main.cpp | 1 + 2 files changed, 3 insertions(+) diff --git a/examples/app/utils_mfem.hpp b/examples/app/utils_mfem.hpp index 74949b0a..f14908c4 100644 --- a/examples/app/utils_mfem.hpp +++ b/examples/app/utils_mfem.hpp @@ -42,6 +42,7 @@ using dt3 = mfem::DeviceTensor<3, T>; template +PERFFASPECT() static inline void pack_ij(const int k, const int sz_i, const int sz_sparse_j, @@ -63,6 +64,7 @@ static inline void pack_ij(const int k, } template +PERFFASPECT() static inline void unpack_ij(const int k, const int sz_i, const int sz_sparse_j, diff --git a/examples/main.cpp b/examples/main.cpp index 99f4d3e5..a5a72c9d 100644 --- a/examples/main.cpp +++ b/examples/main.cpp @@ -130,6 +130,7 @@ double unitrand() { return (double)rand() / RAND_MAX; } // TODO: we could to this on the device but need something more than `rand' template +PERFFASPECT() void random_init(mfem::Array &arr) { T *h_arr = arr.HostWrite(); From 33eda323c9c4a9e114a39ef4abfc9c388979f221 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Sat, 26 Aug 2023 00:34:15 -0700 Subject: [PATCH 4/8] remove PFA macro --- examples/app/utils_mfem.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/app/utils_mfem.hpp b/examples/app/utils_mfem.hpp index f14908c4..f7a410e4 100644 --- a/examples/app/utils_mfem.hpp +++ b/examples/app/utils_mfem.hpp @@ -42,7 +42,7 @@ using dt3 = mfem::DeviceTensor<3, T>; template -PERFFASPECT() +__attribute__((annotate("@critical_path(pointcut='around')"))) static inline void pack_ij(const int k, const int sz_i, const int sz_sparse_j, @@ -64,7 +64,7 @@ static inline void pack_ij(const int k, } template -PERFFASPECT() +__attribute__((annotate("@critical_path(pointcut='around')"))) static inline void unpack_ij(const int k, const int sz_i, const int sz_sparse_j, From 3561b466c173cb0c8b3845cfd665c4a58a15d7ad Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Thu, 23 Nov 2023 12:44:13 -0800 Subject: [PATCH 5/8] add missing PFA annotation --- src/AMSlib/wf/basedb.hpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/AMSlib/wf/basedb.hpp b/src/AMSlib/wf/basedb.hpp index 7f72107a..159758fd 100644 --- a/src/AMSlib/wf/basedb.hpp +++ b/src/AMSlib/wf/basedb.hpp @@ -643,6 +643,7 @@ class RedisDB : public BaseDB return connection_info; } + PERFFASPECT() void store(size_t num_elements, std::vector& inputs, std::vector& outputs) From 0dcbe7b87ae5dc82797160f04685ad32b4f25924 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Thu, 23 Nov 2023 12:52:01 -0800 Subject: [PATCH 6/8] remove setup script related code --- INSTALL.md | 1 - 1 file changed, 1 deletion(-) diff --git a/INSTALL.md b/INSTALL.md index 58ed090a..f008c659 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -66,7 +66,6 @@ To built AMS with [PFA](https://github.com/flux-framework/PerfFlowAspect) suppor ``` $ cd $CODE_ROOT/setup -$ source setup_env_with_pfa.sh $ mkdir build; cd build $ cmake \ -DCMAKE_CXX_COMPILER=/usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ \ From 1b9a233dfac3299c51df3beef8fc2a630e47eea6 Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Thu, 23 Nov 2023 12:54:36 -0800 Subject: [PATCH 7/8] remove hard-coded paths from install instructions --- INSTALL.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index f008c659..9bee47ad 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -68,8 +68,8 @@ To built AMS with [PFA](https://github.com/flux-framework/PerfFlowAspect) suppor $ cd $CODE_ROOT/setup $ mkdir build; cd build $ cmake \ - -DCMAKE_CXX_COMPILER=/usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang++ \ - -DCMAKE_C_COMPILER=/usr/tce/packages/clang/clang-10.0.1-gcc-8.3.1/bin/clang \ + -DCMAKE_CXX_COMPILER=clang++ \ + -DCMAKE_C_COMPILER=clang \ -DWITH_HDF5=On -DHDF5_Dir=$AMS_HDF5_PATH \ -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ -DWITH_DB=On \ From ed0581ec7945aa46dfa20a9dde2e51745d54f29d Mon Sep 17 00:00:00 2001 From: Aliza Lisan Date: Thu, 23 Nov 2023 15:45:46 -0800 Subject: [PATCH 8/8] update cmake command ams+pfa --- INSTALL.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/INSTALL.md b/INSTALL.md index 9bee47ad..61e6392f 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -72,9 +72,11 @@ $ cmake \ -DCMAKE_C_COMPILER=clang \ -DWITH_HDF5=On -DHDF5_Dir=$AMS_HDF5_PATH \ -DCMAKE_PREFIX_PATH=$INSTALL_DIR \ - -DWITH_DB=On \ + -DWITH_DB=On -DWITH_RMQ=On \ + -Damqpcpp_DIR=$AMS_AMQPCPP_PATH \ + -DBUILD_SHARED_LIBS=On \ -DCMAKE_INSTALL_PREFIX=./install \ - -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_BUILD_TYPE=Release \ -DWITH_EXAMPLES=On \ -DMFEM_DIR=$AMS_MFEM_PATH \ -DUMPIRE_DIR=$AMS_UMPIRE_PATH \ @@ -82,8 +84,10 @@ $ cmake \ -DWITH_CUDA=On \ -DWITH_CALIPER=On \ -DWITH_TORCH=On -DTorch_DIR=$AMS_TORCH_PATH \ + -DWITH_TESTS=Off \ -DWITH_FAISS=On -DFAISS_DIR=$AMS_FAISS_PATH \ -DAMS_CUDA_ARCH=${AMS_CUDA_ARCH} \ + -DWITH_AMS_DEBUG=On \ -DWITH_PERFFLOWASPECT=On \ -Dperfflowaspect_DIR=$AMS_PFA_PATH/share \ ../