#2281: Typeless RecursiveDoubling and general code refactor/cleanup

DARMA-tasking · Sep 13, 2024 · 90c5668 · 90c5668
1 parent 4746d13
commit 90c5668
Show file tree

Hide file tree

Showing 15 changed files with 870 additions and 674 deletions.
diff --git a/src/vt/collective/reduce/allreduce/helpers.h b/src/vt/collective/reduce/allreduce/helpers.h
@@ -41,14 +41,12 @@
 //@HEADER
 */
 
-#include "vt/configs/debug/debug_printconst.h"
 #if !defined INCLUDED_VT_COLLECTIVE_REDUCE_ALLREDUCE_HELPERS_H
 #define INCLUDED_VT_COLLECTIVE_REDUCE_ALLREDUCE_HELPERS_H
 
 #include "data_handler.h"
 #include "rabenseifner_msg.h"
 #include "vt/messaging/message/shared_message.h"
-
 #include <vector>
 #include <type_traits>
 
@@ -200,80 +198,5 @@ struct DataHelper<Scalar, Kokkos::View<Scalar*, Kokkos::HostSpace>> {
 
 #endif // MAGISTRATE_KOKKOS_ENABLED
 
-struct StateBase {
-  virtual ~StateBase() = default;
-  size_t size_ = {};
-
-  uint32_t local_col_wait_count_ = 0;
-  bool finished_adjustment_part_ = false;
-
-  int32_t mask_ = 1;
-  int32_t step_ = 0;
-  bool initialized_ = false;
-  bool completed_ = false;
-
-  // Scatter
-  int32_t scatter_mask_ = 1;
-  int32_t scatter_step_ = 0;
-  int32_t scatter_num_recv_ = 0;
-  std::vector<bool> scatter_steps_recv_ = {};
-  std::vector<bool> scatter_steps_reduced_ = {};
-
-  bool finished_scatter_part_ = false;
-
-  // Gather
-  int32_t gather_step_ = 0;
-  int32_t gather_mask_ = 1;
-  int32_t gather_num_recv_ = 0;
-  std::vector<bool> gather_steps_recv_ = {};
-  std::vector<bool> gather_steps_reduced_ = {};
-
-  std::vector<uint32_t> r_index_ = {};
-  std::vector<uint32_t> r_count_ = {};
-  std::vector<uint32_t> s_index_ = {};
-  std::vector<uint32_t> s_count_ = {};
-};
-
-template <typename Scalar, typename DataT>
-struct State : StateBase {
-  ~State() override {
-    left_adjust_message_ = nullptr;
-    right_adjust_message_ = nullptr;
-
-    for(auto& msg : scatter_messages_){
-      msg = nullptr;
-    }
-
-    for(auto& msg : gather_messages_){
-      msg = nullptr;
-    }
-  }
-
-  std::vector<Scalar> val_ = {};
-
-  MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>> left_adjust_message_ = nullptr;
-  MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>> right_adjust_message_ = nullptr;
-  std::vector<MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>>> scatter_messages_ = {};
-  std::vector<MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>>> gather_messages_ = {};
-
-  vt::pipe::callback::cbunion::CallbackTyped<DataT> final_handler_ = {};
-};
-
-#if MAGISTRATE_KOKKOS_ENABLED
-template <typename Scalar>
-struct State<Scalar, Kokkos::View<Scalar*, Kokkos::HostSpace>> : StateBase {
-  using DataT = Kokkos::View<Scalar*, Kokkos::HostSpace>;
-
-  Kokkos::View<Scalar*, Kokkos::HostSpace> val_ = {};
-
-  MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>> left_adjust_message_ = nullptr;
-  MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>> right_adjust_message_ = nullptr;
-  std::vector<MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>>> scatter_messages_ = {};
-  std::vector<MsgSharedPtr<RabenseifnerMsg<Scalar, DataT>>> gather_messages_ = {};
-
-  vt::pipe::callback::cbunion::CallbackTyped<DataT> final_handler_ = {};
-};
-#endif //MAGISTRATE_KOKKOS_ENABLED
-
 } // namespace vt::collective::reduce::allreduce
 #endif /*INCLUDED_VT_COLLECTIVE_REDUCE_ALLREDUCE_HELPERS_H*/
diff --git a/src/vt/collective/reduce/allreduce/rabenseifner.cc b/src/vt/collective/reduce/allreduce/rabenseifner.cc
@@ -2,7 +2,7 @@
 //@HEADER
 // *****************************************************************************
 //
-//                             rabenseifner.impl.h
+//                               rabenseifner.cc
 //                       DARMA/vt => Virtual Transport
 //
 // Copyright 2019-2021 National Technology & Engineering Solutions of Sandia, LLC
@@ -53,11 +53,11 @@ Rabenseifner::Rabenseifner(
     nodes_(theGroup()->GetGroupNodes(group.get())),
     num_nodes_(nodes_.size()),
     this_node_(theContext()->getNode()),
-    num_steps_(static_cast<int32_t>(log2(num_nodes_))),
+    num_steps_(static_cast<int32_t>(std::log2(num_nodes_))),
     nprocs_pof2_(1 << num_steps_),
     nprocs_rem_(num_nodes_ - nprocs_pof2_) {
 
-  auto const is_default_group = group.get() == default_group;
+  auto const is_default_group = theGroup()->isGroupDefault(group.get());
   if (not is_default_group) {
     auto it = std::find(nodes_.begin(), nodes_.end(), theContext()->getNode());
     vtAssert(it != nodes_.end(), "This node was not found in group nodes!");
@@ -97,9 +97,10 @@ Rabenseifner::Rabenseifner(
   for (auto& node : nodes_) {
     nodes_info += fmt::format("{} ", node);
   }
-  auto const is_default_group = group.get() == default_group;
+  auto const is_default_group = theGroup()->isGroupDefault(group_);
+  auto const in_group = theGroup()->inGroup(group_);
   auto const is_part_of_allreduce =
-    (not is_default_group and theGroup()->inGroup(group.get())) or
+    (not is_default_group and in_group) or
     is_default_group;
 
   vt_debug_print(
@@ -108,7 +109,7 @@ Rabenseifner::Rabenseifner(
     "Nodes:[{}]\n",
     is_default_group, is_part_of_allreduce, num_nodes_, nodes_info);
 
-  if (not is_default_group and theGroup()->inGroup(group.get())) {
+  if (not is_default_group and in_group) {
     auto it = std::find(nodes_.begin(), nodes_.end(), theContext()->getNode());
     vtAssert(it != nodes_.end(), "This node was not found in group nodes!");
 

diff --git a/src/vt/collective/reduce/allreduce/rabenseifner.h b/src/vt/collective/reduce/allreduce/rabenseifner.h
@@ -100,7 +100,6 @@ struct Rabenseifner {
 
   template <typename DataT>
   void initializeState(size_t id);
-  size_t generateNewId() { return id_++; }
 
   /**
    * \brief Execute the final handler callback with the reduced result.
@@ -289,8 +288,6 @@ struct Rabenseifner {
 
   size_t local_num_elems_ = {};
 
-  size_t id_ = 0;
-
   /// Sorted list of Nodes that take part in allreduce
   std::vector<NodeType> nodes_ = {};