FillBoundary for Spherical Coordinates (#3286)

This adds a FillBoundary function for spherical coordinates (theta, phi, r). The function can also be used to implement other cases
AMReX-Codes · May 22, 2023 · b625d63 · b625d63
1 parent 1881aca
commit b625d63
Show file tree

Hide file tree

Showing 12 changed files with 629 additions and 76 deletions.
diff --git a/.clang-tidy b/.clang-tidy
@@ -1,6 +1,7 @@
 Checks: >
     '-*,
      bugprone-*,
+       -bugprone-branch-clone,
        -bugprone-easily-swappable-parameters,
        -bugprone-exception-escape,
        -bugprone-implicit-widening-of-multiplication-result,

diff --git a/Src/Base/AMReX_FabArrayBase.H b/Src/Base/AMReX_FabArrayBase.H
@@ -468,6 +468,9 @@ public:
         std::unique_ptr<MapOfCopyComTagContainers> m_RcvTags;
     };
 
+    void define_fb_metadata (CommMetaData& cmd, const IntVect& nghost, bool cross,
+                             const Periodicity& period, bool multi_ghost) const;
+
     //
     //! FillBoundary
     struct FB

diff --git a/Src/Base/AMReX_FabArrayBase.cpp b/Src/Base/AMReX_FabArrayBase.cpp
@@ -656,26 +656,26 @@ FabArrayBase::FB::FB (const FabArrayBase& fa, const IntVect& nghost,
 }
 
 void
-FabArrayBase::FB::define_fb (const FabArrayBase& fa)
+FabArrayBase::define_fb_metadata (CommMetaData& cmd, const IntVect& nghost,
+                                  bool cross, const Periodicity& period,
+                                  bool multi_ghost) const
 {
-    AMREX_ASSERT(m_multi_ghost ? fa.nGrow() >= 2 : true); // must have >= 2 ghost nodes
-    AMREX_ASSERT(m_multi_ghost ? !m_period.isAnyPeriodic() : true); // this only works for non-periodic
     const int                  MyProc   = ParallelDescriptor::MyProc();
-    const BoxArray&            ba       = fa.boxArray();
-    const DistributionMapping& dm       = fa.DistributionMap();
-    const Vector<int>&         imap     = fa.IndexArray();
+    const BoxArray&            ba       = this->boxArray();
+    const DistributionMapping& dm       = this->DistributionMap();
+    const Vector<int>&         imap     = this->IndexArray();
 
     // For local copy, all workers in the same team will have the identical copy of tags
     // so that they can share work.  But for remote communication, they are all different.
 
     const int nlocal = static_cast<int>(imap.size());
-    const IntVect& ng = m_ngrow;
-    const IntVect ng_ng = m_ngrow - 1;
+    const IntVect& ng = nghost;
+    const IntVect ng_ng =nghost - 1;
     std::vector< std::pair<int,Box> > isects;
 
-    const std::vector<IntVect>& pshifts = m_period.shiftIntVect();
+    const std::vector<IntVect>& pshifts = period.shiftIntVect();
 
-    auto& send_tags = *m_SndTags;
+    auto& send_tags = *cmd.m_SndTags;
 
     for (int i = 0; i < nlocal; ++i)
     {
@@ -697,7 +697,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
                     continue;  // local copy will be dealt with later
                 } else if (MyProc == dm[ksnd]) {
                     BoxList bl = amrex::boxDiff(bx, ba[krcv]);
-                    if (m_multi_ghost)
+                    if (multi_ghost)
                     {
                         // In the case where ngrow>1, augment the send/rcv box list
                         // with boxes for overlapping ghost nodes.
@@ -718,7 +718,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
         }
     }
 
-    auto& recv_tags = *m_RcvTags;
+    auto& recv_tags = *cmd.m_RcvTags;
 
     bool check_local = false, check_remote = false;
 #if defined(AMREX_USE_GPU)
@@ -735,8 +735,8 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
         check_local = true;
     }
 
-    m_threadsafe_loc = true;
-    m_threadsafe_rcv = true;
+    cmd.m_threadsafe_loc = true;
+    cmd.m_threadsafe_rcv = true;
     for (int i = 0; i < nlocal; ++i)
     {
         BoxList bl_local(ba.ixType());
@@ -759,7 +759,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
 
                 BoxList bl = amrex::boxDiff(dst_bx, vbx);
 
-                if (m_multi_ghost)
+                if (multi_ghost)
                 {
                     // In the case where ngrow>1, augment the send/rcv box list
                     // with boxes for overlapping ghost nodes.
@@ -779,7 +779,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
                         const BoxList tilelist(blbx, FabArrayBase::comm_tile_size);
                         for (auto const& it_tile : tilelist)
                         {
-                            m_LocTags->emplace_back(it_tile, it_tile+pit, krcv, ksnd);
+                            cmd.m_LocTags->emplace_back(it_tile, it_tile+pit, krcv, ksnd);
                         }
                         if (check_local) {
                             bl_local.push_back(blbx);
@@ -794,28 +794,28 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
             }
         }
 
-        if (m_threadsafe_loc) {
+        if (cmd.m_threadsafe_loc) {
             if ((bl_local.size() > 1)
                 && ! BoxArray(std::move(bl_local)).isDisjoint())
             {
-                m_threadsafe_loc = false;
+                cmd.m_threadsafe_loc = false;
                 check_local = false; // No need to check anymore
             }
         }
 
-        if (m_threadsafe_rcv) {
+        if (cmd.m_threadsafe_rcv) {
             if ((bl_remote.size() > 1)
                 && ! BoxArray(std::move(bl_remote)).isDisjoint())
             {
-                m_threadsafe_rcv = false;
+                cmd.m_threadsafe_rcv = false;
                 check_remote = false; // No need to check anymore
             }
         }
     }
 
     for (int ipass = 0; ipass < 2; ++ipass) // pass 0: send; pass 1: recv
     {
-        CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *m_SndTags : *m_RcvTags;
+        CopyComTag::MapOfCopyComTagContainers & Tags = (ipass == 0) ? *cmd.m_SndTags : *cmd.m_RcvTags;
 
         for (auto& kv : Tags)
         {
@@ -833,7 +833,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
                 const IntVect& d2s = tag.sbox.smallEnd() - tag.dbox.smallEnd();
 
                 std::vector<Box> boxes;
-                if (m_cross) {
+                if (cross) {
                     const Box& dstvbx = ba[tag.dstIndex];
                     for (int dir = 0; dir < AMREX_SPACEDIM; dir++)
                     {
@@ -861,7 +861,7 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
                 {
                     for (auto const& cross_box : boxes)
                     {
-                        if (m_cross)
+                        if (cross)
                         {
                             cctv_tags_cross.emplace_back(cross_box, cross_box+d2s,
                                                          tag.dstIndex, tag.srcIndex);
@@ -877,6 +877,15 @@ FabArrayBase::FB::define_fb (const FabArrayBase& fa)
     }
 }
 
+void
+FabArrayBase::FB::define_fb (const FabArrayBase& fa)
+{
+    AMREX_ASSERT(m_multi_ghost ? fa.nGrow() >= 2 : true); // must have >= 2 ghost nodes
+    AMREX_ASSERT(m_multi_ghost ? !m_period.isAnyPeriodic() : true); // this only works for non-periodic
+
+    fa.define_fb_metadata(*this, m_ngrow, m_cross, m_period, m_multi_ghost);
+}
+
 void
 FabArrayBase::FB::define_epo (const FabArrayBase& fa)
 {

diff --git a/Src/Base/AMReX_IntVect.H b/Src/Base/AMReX_IntVect.H
@@ -122,6 +122,9 @@ public:
     explicit IntVect (const Array<int,AMREX_SPACEDIM>& a) noexcept
         : vect{AMREX_D_DECL(a[0],a[1],a[2])} {}
 
+    explicit constexpr IntVect (Dim3 const& a) noexcept
+        : vect{AMREX_D_DECL(a.x,a.y,a.z)} {}
+
     // dtor, copy-ctor, copy-op=, move-ctor, and move-op= are compiler generated.
 
     AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE