Update fillpatcher (#1126)

* update cmake for CI * Revert "update cmake for CI" This reverts commit a74d7ea. * update ERF_FillPatcher class and start to re-org the Exec directory
erf-model · Jun 27, 2023 · ba61a7d · ba61a7d
1 parent f2e3409
commit ba61a7d
Show file tree

Hide file tree

Showing 26 changed files with 51 additions and 21 deletions.
diff --git a/Docs/sphinx_doc/building.rst b/Docs/sphinx_doc/building.rst
@@ -104,7 +104,7 @@ or if using tcsh,
 
    The name of the resulting executable (generated by the GNUmake system) encodes several of the build characteristics, including dimensionality of the problem, compiler name, and whether MPI and/or OpenMP were linked with the executable.
    Thus, several different build configurations may coexist simultaneously in a problem folder.
-   For example, the default build in ``ERF/Exec/Isntropic`` will look
+   For example, the default build in ``ERF/Exec/IsntropicVortex`` will look
    like ``ERF3d.gnu.MPI.ex``, indicating that this is a 3-d version of the code, made with
    ``COMP=gnu``, and ``USE_MPI=TRUE``.
 
@@ -178,7 +178,6 @@ Analogous to GNU Make, the list of cmake directives is as follows:
    +---------------------------+------------------------------+------------------+-------------+
 
 
-
 Perlmutter (NERSC)
 ~~~~~~~~~~~~~~~~~~
 

diff --git a/Exec/CMakeLists.txt b/Exec/CMakeLists.txt
@@ -15,7 +15,7 @@ else ()
   add_subdirectory(EkmanSpiral_ideal)
   add_subdirectory(EkmanSpiral_input_sounding)
   add_subdirectory(IsentropicVortex)
-  add_subdirectory(MovingTerrain)
+  add_subdirectory(DevTests/MovingTerrain)
   add_subdirectory(PoiseuilleFlow)
   add_subdirectory(ScalarAdvDiff)
   add_subdirectory(TaylorGreenVortex)

diff --git a/Exec/MetGrid/GNUmakefile → Exec/DevTests/MetGrid/GNUmakefile b/Exec/MetGrid/GNUmakefile → Exec/DevTests/MetGrid/GNUmakefile
@@ -30,6 +30,6 @@ USE_NETCDF = TRUE
 # GNU Make
 Bpack := ./Make.package
 Blocs := .
-ERF_HOME := ../..
-ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/MetGrid
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/DevTests/MetGrid
 include $(ERF_HOME)/Exec/Make.ERF
diff --git a/Exec/MetGrid/Make.package → Exec/DevTests/MetGrid/Make.package b/Exec/MetGrid/Make.package → Exec/DevTests/MetGrid/Make.package
diff --git a/Exec/MetGrid/inputs → Exec/DevTests/MetGrid/inputs b/Exec/MetGrid/inputs → Exec/DevTests/MetGrid/inputs
diff --git a/Exec/MetGrid/prob.H → Exec/DevTests/MetGrid/prob.H b/Exec/MetGrid/prob.H → Exec/DevTests/MetGrid/prob.H
diff --git a/Exec/MetGrid/prob.cpp → Exec/DevTests/MetGrid/prob.cpp b/Exec/MetGrid/prob.cpp → Exec/DevTests/MetGrid/prob.cpp
diff --git a/Exec/MovingTerrain/CMakeLists.txt → Exec/DevTests/MovingTerrain/CMakeLists.txt b/Exec/MovingTerrain/CMakeLists.txt → Exec/DevTests/MovingTerrain/CMakeLists.txt
diff --git a/Exec/MovingTerrain/GNUmakefile → Exec/DevTests/MovingTerrain/GNUmakefile b/Exec/MovingTerrain/GNUmakefile → Exec/DevTests/MovingTerrain/GNUmakefile
@@ -30,6 +30,6 @@ USE_ASSERTION = TRUE
 # GNU Make
 Bpack := ./Make.package
 Blocs := .
-ERF_HOME := ../..
-ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/MovingTerrain
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/DevTests/MovingTerrain
 include $(ERF_HOME)/Exec/Make.ERF
diff --git a/Exec/MovingTerrain/Make.package → Exec/DevTests/MovingTerrain/Make.package b/Exec/MovingTerrain/Make.package → Exec/DevTests/MovingTerrain/Make.package
diff --git a/Exec/MovingTerrain/inputs → Exec/DevTests/MovingTerrain/inputs b/Exec/MovingTerrain/inputs → Exec/DevTests/MovingTerrain/inputs
diff --git a/Exec/MovingTerrain/prob.H → Exec/DevTests/MovingTerrain/prob.H b/Exec/MovingTerrain/prob.H → Exec/DevTests/MovingTerrain/prob.H
diff --git a/Exec/MovingTerrain/prob.cpp → Exec/DevTests/MovingTerrain/prob.cpp b/Exec/MovingTerrain/prob.cpp → Exec/DevTests/MovingTerrain/prob.cpp
diff --git a/Exec/MultiBlock/CMakeLists.txt → Exec/DevTests/MultiBlock/CMakeLists.txt b/Exec/MultiBlock/CMakeLists.txt → Exec/DevTests/MultiBlock/CMakeLists.txt
diff --git a/Exec/MultiBlock/GNUmakefile → Exec/DevTests/MultiBlock/GNUmakefile b/Exec/MultiBlock/GNUmakefile → Exec/DevTests/MultiBlock/GNUmakefile
@@ -30,6 +30,6 @@ USE_MULTIBLOCK = TRUE
 # GNU Make
 Bpack := ./Make.package
 Blocs := .
-ERF_HOME := ../..
-ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/MultiBlock
+ERF_HOME := ../../..
+ERF_PROBLEM_DIR = $(ERF_HOME)/Exec/DevTests/MultiBlock
 include $(ERF_HOME)/Exec/Make.ERF
diff --git a/Exec/MultiBlock/Make.package → Exec/DevTests/MultiBlock/Make.package b/Exec/MultiBlock/Make.package → Exec/DevTests/MultiBlock/Make.package
diff --git a/Exec/MultiBlock/inputs_amrex → Exec/DevTests/MultiBlock/inputs_amrex b/Exec/MultiBlock/inputs_amrex → Exec/DevTests/MultiBlock/inputs_amrex
diff --git a/Exec/MultiBlock/inputs_crse_periodic → .../DevTests/MultiBlock/inputs_crse_periodic b/Exec/MultiBlock/inputs_crse_periodic → .../DevTests/MultiBlock/inputs_crse_periodic
diff --git a/Exec/MultiBlock/inputs_erf1 → Exec/DevTests/MultiBlock/inputs_erf1 b/Exec/MultiBlock/inputs_erf1 → Exec/DevTests/MultiBlock/inputs_erf1
diff --git a/Exec/MultiBlock/inputs_erf2 → Exec/DevTests/MultiBlock/inputs_erf2 b/Exec/MultiBlock/inputs_erf2 → Exec/DevTests/MultiBlock/inputs_erf2
diff --git a/Exec/MultiBlock/prob.H → Exec/DevTests/MultiBlock/prob.H b/Exec/MultiBlock/prob.H → Exec/DevTests/MultiBlock/prob.H
diff --git a/Exec/MultiBlock/prob.cpp → Exec/DevTests/MultiBlock/prob.cpp b/Exec/MultiBlock/prob.cpp → Exec/DevTests/MultiBlock/prob.cpp
diff --git a/Source/BoundaryConditions/ERF_FillPatcher.H b/Source/BoundaryConditions/ERF_FillPatcher.H
@@ -17,8 +17,8 @@ public:
                              amrex::Vector<amrex::Real> const& crse_time);
 
     template <typename BC>
-    void fillCoarseFineBoundary (amrex::MultiFab& mf, amrex::Real time,
-                                 BC& cbc, amrex::Vector<amrex::BCRec> const& bcs);
+    void fill(amrex::MultiFab& mf, amrex::Real time,
+              BC& cbc, amrex::Vector<amrex::BCRec> const& bcs);
 
 private:
 
@@ -34,6 +34,8 @@ private:
     amrex::IntVect m_ratio;
     amrex::Vector<amrex::MultiFab> m_cf_crse_data;
     amrex::MultiFab m_cf_fine_data;
+    amrex::Vector<amrex::Real> m_crse_times;
+    amrex::Real m_dt_crse;
 };
 
 /*
@@ -46,13 +48,28 @@ private:
  */
 template <typename BC>
 void
-ERFFillPatcher::fillCoarseFineBoundary (amrex::MultiFab& mf, amrex::Real time,
-                                        BC& cbc, amrex::Vector<amrex::BCRec> const& bcs)
+ERFFillPatcher::fill (amrex::MultiFab& mf, amrex::Real time,
+                      BC& cbc, amrex::Vector<amrex::BCRec> const& bcs)
 {
     cbc(m_cf_crse_data[0], 0, m_ncomp, amrex::IntVect(0), time, 0);
-    amrex::FillPatchInterp(m_cf_fine_data, 0, m_cf_crse_data[0], 0, m_ncomp, amrex::IntVect(0),
+
+    amrex::MultiFab crse_data_time_interp(m_cf_crse_data[0].boxArray(), m_cf_crse_data[0].DistributionMap(),
+                                          m_ncomp, amrex::IntVect{0});
+
+    amrex::Real fac_new = (time - m_crse_times[0]) / m_dt_crse;
+    amrex::Real fac_old = 1.0 - fac_new;
+
+    // Time interpolate the coarse data
+    amrex::MultiFab::LinComb(crse_data_time_interp,fac_old, m_cf_crse_data[0], 0,
+                                                   fac_new, m_cf_crse_data[1], 0,
+                             0, m_ncomp, 0);
+
+
+    // Spatially interpolate the time-interpolated coarse data
+    amrex::FillPatchInterp(m_cf_fine_data, 0, crse_data_time_interp, 0, m_ncomp, amrex::IntVect(0),
                            m_cgeom, m_fgeom, m_fgeom.Domain(), m_ratio, m_interp, bcs, 0);
+
+    // Copy the spatially and temporally interpolated data back into the mf
     mf.ParallelCopy(m_cf_fine_data, 0, 0, m_ncomp, amrex::IntVect{0}, mf.nGrowVect());
 }
-
 #endif
diff --git a/Source/BoundaryConditions/ERF_FillPatcher.cpp b/Source/BoundaryConditions/ERF_FillPatcher.cpp
@@ -57,7 +57,12 @@ ERFFillPatcher::ERFFillPatcher (BoxArray const& fba, DistributionMapping  fdm,
 
     BoxArray cf_fba(std::move(bl));
     DistributionMapping cf_dm(cf_fba);
+
+    // This will be used as a temporary to hold the time-interpolated state
     m_cf_fine_data.define(cf_fba, cf_dm, m_ncomp, 0);
+
+    // These will hold the coarse data on the m_cf grids
+    m_cf_crse_data.emplace_back(BoxArray(std::move(cbl)), cf_dm, m_ncomp, 0);
     m_cf_crse_data.emplace_back(BoxArray(std::move(cbl)), cf_dm, m_ncomp, 0);
 }
 
@@ -70,8 +75,17 @@ ERFFillPatcher::ERFFillPatcher (BoxArray const& fba, DistributionMapping  fdm,
  */
 
 void ERFFillPatcher::registerCoarseData (Vector<MultiFab const*> const& crse_data,
-                                         Vector<amrex::Real> const& /*crse_time*/)
+                                         Vector<Real> const& crse_time)
 {
-    AMREX_ALWAYS_ASSERT(crse_data.size() == 1);
-    m_cf_crse_data[0].ParallelCopy(*crse_data[0], m_cgeom.periodicity());
+    AMREX_ALWAYS_ASSERT(crse_data.size() == 2); // old and new
+    AMREX_ALWAYS_ASSERT(crse_time[0] != crse_time[1]);
+
+    m_cf_crse_data[0].ParallelCopy(*crse_data[0], m_cgeom.periodicity()); // old data
+    m_cf_crse_data[1].ParallelCopy(*crse_data[1], m_cgeom.periodicity()); // new data
+
+    m_crse_times.resize(2);
+    m_crse_times[0] = crse_time[0]; // time of "old" coarse data
+    m_crse_times[1] = crse_time[1]; // time of "new" coarse data
+
+    m_dt_crse = crse_time[1] - crse_time[0];
 }
diff --git a/Submodules/AMReX b/Submodules/AMReX
diff --git a/Tests/CTestList.cmake b/Tests/CTestList.cmake
@@ -101,8 +101,8 @@ add_test_r(DensityCurrent_detJ2_MT           "DensityCurrent/density_current" "p
 add_test_r(EkmanSpiral                       "EkmanSpiral_custom/ekman_spiral_custom" "plt00010")
 add_test_r(IsentropicVortexStationary       "IsentropicVortex/erf_isentropic_vortex" "plt00010")
 add_test_r(IsentropicVortexAdvecting        "IsentropicVortex/erf_isentropic_vortex" "plt00010")
-add_test_r(MovingTerrain_nosub               "MovingTerrain/moving_terrain"   "plt00020")
-add_test_r(MovingTerrain_sub                 "MovingTerrain/moving_terrain"   "plt00010")
+add_test_r(MovingTerrain_nosub               "DevTests/MovingTerrain/moving_terrain"   "plt00020")
+add_test_r(MovingTerrain_sub                 "DevTests/MovingTerrain/moving_terrain"   "plt00010")
 add_test_r(PoiseuilleFlow                    "PoiseuilleFlow/erf_poiseuille_flow" "plt00010")
 add_test_r(RayleighDamping                  "ScalarAdvDiff/erf_scalar_advdiff" "plt00100")
 add_test_r(ScalarAdvectionUniformU          "ScalarAdvDiff/erf_scalar_advdiff" "plt00020")