Black Holes: Port puncture tracking and enable tagging around the punctures

Examples/BinaryBH/BinaryBHLevel.cpp

@@ -11,11 +11,23 @@
 #include "PunctureTracker.hpp"
 // xxxxx #include "SixthOrderDerivatives.hpp"
 #include "Constraints.hpp"
+#include "PunctureTagger.hpp"
 #include "TraceARemoval.hpp"
 #include "TwoPuncturesInitialData.hpp"
 #include "Weyl4.hpp"
 #include "WeylExtraction.hpp"
 
+BHAMR<BinaryBHLevel::num_punctures> *BinaryBHLevel::get_bhamr_ptr()
+{
+    return dynamic_cast<BHAMR<num_punctures> *>(get_gramr_ptr());
+}
+
+PunctureTracker<BinaryBHLevel::num_punctures> &
+BinaryBHLevel::get_puncture_tracker()
+{
+    return get_bhamr_ptr()->get_puncture_tracker();
+}
+
 void BinaryBHLevel::variableSetUp()
 {
     BL_PROFILE("BinaryBHLevel::variableSetUp()");
@@ -100,6 +112,19 @@
                        });
 #endif
     amrex::Gpu::streamSynchronize();
+
+    if (simParams().puncture_tracking_enabled && Level() == 0)
+    {
+        // need to set the puncture coordinates as we use it for the puncture
+        // tagging
+        get_puncture_tracker().set_puncture_coords(
+            {simParams().bh1_params.center[0], simParams().bh1_params.center[1],
+             simParams().bh1_params.center[2], simParams().bh2_params.center[0],
+             simParams().bh2_params.center[1],
+             simParams().bh2_params.center[2]});
+        // can't call start_from_initial_punctures() because we need the full
+        // AMR grid first
+    }
 }
 
 // Calculate RHS during RK4 substeps
@@ -188,25 +213,89 @@
     BL_PROFILE("BinaryBHLevel::tag_cells()");
     amrex::MultiFab &state_new = get_new_data(State_Type);
 
-    if (simParams().track_punctures)
+    const auto &tag_arrs       = a_tag_box_array.arrays();
+    const auto &state_new_arrs = state_new.const_arrays();
+
+    ChiExtractionTagger chi_extraction_tagger(
+        Geom().CellSize(0), Level(), a_regrid_threshold,
+        simParams().extraction_params, simParams().activate_extraction);
+
+    const bool puncture_tracking_enabled =
+        simParams().puncture_tracking_enabled;
+    std::array<amrex::Real, AMREX_SPACEDIM * num_punctures> puncture_coords{};
+
+    if (puncture_tracking_enabled)
     {
-        amrex::Abort("BinaryBHLevel::tag_cells:track_punctures TODO");
+        puncture_coords = get_puncture_tracker().get_puncture_coords();
     }
 
-    const auto &tag_arrs       = a_tag_box_array.arrays();
-    const auto &state_new_arrs = state_new.const_arrays();
-    ChiExtractionTagger tagger(Geom().CellSize(0), Level(), a_regrid_threshold,
-                               simParams().extraction_params,
-                               simParams().activate_extraction);
-    amrex::ParallelFor(
-        state_new, amrex::IntVect(0),
-        [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k)
-        { tagger(i, j, k, tag_arrs[box_no], state_new_arrs[box_no]); });
+    // Even though we create this object, it won't be used if puncture tracking
+    // is not enabled.
+    PunctureTagger<num_punctures> puncture_tagger(
+        Geom().CellSize(0), Level(), get_gramr_ptr()->maxLevel(),
+        puncture_coords,
+        {simParams().bh1_params.mass, simParams().bh2_params.mass});
+
+    amrex::ParallelFor(state_new, amrex::IntVect(0),
+                       [=] AMREX_GPU_DEVICE(int box_no, int i, int j, int k)
+                       {
+                           chi_extraction_tagger(i, j, k, tag_arrs[box_no],
+                                                 state_new_arrs[box_no]);
+                           if (puncture_tracking_enabled)
+                           {
+                               puncture_tagger(i, j, k, tag_arrs[box_no]);
+                           }
+                       });
     amrex::Gpu::streamSynchronize();
 }
 
+void BinaryBHLevel::specific_post_init()
+{
+    BL_PROFILE("BinaryBHLevel::specific_post_init()");
+
+    if (simParams().puncture_tracking_enabled)
+    {
+        get_puncture_tracker().start_from_initial_punctures();
+    }
+}
+
+void BinaryBHLevel::specific_post_restart()
+{
+    BL_PROFILE("BinaryBHLevel::specific_post_restart()");
+
+    if (simParams().puncture_tracking_enabled)
+    {
+        std::string restart_checkpoint{};
+        GRParmParse pp("amr");
+        pp.get("restart", restart_checkpoint);
+        get_puncture_tracker().restart(restart_checkpoint);
+    }
+}
+
+void BinaryBHLevel::specificPostCheckpoint(const std::string &a_chk_dir,
+                                           std::ostream & /*a_os*/)
+{
+    if (simParams().puncture_tracking_enabled)
+    {
+        get_puncture_tracker().checkpoint(a_chk_dir);
+    }
+}
+
 void BinaryBHLevel::specificPostTimeStep()
 {
+    // do puncture tracking on requested level
+    if (simParams().puncture_tracking_enabled &&
+        Level() == simParams().puncture_tracking_level)
+    {
+        BL_PROFILE("PunctureTracking");
+        // only do the write out for every coarsest level timestep
+        // int coarsest_level = 0;
+        // bool write_punctures = at_level_timestep_multiple(coarsest_level);
+        bool write_punctures = true;
+        amrex::Real cur_time = get_state_data(State_Type).curTime();
+        amrex::Real dt       = get_gramr_ptr()->dtLevel(Level());
+        get_puncture_tracker().track(cur_time, dt, write_punctures);
+    }
 #if 0
 //xxxxx specificPostTimeStep
     BL_PROFILE("BinaryBHLevel::specificPostTimeStep");
@@ -269,15 +358,5 @@
         }
     }
 
-    // do puncture tracking on requested level
-    if (m_p.track_punctures && m_level == m_p.puncture_tracking_level)
-    {
-        BL_PROFILE("PunctureTracking");
-        // only do the write out for every coarsest level timestep
-        int coarsest_level = 0;
-        bool write_punctures = at_level_timestep_multiple(coarsest_level);
-        m_bh_amr.m_puncture_tracker.execute_tracking(m_time, m_restart_time,
-                                                     m_dt, write_punctures);
-    }
 #endif
 }

Examples/BinaryBH/BinaryBHLevel.hpp

@@ -19,6 +19,13 @@ class BinaryBHLevel : public GRAMRLevel
     // Inherit the contructors from GRAMRLevel
     using GRAMRLevel::GRAMRLevel;
 
+    static constexpr int num_punctures = 2;
+
+    BHAMR<num_punctures> *get_bhamr_ptr();
+
+    /// Get a reference to the PunctureTracker object stored by BHAMR
+    PunctureTracker<num_punctures> &get_puncture_tracker();
+
     /// Things to do at every full timestep
     ///(might include several substeps, e.g. in RK4)
     void specificAdvance() override;
@@ -42,6 +49,16 @@ class BinaryBHLevel : public GRAMRLevel
     /// Tag cells for regridding
     void tag_cells(amrex::TagBoxArray &a_tag_box_array,
                    amrex::Real a_regrid_threshold) final;
+
+    //! Things to do after a restart
+    void specific_post_restart() override;
+
+    //! Things to do after init
+    void specific_post_init() override;
+
+    //! Things to do after writing a checkpoint
+    void specificPostCheckpoint(const std::string &a_dir,
+                                std::ostream & /*a_os*/) override;
 };
 
 #endif /* BINARYBHLEVEL_HPP_ */

Examples/BinaryBH/GNUmakefile

@@ -2,6 +2,9 @@ GRTECLYN_HOME = $(realpath ../..)
 
 include $(GRTECLYN_HOME)/Tools/GNUMake/Make.defaults
 
+# We use particles for the puncture tracking
+USE_PARTICLES = TRUE
+
 AMREX_HOME ?= $(realpath ../../../amrex)
 
 include $(AMREX_HOME)/Tools/GNUMake/Make.defs
@@ -11,6 +14,7 @@ include $(AMREX_HOME)/Src/Base/Make.package
 include $(AMREX_HOME)/Src/Boundary/Make.package
 include $(AMREX_HOME)/Src/AmrCore/Make.package
 include $(AMREX_HOME)/Src/Amr/Make.package
+include $(AMREX_HOME)/Src/Particle/Make.package
 
 GRTECLYN_HOME = $(realpath ../..)
 GRTECLYN_SOURCE = $(GRTECLYN_HOME)/Source

Examples/BinaryBH/Main_BinaryBH.cpp

@@ -44,7 +44,7 @@ int runGRTeclyn(int /*argc*/, char * /*argv*/[])
     // Run TwoPunctures solver
     bh_amr.m_two_punctures.Run();
 #else
-    BHAMR bh_amr(&bh_level_bld);
+    BHAMR<2> bh_amr(&bh_level_bld);
 #endif
 
     bh_amr.init(0., sim_params.stop_time);

Examples/BinaryBH/SimulationParameters.hpp

@@ -37,8 +37,8 @@ class SimulationParameters : public SimulationParametersBase
     void read_shared_params(GRParmParse &pp)
     {
         // Do we want puncture tracking and constraint norm calculation?
-        pp.load("track_punctures", track_punctures, false);
-        pp.load("puncture_tracking_level", puncture_tracking_level, max_level);
+        pp.load("puncture_tracking.enabled", puncture_tracking_enabled, false);
+        pp.load("puncture_tracking.level", puncture_tracking_level, max_level);
         pp.load("calculate_constraint_norms", calculate_constraint_norms,
                 false);
     }
@@ -312,9 +312,9 @@ class SimulationParameters : public SimulationParametersBase
                         "must be between 0 and max_level (inclusive)");
     }
 
-    bool track_punctures{};
-    bool calculate_constraint_norms{};
+    bool puncture_tracking_enabled{};
     int puncture_tracking_level{};
+    bool calculate_constraint_norms{};
 
     // Collection of parameters necessary for initial conditions
     // Set these even in the case of TwoPunctures as they are used elsewhere

Examples/BinaryBH/inputs.test

@@ -135,8 +135,8 @@ covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse
 # coefficient for KO numerical dissipation
 sigma = 0.3
 
-track_punctures = 1
-puncture_tracking_level = 1
+puncture_tracking.enabled = 1
+puncture_tracking.level = 1
 
 # calculate_constraint_norms = 0
 
@@ -166,8 +166,6 @@ modes = 2 0 \
 #################################################
 
 output_path = foo
-track_punctures = 0
-puncture_tracking_level=0
 verbosity = 1
 amrex.fpe_trap_invalid=1
 amrex.fpe_trap_zero=1

Examples/BinaryBH/params.txt

@@ -137,8 +137,8 @@ covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse
 # coefficient for KO numerical dissipation
 sigma = 1.0
 
-track_punctures = 1
-puncture_tracking_level = 5
+puncture_tracking.enabled = 1
+puncture_tracking.level = 5
 
 # calculate_constraint_norms = 0
 

Examples/BinaryBH/params_test.txt

@@ -114,3 +114,9 @@ activate_extraction = true
 # num_points_theta = 37
 # num_modes = 8
 # modes = 2 0  2 1  2 2  4 0  4 1  4 2  4 3  4 4
+
+#################################################
+# Puncture Tracking parameters
+
+puncture_tracking.enabled = 1
+puncture_tracking.level = 0

Examples/BinaryBH/params_two_punctures.txt

@@ -133,8 +133,8 @@ covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse
 # coefficient for KO numerical dissipation
 sigma = 1.0
 
-track_punctures = 1
-puncture_tracking_level = 5
+puncture_tracking.enabled = 1
+puncture_tracking.level = 5
 
 # calculate_constraint_norms = 0
 

Examples/BinaryBH/params_very_cheap.txt

@@ -133,8 +133,8 @@ covariantZ4 = 1 # 0: keep kappa1; 1 [default]: replace kappa1 -> kappa1/lapse
 # coefficient for KO numerical dissipation
 sigma = 0.3
 
-track_punctures = 1
-puncture_tracking_level = 1
+puncture_tracking.enabled = 1
+puncture_tracking.level = 1
 
 # calculate_constraint_norms = 0
 

Source/BlackHoles/BHAMR.hpp

@@ -9,23 +9,35 @@
 #include "GRAMR.hpp"
 #include "PunctureTracker.hpp"
 
+#include <AMReX_ParmParse.H>
+
 /// A child of Chombo's AMR class to interface with tools which require
 /// access to the whole AMR hierarchy, and those of GRAMR
 /**
  * This object inherits from GRAMR and adds tools required for BH spacetimes
  */
-class BHAMR : public GRAMR
+
+template <int num_punctures> class BHAMR : public GRAMR
 {
+  private:
+    PunctureTracker<num_punctures> m_puncture_tracker;
+
   public:
-    PunctureTracker m_puncture_tracker;
+    BHAMR(amrex::LevelBld *a_levelbld) : GRAMR(a_levelbld)
+    {
+        amrex::ParmParse puncture_tracking_pp("puncture_tracking");
+        bool puncture_tracking_enabled = false; // default
 
-    BHAMR(amrex::LevelBld *a_levelbld) : GRAMR(a_levelbld) {}
+        puncture_tracking_pp.query("enabled", puncture_tracking_enabled);
+        if (puncture_tracking_enabled)
+        {
+            m_puncture_tracker.initialize(this);
+        }
+    }
 
-    void set_interpolator(
-        AMRInterpolator<Lagrange<4>> *a_interpolator) // xxxxx override
+    PunctureTracker<num_punctures> &get_puncture_tracker()
     {
-        // xxxxx   GRAMR::set_interpolator(a_interpolator);
-        m_puncture_tracker.set_interpolator(a_interpolator);
+        return m_puncture_tracker;
     }
 };
 

Source/BlackHoles/Make.package

@@ -1,2 +1 @@
-GRTECLYN_CEXE_sources += PunctureTracker.cpp
-GRTECLYN_CEXE_headers += BHAMR.hpp PunctureTracker.hpp
+GRTECLYN_CEXE_headers += BHAMR.hpp PunctureTracker.hpp PunctureTracker.impl.hpp