Merge pull request #190 from lanl/blb/tracers

Add tracer particles to phoebus

inputs/blandford_mckee.pin

@@ -87,6 +87,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/blast_wave.pin

@@ -92,6 +92,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/blast_wave_2d.pin

@@ -93,6 +93,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/blast_wave_3d.pin

@@ -95,6 +95,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/bondi.pin

@@ -88,6 +88,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/friedmann.pin

@@ -79,6 +79,7 @@ Cv = 1.0
 <physics>
 hydro = true
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/homogeneous_sphere.pin

@@ -81,6 +81,7 @@ hydro = false
 he = false
 3t = false
 rad = true
+tracers = false
 
 <fluid>
 xorder = 2

inputs/homologous.pin

@@ -101,6 +101,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2
@@ -142,4 +143,4 @@ dtwarn_eps = 1e-5
 
 <progenitor>
 enabled=true
-tablepath=../Data/converted_ADM_homologouscollapse10000.dat 
+tablepath=../Data/converted_ADM_homologouscollapse10000.dat 

inputs/kh.pin

@@ -80,6 +80,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/leptoneq.pin

@@ -67,6 +67,7 @@ he = false
 3t = false
 rad = true
 mhd = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/linear_modes.pin

@@ -105,6 +105,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 mhd = true

inputs/linear_modes_3d.pin

@@ -103,6 +103,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 mhd = true

inputs/mhd_shocktube.pin

@@ -82,6 +82,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/p2c2p.pin

@@ -76,6 +76,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 mhd = true

inputs/progenitor.pin

@@ -113,6 +113,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/radiation_advection.pin

@@ -76,6 +76,11 @@ field = c.c.bulk.rho
 method = derivative_order_1
 max_level = 3
 
+# Uses this one for Boosted Minkowski
+<coordinates>
+vx = - 0.3 
+vy = 0.0
+
 <eos>
 type = IdealGas
 Gamma = 1.6666666666666666667
@@ -86,6 +91,7 @@ hydro = false
 he = false
 3t = false
 rad = true
+tracers = false
 
 <fluid>
 xorder = 2

inputs/radiation_equilibration.pin

@@ -84,6 +84,7 @@ hydro = true
 he = false
 3t = false
 rad = true
+tracers = false
 
 <fluid>
 xorder = 2

inputs/rhs_tester.pin

@@ -90,6 +90,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/rotor.pin

@@ -108,6 +108,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 mhd = true

inputs/shocktube.pin

@@ -85,6 +85,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/shocktube_nonrel.pin

@@ -82,6 +82,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/shocktube_stellarcollapse.pin

@@ -88,6 +88,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/shocktube_stellarcollapse_faketable.pin

@@ -94,6 +94,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/spherical_shock_tube.pin

@@ -120,6 +120,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/thincooling.pin

@@ -66,6 +66,7 @@ he = false
 3t = false
 rad = true
 mhd = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/torus.pin

@@ -78,10 +78,10 @@ num_threads = 1         # maximum number of OMP threads
 pack_size = -1
 
 # Commenting out meshblock sizes makes the simulation 1 meshblock
-# <parthenon/meshblock>
-# nx1 = 128
-# nx2 = 128
-# nx3 = 1
+#<parthenon/meshblock>
+#nx1 = 128
+#nx2 = 128
+#nx3 = 1
 
 <parthenon/refinement0>
 field = c.c.bulk.rho
@@ -104,6 +104,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/torus_cutout.pin

@@ -86,6 +86,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 bc_vars = primitive

inputs/torus_rad.pin

@@ -96,6 +96,7 @@ Cv = 1.0
 <physics>
 hydro = true
 rad = true
+tracers = false
 
 <fluid>
 xorder = 2

inputs/tov-static.pin

@@ -99,6 +99,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/tov.pin

@@ -101,6 +101,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

inputs/tov_cart.pin

@@ -113,6 +113,7 @@ hydro = true
 he = false
 3t = false
 rad = false
+tracers = false
 
 <fluid>
 xorder = 2

src/CMakeLists.txt

@@ -18,6 +18,7 @@ add_executable(phoebus
   fixup/fixup.hpp
   fixup/fixup.cpp
   fixup/fixup_c2p.cpp
+  fixup/fixup_particles.cpp
   fixup/fixup_radc2p.cpp
   fixup/fixup_src.cpp
   fixup/fixup_netfield.cpp
@@ -145,6 +146,9 @@ add_executable(phoebus
 
   tov/tov.hpp
   tov/tov.cpp
+
+  tracers/tracers.hpp
+  tracers/tracers.cpp
 )
 
 set(MAX_NUMBER_CONSERVED_VARS 9 CACHE INTEGER "Maximum number of conserved variables")

src/fixup/fixup.hpp

@@ -40,6 +40,7 @@ template <typename T>
 TaskStatus ConservedToPrimitiveFixup(T *rc);
 template <typename T>
 TaskStatus SourceFixup(T *rc);
+TaskStatus PurgeParticles(MeshBlockData<Real> *rc, const std::string swarmName);
 TaskStatus SumMdotPhiForNetFieldScaling(MeshData<Real> *md, const Real t, const int stage,
                                         std::vector<Real> *sums);
 TaskStatus ModifyNetField(MeshData<Real> *, const Real t, const Real dt, const int stage,

src/fixup/fixup_particles.cpp

@@ -0,0 +1,72 @@
+// © 2021-2023. Triad National Security, LLC. All rights reserved.
+// This program was produced under U.S. Government contract
+// 89233218CNA000001 for Los Alamos National Laboratory (LANL), which
+// is operated by Triad National Security, LLC for the U.S.
+// Department of Energy/National Nuclear Security Administration. All
+// rights in the program are reserved by Triad National Security, LLC,
+// and the U.S. Department of Energy/National Nuclear Security
+// Administration. The Government is granted for itself and others
+// acting on its behalf a nonexclusive, paid-up, irrevocable worldwide
+// license in this material to reproduce, prepare derivative works,
+// distribute copies to the public, perform publicly and display
+// publicly, and to permit others to do so.
+
+#include <cmath>
+
+#include "fixup.hpp"
+
+#include "geometry/analytic_system.hpp"
+#include "geometry/cached_system.hpp"
+#include "geometry/geometry.hpp"
+#include "geometry/geometry_defaults.hpp"
+#include "geometry/geometry_utils.hpp"
+#include "geometry/mckinney_gammie_ryan.hpp"
+#include "geometry/modified_system.hpp"
+#include "geometry/spherical_kerr_schild.hpp"
+#include "phoebus_utils/cell_locations.hpp"
+#include "phoebus_utils/linear_algebra.hpp"
+#include "phoebus_utils/variables.hpp"
+
+namespace fixup {
+
+/**
+ * PurgeParticles machinery to remove particles when criteria are met.
+ * Current criteria: particle falls into event horizon.
+ **/
+TaskStatus PurgeParticles(MeshBlockData<Real> *rc, const std::string swarmName) {
+
+  /* only do this when FMKS is used. */
+  if constexpr (std::is_same<PHOEBUS_GEOMETRY, Geometry::FMKS>::value) {
+    auto *pmb = rc->GetParentPointer();
+    auto &swarm = pmb->swarm_data.Get()->Get(swarmName);
+
+    auto &x = swarm->Get<Real>("x").Get();
+    auto &y = swarm->Get<Real>("y").Get();
+    auto &z = swarm->Get<Real>("z").Get();
+
+    auto swarm_d = swarm->GetDeviceContext();
+
+    auto &pars = pmb->packages.Get("geometry")->AllParams();
+    const Real Rh = pars.Get<Real>("Rh");
+    const Real xh = std::log(Rh);
+
+    const int max_active_index = swarm->GetMaxActiveIndex();
+    pmb->par_for(
+        "fixup::PurgeParticles", 0, max_active_index - 1, KOKKOS_LAMBDA(const int n) {
+          if (swarm_d.IsActive(n)) {
+
+            if (x(n) <= xh) {
+              swarm_d.MarkParticleForRemoval(n);
+            }
+
+            bool on_current_mesh_block = true;
+            swarm_d.GetNeighborBlockIndex(n, x(n), y(n), z(n), on_current_mesh_block);
+          }
+        }); // par for
+
+    swarm->RemoveMarkedParticles();
+  }
+  return TaskStatus::complete;
+} // PurgeParticles
+
+} // namespace fixup

src/main.cpp

@@ -51,6 +51,7 @@ int main(int argc, char *argv[]) {
   pman.app_input->InitMeshBlockUserData = Geometry::SetGeometryBlock;
   // pman.app_input->UserWorkAfterLoop = phoebus::UserWorkAfterLoop;
   // pman.app_input->SetFillDerivedFunctions = phoebus::SetFillDerivedFunctions;
+  pman.app_input->MeshBlockUserWorkBeforeOutput = phoebus::UserWorkBeforeOutput;
 
   phoebus::ProblemModifier(pman.pinput.get());