EB: Cleanup & Simplify (#5223)

* Review Comments by Remi

Co-authored-by: Remi Lehe <[email protected]>

* WarpX Class: Remove `m_eb_enabled`

Remove double book-keeping.

* Review comment from Edo

* Fix compilation errors

---------

Co-authored-by: Remi Lehe <[email protected]>

.github/workflows/source/check_inputs.py

@@ -27,7 +27,7 @@
                     # skip lines related to other function arguments
                     # NOTE: update range call to reflect changes
                     #       in the interface of 'add_warpx_test'
-                    for _ in range(2):  # skip over: dims, numprocs
+                    for _ in range(2):  # skip over: dims, nprocs
                         next(f)
                     # strip leading whitespaces, remove end-of-line comments
                     testinput = next(f).lstrip().split(" ")[0]

Source/BoundaryConditions/WarpXEvolvePML.cpp

@@ -11,6 +11,7 @@
 #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT)
 #   include "BoundaryConditions/PML_RZ.H"
 #endif
+#include "EmbeddedBoundary/Enabled.H"
 #include "PML_current.H"
 #include "Utils/WarpXProfilerWrapper.H"
 #include "WarpX_PML_kernels.H"
@@ -269,15 +270,16 @@ WarpX::DampJPML (int lev, PatchType patch_type)
             const Real* sigma_star_cumsum_fac_j_z = sigba[mfi].sigma_star_cumsum_fac[1].data();
 #endif
 
-            amrex::Array4<amrex::Real> pml_lxfab, pml_lyfab, pml_lzfab;
-            if (m_eb_enabled) {
+            // Skip the field update if this gridpoint is inside the embedded boundary
+            amrex::Array4<amrex::Real> eb_lxfab, eb_lyfab, eb_lzfab;
+            if (EB::enabled()) {
                 const auto &pml_edge_lenghts = pml[lev]->Get_edge_lengths();
 
-                pml_lxfab = pml_edge_lenghts[0]->array(mfi);
-                pml_lyfab = pml_edge_lenghts[1]->array(mfi);
-                pml_lzfab = pml_edge_lenghts[2]->array(mfi);
+                eb_lxfab = pml_edge_lenghts[0]->array(mfi);
+                eb_lyfab = pml_edge_lenghts[1]->array(mfi);
+                eb_lzfab = pml_edge_lenghts[2]->array(mfi);
             } else {
-                amrex::ignore_unused(pml_lxfab, pml_lyfab, pml_lzfab);
+                amrex::ignore_unused(eb_lxfab, eb_lyfab, eb_lzfab);
             }
 
             const Box& tjx  = mfi.tilebox( pml_j[0]->ixType().toIntVect() );
@@ -304,21 +306,21 @@ WarpX::DampJPML (int lev, PatchType patch_type)
 
             amrex::ParallelFor( tjx, tjy, tjz,
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) {
-                    if (pml_lxfab && pml_lxfab(i, j, k) <= 0) { return; }
+                    if (eb_lxfab && eb_lxfab(i, j, k) <= 0) { return; }
 
                     damp_jx_pml(i, j, k, pml_jxfab, sigma_star_cumsum_fac_j_x,
                                 sigma_cumsum_fac_j_y, sigma_cumsum_fac_j_z,
                                 xs_lo,y_lo, z_lo);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) {
-                    if (pml_lyfab && pml_lyfab(i, j, k) <= 0) { return; }
+                    if (eb_lyfab && eb_lyfab(i, j, k) <= 0) { return; }
 
                     damp_jy_pml(i, j, k, pml_jyfab, sigma_cumsum_fac_j_x,
                                 sigma_star_cumsum_fac_j_y, sigma_cumsum_fac_j_z,
                                 x_lo,ys_lo, z_lo);
                 },
                 [=] AMREX_GPU_DEVICE (int i, int j, int k) {
-                    if (pml_lzfab && pml_lzfab(i, j, k)<=0) { return; }
+                    if (eb_lzfab && eb_lzfab(i, j, k) <= 0) { return; }
 
                     damp_jz_pml(i, j, k, pml_jzfab, sigma_cumsum_fac_j_x,
                                 sigma_cumsum_fac_j_y, sigma_star_cumsum_fac_j_z,

Source/Diagnostics/ReducedDiags/ChargeOnEB.cpp

@@ -95,7 +95,7 @@ void ChargeOnEB::ComputeDiags (const int step)
     if (!m_intervals.contains(step+1)) { return; }
 
     if (!EB::enabled()) {
-        throw std::runtime_error("ComputeDiags only works when EBs are enabled at runtime");
+        throw std::runtime_error("ChargeOnEB::ComputeDiags only works when EBs are enabled at runtime");
     }
 #if ((defined WARPX_DIM_3D) && (defined AMREX_USE_EB))
     // get a reference to WarpX instance

Source/Diagnostics/WarpXIO.cpp

@@ -11,6 +11,7 @@
 #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT)
 #    include "BoundaryConditions/PML_RZ.H"
 #endif
+#include "EmbeddedBoundary/Enabled.H"
 #include "FieldIO.H"
 #include "Particles/MultiParticleContainer.H"
 #include "Utils/TextMsg.H"
@@ -393,7 +394,7 @@ WarpX::InitFromCheckpoint ()
         }
     }
 
-    if (m_eb_enabled) { InitializeEBGridData(maxLevel()); }
+    if (EB::enabled()) { InitializeEBGridData(maxLevel()); }
 
     // Initialize particles
     mypc->AllocData();

Source/Evolve/WarpXEvolve.cpp

@@ -13,6 +13,7 @@
 #include "BoundaryConditions/PML.H"
 #include "Diagnostics/MultiDiagnostics.H"
 #include "Diagnostics/ReducedDiags/MultiReducedDiags.H"
+#include "EmbeddedBoundary/Enabled.H"
 #include "Evolve/WarpXDtType.H"
 #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H"
 #ifdef WARPX_USE_FFT
@@ -525,7 +526,7 @@ void WarpX::HandleParticlesAtBoundaries (int step, amrex::Real cur_time, int num
     }
 
     // interact the particles with EB walls (if present)
-    if (m_eb_enabled) {
+    if (EB::enabled()) {
         mypc->ScrapeParticlesAtEB(amrex::GetVecOfConstPtrs(m_distance_to_eb));
         m_particle_boundary_buffer->gatherParticlesFromEmbeddedBoundaries(
             *mypc, amrex::GetVecOfConstPtrs(m_distance_to_eb));

Source/FieldSolver/ElectrostaticSolver.cpp

@@ -7,6 +7,7 @@
 #include "WarpX.H"
 
 #include "FieldSolver/ElectrostaticSolver.H"
+#include "EmbeddedBoundary/Enabled.H"
 #include "Fluids/MultiFluidContainer.H"
 #include "Fluids/WarpXFluidContainer.H"
 #include "Parallelization/GuardCellManager.H"
@@ -289,7 +290,7 @@ WarpX::AddSpaceChargeFieldLabFrame ()
 
     // Compute the electric field. Note that if an EB is used the electric
     // field will be calculated in the computePhi call.
-    if (!m_eb_enabled) { computeE( Efield_fp, phi_fp, beta ); }
+    if (!EB::enabled()) { computeE( Efield_fp, phi_fp, beta ); }
     else {
         if (IsPythonCallbackInstalled("poissonsolver")) { computeE(Efield_fp, phi_fp, beta); }
     }
@@ -333,12 +334,11 @@ WarpX::computePhi (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho,
 
     std::optional<ElectrostaticSolver::EBCalcEfromPhiPerLevel> post_phi_calculation;
 #ifdef AMREX_USE_EB
-    // TODO: double check no overhead occurs on "m_eb_enabled == false"
     std::optional<amrex::Vector<amrex::EBFArrayBoxFactory const *> > eb_farray_box_factory;
 #else
     std::optional<amrex::Vector<amrex::FArrayBoxFactory const *> > const eb_farray_box_factory;
 #endif
-    if (m_eb_enabled)
+    if (EB::enabled())
     {
         // EB: use AMReX to directly calculate the electric field since with EB's the
         // simple finite difference scheme in WarpX::computeE sometimes fails
@@ -400,7 +400,7 @@ WarpX::computePhi (const amrex::Vector<std::unique_ptr<amrex::MultiFab> >& rho,
         WarpX::grid_type,
         this->m_poisson_boundary_handler,
         is_solver_igf_on_lev0,
-        m_eb_enabled,
+        EB::enabled(),
         WarpX::do_single_precision_comms,
         this->ref_ratio,
         post_phi_calculation,

Source/FieldSolver/FiniteDifferenceSolver/MacroscopicEvolveE.cpp

@@ -140,14 +140,14 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian (
         Array4<Real> const& jy = Jfield[1]->array(mfi);
         Array4<Real> const& jz = Jfield[2]->array(mfi);
 
-        amrex::Array4<amrex::Real> lx, ly, lz;
+        amrex::Array4<amrex::Real> eb_lx, eb_ly, eb_lz;
         if (EB::enabled()) {
-            lx = edge_lengths[0]->array(mfi);
-            ly = edge_lengths[1]->array(mfi);
-            lz = edge_lengths[2]->array(mfi);
+            eb_lx = edge_lengths[0]->array(mfi);
+            eb_ly = edge_lengths[1]->array(mfi);
+            eb_lz = edge_lengths[2]->array(mfi);
         }
 #ifdef WARPX_DIM_XZ
-        amrex::ignore_unused(ly);
+        amrex::ignore_unused(eb_ly);
 #endif
 
         // material prop //
@@ -180,7 +180,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian (
         amrex::ParallelFor(tex, tey, tez,
             [=] AMREX_GPU_DEVICE (int i, int j, int k){
                 // Skip field push if this cell is fully covered by embedded boundaries
-                if (lx && lx(i, j, k) <= 0) { return; }
+                if (eb_lx && eb_lx(i, j, k) <= 0) { return; }
 
                 // Interpolate conductivity, sigma, to Ex position on the grid
                 amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag,
@@ -198,10 +198,10 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian (
 
             [=] AMREX_GPU_DEVICE (int i, int j, int k){
 #ifdef WARPX_DIM_3D
-                if (ly && ly(i,j,k) <= 0) { return; }
+                if (eb_ly && eb_ly(i,j,k) <= 0) { return; }
 #elif defined(WARPX_DIM_XZ)
                 //In XZ Ey is associated with a mesh node, so we need to check if the mesh node is covered
-                if (lx && (lx(i, j, k)<=0 || lx(i-1, j, k)<=0 || lz(i, j, k)<=0 || lz(i, j-1, k)<=0)) { return; }
+                if (eb_lx && (eb_lx(i, j, k)<=0 || eb_lx(i-1, j, k)<=0 || eb_lz(i, j, k)<=0 || eb_lz(i, j-1, k)<=0)) { return; }
 #endif
 
                 // Interpolate conductivity, sigma, to Ey position on the grid
@@ -221,7 +221,7 @@ void FiniteDifferenceSolver::MacroscopicEvolveECartesian (
 
             [=] AMREX_GPU_DEVICE (int i, int j, int k){
                 // Skip field push if this cell is fully covered by embedded boundaries
-                if (lz && lz(i,j,k) <= 0) { return; }
+                if (eb_lz && eb_lz(i, j, k) <= 0) { return; }
 
                 // Interpolate conductivity, sigma, to Ez position on the grid
                 amrex::Real const sigma_interp = ablastr::coarsen::sample::Interp(sigma_arr, sigma_stag,

Source/FieldSolver/MagnetostaticSolver/MagnetostaticSolver.cpp

@@ -7,6 +7,7 @@
 #include "WarpX.H"
 
 #include "FieldSolver/MagnetostaticSolver/MagnetostaticSolver.H"
+#include "EmbeddedBoundary/Enabled.H"
 #include "Parallelization/GuardCellManager.H"
 #include "Particles/MultiParticleContainer.H"
 #include "Particles/WarpXParticleContainer.H"
@@ -184,7 +185,7 @@ WarpX::computeVectorPotential (const amrex::Vector<amrex::Array<std::unique_ptr<
         this->dmap,
         this->grids,
         this->m_vector_poisson_boundary_handler,
-        m_eb_enabled,
+        EB::enabled(),
         WarpX::do_single_precision_comms,
         this->ref_ratio,
         post_A_calculation,

Source/Initialization/WarpXInitData.cpp

@@ -16,6 +16,7 @@
 #endif
 #include "Diagnostics/MultiDiagnostics.H"
 #include "Diagnostics/ReducedDiags/MultiReducedDiags.H"
+#include "EmbeddedBoundary/Enabled.H"
 #include "FieldSolver/Fields.H"
 #include "FieldSolver/FiniteDifferenceSolver/MacroscopicProperties/MacroscopicProperties.H"
 #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H"
@@ -702,6 +703,7 @@ WarpX::InitPML ()
     if (finest_level > 0) { do_pml = 1; }
     if (do_pml)
     {
+        bool const eb_enabled = EB::enabled();
 #if (defined WARPX_DIM_RZ) && (defined WARPX_USE_FFT)
         do_pml_Lo[0][0] = 0; // no PML at r=0, in cylindrical geometry
         pml_rz[0] = std::make_unique<PML_RZ>(0, boxArray(0), DistributionMap(0), &Geom(0), pml_ncell, do_pml_in_domain);
@@ -717,7 +719,7 @@ WarpX::InitPML ()
             psatd_solution_type, J_in_time, rho_in_time,
             do_pml_dive_cleaning, do_pml_divb_cleaning,
             amrex::IntVect(0), amrex::IntVect(0),
-            m_eb_enabled,
+            eb_enabled,
             guard_cells.ng_FieldSolver.max(),
             v_particle_pml,
             do_pml_Lo[0], do_pml_Hi[0]);
@@ -757,7 +759,7 @@ WarpX::InitPML ()
                 do_moving_window, pml_has_particles, do_pml_in_domain,
                 psatd_solution_type, J_in_time, rho_in_time, do_pml_dive_cleaning, do_pml_divb_cleaning,
                 amrex::IntVect(0), amrex::IntVect(0),
-                m_eb_enabled,
+                eb_enabled,
                 guard_cells.ng_FieldSolver.max(),
                 v_particle_pml,
                 do_pml_Lo[lev], do_pml_Hi[lev]);
@@ -936,7 +938,8 @@ WarpX::InitLevelData (int lev, Real /*time*/)
     }
 
 #ifdef AMREX_USE_EB
-    if (m_eb_enabled) { InitializeEBGridData(lev); }
+    bool const eb_enabled = EB::enabled();
+    if (eb_enabled) { InitializeEBGridData(lev); }
 #endif
 
     // if the input string for the B-field is "parse_b_ext_grid_function",
@@ -981,7 +984,7 @@ WarpX::InitLevelData (int lev, Real /*time*/)
         && (lev <= maxlevel_extEMfield_init)) {
 
 #ifdef AMREX_USE_EB
-        if (m_eb_enabled) {
+        if (eb_enabled) {
             // We initialize ECTRhofield consistently with the Efield
             if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) {
                 m_fdtd_solver_fp[lev]->EvolveECTRho(
@@ -1016,7 +1019,7 @@ WarpX::InitLevelData (int lev, Real /*time*/)
                 'E',
                 lev, PatchType::coarse);
 #ifdef AMREX_USE_EB
-            if (m_eb_enabled) {
+            if (eb_enabled) {
                 if (WarpX::electromagnetic_solver_id == ElectromagneticSolverAlgo::ECT) {
                     // We initialize ECTRhofield consistently with the Efield
                     m_fdtd_solver_cp[lev]->EvolveECTRho(Efield_cp[lev], m_edge_lengths[lev],
@@ -1063,6 +1066,8 @@ WarpX::InitializeExternalFieldsOnGridUsingParser (
     const amrex::IntVect y_nodal_flag = mfy->ixType().toIntVect();
     const amrex::IntVect z_nodal_flag = mfz->ixType().toIntVect();
 
+    bool const eb_enabled = EB::enabled();
+
     for ( MFIter mfi(*mfx, TilingIfNotGPU()); mfi.isValid(); ++mfi) {
         const amrex::Box &tbx = mfi.tilebox(x_nodal_flag, mfx->nGrowVect());
         const amrex::Box &tby = mfi.tilebox(y_nodal_flag, mfy->nGrowVect());
@@ -1073,7 +1078,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser (
         auto const &mfzfab = mfz->array(mfi);
 
         amrex::Array4<amrex::Real> lx, ly, lz, Sx, Sy, Sz;
-        if (m_eb_enabled) {
+        if (eb_enabled) {
             lx = edge_lengths[0]->array(mfi);
             ly = edge_lengths[1]->array(mfi);
             lz = edge_lengths[2]->array(mfi);
@@ -1085,7 +1090,7 @@ WarpX::InitializeExternalFieldsOnGridUsingParser (
 #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
         amrex::Dim3 lx_lo, lx_hi, lz_lo, lz_hi;
 #endif
-        if (m_eb_enabled) {
+        if (eb_enabled) {
 #if defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ)
             lx_lo = amrex::lbound(lx);
             lx_hi = amrex::ubound(lx);
@@ -1271,7 +1276,7 @@ void WarpX::InitializeEBGridData (int lev)
     if (lev == maxLevel()) {
 
         // Throw a warning if EB is on and particle_shape > 1
-        if ((nox > 1 or noy > 1 or noz > 1) and m_eb_enabled)
+        if ((nox > 1 or noy > 1 or noz > 1) and EB::enabled())
         {
             ablastr::warn_manager::WMRecordWarning("Particles",
               "when algo.particle_shape > 1, numerical artifacts will be present when\n"

Source/Parallelization/WarpXRegrid.cpp

@@ -10,6 +10,7 @@
 
 #include "Diagnostics/MultiDiagnostics.H"
 #include "Diagnostics/ReducedDiags/MultiReducedDiags.H"
+#include "EmbeddedBoundary/Enabled.H"
 #include "EmbeddedBoundary/WarpXFaceInfoBox.H"
 #include "FieldSolver/FiniteDifferenceSolver/HybridPICModel/HybridPICModel.H"
 #include "Initialization/ExternalField.H"
@@ -177,6 +178,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
         mf = std::move(pmf);
     };
 
+    bool const eb_enabled = EB::enabled();
     if (ba == boxArray(lev))
     {
         if (ParallelDescriptor::NProcs() == 1) { return; }
@@ -215,7 +217,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
                 RemakeMultiFab(m_hybrid_pic_model->current_fp_ampere[lev][idim], false);
                 RemakeMultiFab(m_hybrid_pic_model->current_fp_external[lev][idim],true);
             }
-            if (m_eb_enabled) {
+            if (eb_enabled) {
                 if (WarpX::electromagnetic_solver_id != ElectromagneticSolverAlgo::PSATD) {
                     RemakeMultiFab(m_edge_lengths[lev][idim], false);
                     RemakeMultiFab(m_face_areas[lev][idim], false);
@@ -242,7 +244,7 @@ WarpX::RemakeLevel (int lev, Real /*time*/, const BoxArray& ba, const Distributi
             RemakeMultiFab(m_hybrid_pic_model->electron_pressure_fp[lev], false);
         }
 
-        if (m_eb_enabled) {
+        if (eb_enabled) {
             RemakeMultiFab(m_distance_to_eb[lev], false);
 
 #ifdef AMREX_USE_EB

Source/WarpX.H

@@ -998,7 +998,6 @@ public:
     [[nodiscard]] amrex::IntVect get_numprocs() const {return numprocs;}
 
     /** Enable embedded boundaries */
-    bool m_eb_enabled = false;
     bool m_boundary_potential_specified = false;
     ElectrostaticSolver::PoissonBoundaryHandler m_poisson_boundary_handler;
     void ComputeSpaceChargeField (bool reset_fields);