Merge pull request #145 from seahorce-scidac/update_particles_and_amrex

Update particle functionality and amrex submodule
seahorce-scidac · Jan 24, 2024 · 479f322 · 479f322
2 parents db2f0c9 + 59db60f
commit 479f322
Show file tree

Hide file tree

Showing 4 changed files with 40 additions and 19 deletions.
diff --git a/Source/BoundaryConditions/BoundaryConditions_cons.cpp b/Source/BoundaryConditions/BoundaryConditions_cons.cpp
@@ -45,9 +45,11 @@ void REMORAPhysBCFunct::impose_cons_bcs (const Array4<Real>& dest_arr, const Box
 
     GpuArray<GpuArray<Real, AMREX_SPACEDIM*2>,AMREX_SPACEDIM+NCONS> l_bc_extdir_vals_d;
 
-    for (int i = 0; i < ncomp; i++)
-        for (int ori = 0; ori < 2*AMREX_SPACEDIM; ori++)
+    for (int i = 0; i < ncomp; i++) {
+        for (int ori = 0; ori < 2*AMREX_SPACEDIM; ori++) {
             l_bc_extdir_vals_d[i][ori] = m_bc_extdir_vals[bccomp+i][ori];
+        }
+    }
 
     GeometryData const& geomdata = m_geom.data();
     bool is_periodic_in_x = geomdata.isPeriodic(0);

diff --git a/Source/Particles/TracerPC.H b/Source/Particles/TracerPC.H
@@ -66,4 +66,32 @@ public:
                          amrex::MultiFab& a_z_height);
 };
 
+    template <typename P>
+    AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE
+    void update_location_idata (P& p,
+                                amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& plo,
+                                amrex::GpuArray<amrex::Real,AMREX_SPACEDIM> const& dxi,
+                                const amrex::Array4<amrex::Real const>&  height_arr)
+    {
+        amrex::IntVect iv( int(amrex::Math::floor((p.pos(0)-plo[0])*dxi[0])),
+                           int(amrex::Math::floor((p.pos(1)-plo[1])*dxi[1])),
+                           p.idata(0) );
+
+        amrex::Real lx = (p.pos(0)-plo[0])*dxi[0] - static_cast<amrex::Real>(iv[0]);
+        amrex::Real ly = (p.pos(1)-plo[1])*dxi[1] - static_cast<amrex::Real>(iv[1]);
+        auto zlo = height_arr(iv[0]  ,iv[1]  ,iv[2]  ) * (1.0-lx) * (1.0-ly) +
+                   height_arr(iv[0]+1,iv[1]  ,iv[2]  ) *      lx  * (1.0-ly) +
+                   height_arr(iv[0]  ,iv[1]+1,iv[2]  ) * (1.0-lx) * ly +
+                   height_arr(iv[0]+1,iv[1]+1,iv[2]  ) *      lx  * ly;
+        auto zhi = height_arr(iv[0]  ,iv[1]  ,iv[2]+1) * (1.0-lx) * (1.0-ly) +
+                   height_arr(iv[0]+1,iv[1]  ,iv[2]+1) *      lx  * (1.0-ly) +
+                   height_arr(iv[0]  ,iv[1]+1,iv[2]+1) * (1.0-lx) * ly +
+                   height_arr(iv[0]+1,iv[1]+1,iv[2]+1) *      lx  * ly;
+
+        if (p.pos(2) > zhi) {
+            p.idata(0) += 1;
+        } else if (p.pos(2) <= zlo) {
+            p.idata(0) -= 1;
+        }
+    }
 #endif
diff --git a/Source/Particles/TracerPC.cpp b/Source/Particles/TracerPC.cpp
@@ -135,7 +135,7 @@ TracerPC::AdvectWithUmac (Array<MultiFab const*, AMREX_SPACEDIM> umac,
                 ParticleType& p = p_pbox[i];
                 if (p.id() <= 0) { return; }
                 ParticleReal v[AMREX_SPACEDIM];
-                mac_interpolate(p, plo, dxi, umacarr, v);
+                mac_interpolate_mapped_z(p, plo, dxi, umacarr, zheight, v);
                 if (ipass == 0)
                 {
                     for (int dim=0; dim < AMREX_SPACEDIM; dim++)
@@ -161,24 +161,15 @@ TracerPC::AdvectWithUmac (Array<MultiFab const*, AMREX_SPACEDIM> umac,
                     iv[1] += domain.smallEnd()[1];
                     ParticleReal zlo, zhi;
                     if (use_terrain) {
-                        Real lx = (p.pos(0)-plo[0])*dxi[0] - static_cast<Real>(iv[0]-domain.smallEnd()[0]);
-                        Real ly = (p.pos(1)-plo[1])*dxi[1] - static_cast<Real>(iv[1]-domain.smallEnd()[1]);
-                        zlo =  zheight(iv[0]  ,iv[1]  ,iv[2]  ) * (1.0_rt-lx) * (1.0_rt-ly) +
-                               zheight(iv[0]+1,iv[1]  ,iv[2]  ) *      lx  * (1.0_rt-ly) +
-                               zheight(iv[0]  ,iv[1]+1,iv[2]  ) * (1.0_rt-lx) * ly +
-                               zheight(iv[0]+1,iv[1]+1,iv[2]  ) *      lx  * ly;
-                        zhi =  zheight(iv[0]  ,iv[1]  ,iv[2]+1) * (1.0_rt-lx) * (1.0_rt-ly) +
-                               zheight(iv[0]+1,iv[1]  ,iv[2]+1) *      lx  * (1.0_rt-ly) +
-                               zheight(iv[0]  ,iv[1]+1,iv[2]+1) * (1.0_rt-lx) * ly +
-                               zheight(iv[0]+1,iv[1]+1,iv[2]+1) *      lx  * ly;
+                        update_location_idata(p,plo,dxi,zheight);
                     } else {
                         zlo =  iv[2]    * dx[2];
                         zhi = (iv[2]+1) * dx[2];
-                    }
-                    if (p.pos(2) > zhi) { // need to be careful here
-                        p.idata(0) += 1;
-                    } else if (p.pos(2) <= zlo) {
-                        p.idata(0) -= 1;
+                        if (p.pos(2) > zhi) { // need to be careful here
+                            p.idata(0) += 1;
+                        } else if (p.pos(2) <= zlo) {
+                            p.idata(0) -= 1;
+                        }
                     }
                 }
             });

diff --git a/Submodules/AMReX b/Submodules/AMReX
+8 −8		.github/workflows/apps.yml
+1 −1		.github/workflows/ascent.yml
+3 −3		.github/workflows/bittree.yml
+7 −7		.github/workflows/clang.yml
+1 −1		.github/workflows/cleanup-cache-postpr.yml
+1 −1		.github/workflows/cleanup-cache.yml
+6 −6		.github/workflows/codeql.yml
+1 −1		.github/workflows/codespell.yml
+9 −9		.github/workflows/cuda.yml
+8 −9		.github/workflows/docs.yml
+28 −28		.github/workflows/gcc.yml
+9 −9		.github/workflows/hip.yml
+7 −7		.github/workflows/hypre.yml
+11 −11		.github/workflows/intel.yml
+5 −5		.github/workflows/macos.yml
+3 −3		.github/workflows/petsc.yml
+1 −1		.github/workflows/post-pr.yml
+1 −1		.github/workflows/sensei.yml
+3 −3		.github/workflows/smoke.yml
+3 −3		.github/workflows/style.yml
+5 −5		.github/workflows/sundials.yml
+3 −3		.github/workflows/windows.yml
+5 −1		Src/Base/AMReX.cpp
+18 −0		Src/Base/AMReX_Algorithm.H
+1 −1		Src/Base/AMReX_BLProfiler.H
+121 −4		Src/Base/AMReX_BaseFab.H
+12 −0		Src/Base/AMReX_Extension.H
+4 −1		Src/Base/AMReX_GpuComplex.H
+7 −2		Src/Base/AMReX_OpenMP.H
+31 −1		Src/Base/AMReX_OpenMP.cpp
+30 −2		Src/Base/AMReX_Reduce.H
+7 −14		Src/Base/AMReX_TableData.H
+14 −0		Src/Base/AMReX_Tuple.H
+48 −1		Src/Base/AMReX_TypeList.H
+406 −0		Src/LinearSolvers/AMReX_GMRES.H
+147 −0		Src/LinearSolvers/AMReX_GMRES_MLMG.H
+3 −0		Src/LinearSolvers/CMakeLists.txt
+3 −0		Src/LinearSolvers/MLMG/AMReX_MLLinOp.H
+19 −12		Src/LinearSolvers/MLMG/AMReX_MLMG.H
+4 −0		Src/LinearSolvers/MLMG/Make.package
+9 −0		Src/LinearSolvers/Make.package
+4 −0		Src/LinearSolvers/OpenBC/Make.package
+2 −2		Src/Particle/AMReX_DenseBins.H
+10 −1		Src/Particle/AMReX_Particle.H
+0 −5		Src/Particle/AMReX_ParticleContainer.H
+0 −50		Src/Particle/AMReX_ParticleContainerI.H
+56 −18		Src/Particle/AMReX_ParticleIO.H
+0 −74		Src/Particle/AMReX_Particle_mod_K.H
+357 −153		Src/Particle/AMReX_TracerParticle_mod_K.H
+25 −12		Src/Particle/AMReX_WriteBinaryParticleData.H
+1 −1		Tests/LinearSolvers/ABecLaplacian_C/GNUmakefile
+4 −0		Tests/LinearSolvers/ABecLaplacian_C/MyTest.H
+90 −1		Tests/LinearSolvers/ABecLaplacian_C/MyTest.cpp
+2 −1		Tests/LinearSolvers/ABecLaplacian_C/MyTestPlotfile.cpp
+17 −0		Tests/LinearSolvers/ABecLaplacian_C/inputs.gmres
+0 −4		Tests/Particles/AssignDensity/main.cpp
+0 −4		Tests/Particles/AssignMultiLevelDensity/main.cpp
+1 −1		Tests/Particles/CheckpointRestartSOA/main.cpp