diff --git a/GNUmakefile b/GNUmakefile index be8c80650eb..1acd53be772 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -17,8 +17,6 @@ TINY_PROFILE = TRUE #COMM_PROFILE = TRUE #TRACE_PROFILE = TRUE -STORE_OLD_PARTICLE_ATTRIBS = FALSE - USE_OMP = TRUE USE_GPU = FALSE diff --git a/Source/Diagnostics/WarpXIO.cpp b/Source/Diagnostics/WarpXIO.cpp index ccbdd280c06..186a8d45e9d 100644 --- a/Source/Diagnostics/WarpXIO.cpp +++ b/Source/Diagnostics/WarpXIO.cpp @@ -631,15 +631,16 @@ WarpX::WritePlotFile () const particle_varnames.push_back("theta"); #endif -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - particle_varnames.push_back("xold"); - particle_varnames.push_back("yold"); - particle_varnames.push_back("zold"); - - particle_varnames.push_back("uxold"); - particle_varnames.push_back("uyold"); - particle_varnames.push_back("uzold"); -#endif + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + particle_varnames.push_back("xold"); + particle_varnames.push_back("yold"); + particle_varnames.push_back("zold"); + + particle_varnames.push_back("uxold"); + particle_varnames.push_back("uyold"); + particle_varnames.push_back("uzold"); + } mypc->WritePlotFile(plotfilename, particle_plot_flags, particle_varnames); diff --git a/Source/FortranInterface/WarpX_f.H b/Source/FortranInterface/WarpX_f.H index 473ca645ae3..f246f9f54b2 100644 --- a/Source/FortranInterface/WarpX_f.H +++ b/Source/FortranInterface/WarpX_f.H @@ -81,17 +81,16 @@ extern "C" { #endif -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS void warpx_copy_attribs(const long* np, const amrex_real* xp, const amrex_real* yp, const amrex_real* zp, const amrex_real* uxp, const amrex_real* uyp, const amrex_real* uzp, amrex_real* xpold, amrex_real* ypold, amrex_real* zpold, amrex_real* uxpold, amrex_real* uypold, amrex_real* uzpold); -#endif - void WRPX_COPY_SLICE(const int* lo, const int* hi, - const amrex_real* tmp, const int* tlo, const int* thi, - amrex_real* buf, const int* blo, const int* bhi, - const int* ncomp, const int* i_boost, const int* i_lab); + + void WRPX_COPY_SLICE(const int* lo, const int* hi, + const amrex_real* tmp, const int* tlo, const int* thi, + amrex_real* buf, const int* blo, const int* bhi, + const int* ncomp, const int* i_boost, const int* i_lab); // Charge deposition void warpx_charge_deposition(amrex::Real* rho, diff --git a/Source/Make.WarpX b/Source/Make.WarpX index 926523d82f7..95e2b4ec078 100644 --- a/Source/Make.WarpX +++ b/Source/Make.WarpX @@ -123,10 +123,6 @@ ifeq ($(USE_RZ),TRUE) DEFINES += -DWARPX_RZ endif -ifeq ($(STORE_OLD_PARTICLE_ATTRIBS),TRUE) - DEFINES += -DWARPX_STORE_OLD_PARTICLE_ATTRIBS -endif - ifeq ($(DO_ELECTROSTATIC),TRUE) include $(AMREX_HOME)/Src/LinearSolvers/C_to_F_MG/Make.package include $(AMREX_HOME)/Src/LinearSolvers/F_MG/FParallelMG.mak diff --git a/Source/Particles/MultiParticleContainer.cpp b/Source/Particles/MultiParticleContainer.cpp index 0485f7b8fa5..a4df1f83a98 100644 --- a/Source/Particles/MultiParticleContainer.cpp +++ b/Source/Particles/MultiParticleContainer.cpp @@ -29,7 +29,26 @@ MultiParticleContainer::MultiParticleContainer (AmrCore* amr_core) allcontainers[i].reset(new LaserParticleContainer(amr_core,i, lasers_names[i-nspecies])); } - pc_tmp.reset(new PhysicalParticleContainer(amr_core)); + pc_tmp.reset(new PhysicalParticleContainer(amr_core)); + + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + for (int i = 0; i < nspecies + nlasers; ++i) + { + allcontainers[i]->AddRealComp("xold"); + allcontainers[i]->AddRealComp("yold"); + allcontainers[i]->AddRealComp("zold"); + allcontainers[i]->AddRealComp("uxold"); + allcontainers[i]->AddRealComp("uyold"); + allcontainers[i]->AddRealComp("uzold"); + } + pc_tmp->AddRealComp("xold"); + pc_tmp->AddRealComp("yold"); + pc_tmp->AddRealComp("zold"); + pc_tmp->AddRealComp("uxold"); + pc_tmp->AddRealComp("uyold"); + pc_tmp->AddRealComp("uzold"); + } } void diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index e31d432042c..17e6d98d9a2 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -184,6 +184,18 @@ PhysicalParticleContainer::AddGaussianBeam(Real x_m, Real y_m, Real z_m, attribs[PIdx::uz] = u[2]; attribs[PIdx::w ] = weight; + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); + particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["yold"], y); + particle_tile.push_back_real(particle_comps["zold"], z); + + particle_tile.push_back_real(particle_comps["uxold"], u[0]); + particle_tile.push_back_real(particle_comps["uyold"], u[1]); + particle_tile.push_back_real(particle_comps["uzold"], u[2]); + } + AddOneParticle(0, 0, 0, x, y, z, attribs); } } @@ -455,16 +467,18 @@ PhysicalParticleContainer::AddPlasmaCPU (int lev, RealBox part_realbox) attribs[PIdx::ux] = u[0]; attribs[PIdx::uy] = u[1]; attribs[PIdx::uz] = u[2]; - -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - attribs[PIdx::xold] = x; - attribs[PIdx::yold] = y; - attribs[PIdx::zold] = z; - - attribs[PIdx::uxold] = u[0]; - attribs[PIdx::uyold] = u[1]; - attribs[PIdx::uzold] = u[2]; -#endif + + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); + particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["yold"], y); + particle_tile.push_back_real(particle_comps["zold"], z); + + particle_tile.push_back_real(particle_comps["uxold"], u[0]); + particle_tile.push_back_real(particle_comps["uyold"], u[1]); + particle_tile.push_back_real(particle_comps["uzold"], u[2]); + } AddOneParticle(lev, grid_id, tile_id, x, y, z, attribs); } @@ -695,15 +709,18 @@ PhysicalParticleContainer::AddPlasmaGPU (int lev, RealBox part_realbox) attribs[PIdx::uy] = u[1]; attribs[PIdx::uz] = u[2]; -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - attribs[PIdx::xold] = x; - attribs[PIdx::yold] = y; - attribs[PIdx::zold] = z; - - attribs[PIdx::uxold] = u[0]; - attribs[PIdx::uyold] = u[1]; - attribs[PIdx::uzold] = u[2]; -#endif + // note - this will be slow on the GPU, need to revisit + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& particle_tile = DefineAndReturnParticleTile(lev, grid_id, tile_id); + particle_tile.push_back_real(particle_comps["xold"], x); + particle_tile.push_back_real(particle_comps["yold"], y); + particle_tile.push_back_real(particle_comps["zold"], z); + + particle_tile.push_back_real(particle_comps["uxold"], u[0]); + particle_tile.push_back_real(particle_comps["uyold"], u[1]); + particle_tile.push_back_real(particle_comps["uzold"], u[2]); + } ParticleType p; p.id() = ParticleType::NextID(); @@ -1666,20 +1683,20 @@ PhysicalParticleContainer::PushPX(WarpXParIter& pti, auto& Bzp = attribs[PIdx::Bz]; const long np = pti.numParticles(); -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - auto& xpold = attribs[PIdx::xold]; - auto& ypold = attribs[PIdx::yold]; - auto& zpold = attribs[PIdx::zold]; - auto& uxpold = attribs[PIdx::uxold]; - auto& uypold = attribs[PIdx::uyold]; - auto& uzpold = attribs[PIdx::uzold]; - - warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), - uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); - -#endif + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& xpold = pti.GetAttribs(particle_comps["xold"]); + auto& ypold = pti.GetAttribs(particle_comps["yold"]); + auto& zpold = pti.GetAttribs(particle_comps["zold"]); + auto& uxpold = pti.GetAttribs(particle_comps["uxold"]); + auto& uypold = pti.GetAttribs(particle_comps["uyold"]); + auto& uzpold = pti.GetAttribs(particle_comps["uzold"]); + + warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), + uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), + xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), + uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); + } warpx_particle_pusher(&np, xp.dataPtr(), @@ -1801,7 +1818,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real { BL_PROFILE("PhysicalParticleContainer::GetParticleSlice"); -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS // Assume that the boost in the positive z direction. #if (AMREX_SPACEDIM == 2) AMREX_ALWAYS_ASSERT(direction == 1); @@ -1864,12 +1880,12 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real auto& uyp_new = attribs[PIdx::uy ]; auto& uzp_new = attribs[PIdx::uz ]; - auto& xp_old = attribs[PIdx::xold ]; - auto& yp_old = attribs[PIdx::yold ]; - auto& zp_old = attribs[PIdx::zold ]; - auto& uxp_old = attribs[PIdx::uxold]; - auto& uyp_old = attribs[PIdx::uyold]; - auto& uzp_old = attribs[PIdx::uzold]; + auto& xp_old = pti.GetAttribs(particle_comps["xold"]); + auto& yp_old = pti.GetAttribs(particle_comps["yold"]); + auto& zp_old = pti.GetAttribs(particle_comps["zold"]); + auto& uxp_old = pti.GetAttribs(particle_comps["uxold"]); + auto& uyp_old = pti.GetAttribs(particle_comps["uyold"]); + auto& uzp_old = pti.GetAttribs(particle_comps["uzold"]); const long np = pti.numParticles(); @@ -1919,10 +1935,6 @@ void PhysicalParticleContainer::GetParticleSlice(const int direction, const Real } } } -#else - AMREX_ALWAYS_ASSERT_WITH_MESSAGE( false , -"ERROR: WarpX must be compiled with STORE_OLD_PARTICLE_ATTRIBS=TRUE to use the back-transformed diagnostics"); -#endif } int PhysicalParticleContainer::GetRefineFac(const Real x, const Real y, const Real z) diff --git a/Source/Particles/RigidInjectedParticleContainer.cpp b/Source/Particles/RigidInjectedParticleContainer.cpp index 3ee4d87e53b..a5acca2815b 100644 --- a/Source/Particles/RigidInjectedParticleContainer.cpp +++ b/Source/Particles/RigidInjectedParticleContainer.cpp @@ -225,20 +225,20 @@ RigidInjectedParticleContainer::PushPX(WarpXParIter& pti, auto& Bzp = attribs[PIdx::Bz]; const long np = pti.numParticles(); -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - auto& xpold = attribs[PIdx::xold]; - auto& ypold = attribs[PIdx::yold]; - auto& zpold = attribs[PIdx::zold]; - auto& uxpold = attribs[PIdx::uxold]; - auto& uypold = attribs[PIdx::uyold]; - auto& uzpold = attribs[PIdx::uzold]; - - warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), - uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), - xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), - uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); - -#endif + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& xpold = pti.GetAttribs(particle_comps["xold"]); + auto& ypold = pti.GetAttribs(particle_comps["yold"]); + auto& zpold = pti.GetAttribs(particle_comps["zold"]); + auto& uxpold = pti.GetAttribs(particle_comps["uxold"]); + auto& uypold = pti.GetAttribs(particle_comps["uyold"]); + auto& uzpold = pti.GetAttribs(particle_comps["uzold"]); + + warpx_copy_attribs(&np, xp.dataPtr(), yp.dataPtr(), zp.dataPtr(), + uxp.dataPtr(), uyp.dataPtr(), uzp.dataPtr(), + xpold.dataPtr(), ypold.dataPtr(), zpold.dataPtr(), + uxpold.dataPtr(), uypold.dataPtr(), uzpold.dataPtr()); + } // Save the position and momenta, making copies Cuda::ManagedDeviceVector xp_save, yp_save, zp_save; diff --git a/Source/Particles/WarpXParticleContainer.H b/Source/Particles/WarpXParticleContainer.H index 050060b475d..275554cd895 100644 --- a/Source/Particles/WarpXParticleContainer.H +++ b/Source/Particles/WarpXParticleContainer.H @@ -14,9 +14,6 @@ struct PIdx #ifdef WARPX_RZ theta, // RZ needs all three position components #endif -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - xold, yold, zold, uxold, uyold, uzold, -#endif nattribs }; }; @@ -42,15 +39,7 @@ namespace ParticleStringNames {"Ez", PIdx::Ez }, {"Bx", PIdx::Bx }, {"By", PIdx::By }, - {"Bz", PIdx::Bz }, -#ifdef WARPX_STORE_OLD_PARTICLE_ATTRIBS - {"xold", PIdx::xold }, - {"yold", PIdx::yold }, - {"zold", PIdx::zold }, - {"uxold", PIdx::uxold}, - {"uyold", PIdx::uyold}, - {"uzold", PIdx::uzold}, -#endif + {"Bz", PIdx::Bz } }; } @@ -231,8 +220,25 @@ public: // split along axes (0) or diagonals (1) int split_type = 0; + using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddRealComp; + using amrex::ParticleContainer<0, 0, PIdx::nattribs>::AddIntComp; + + void AddRealComp (const std::string& name, bool comm=true) + { + particle_comps[name] = NumRealComps(); + AddRealComp(comm); + } + + void AddIntComp (const std::string& name, bool comm=true) + { + particle_comps[name] = NumIntComps(); + AddIntComp(comm); + } + protected: + std::map particle_comps; + int species_id; amrex::Real charge; diff --git a/Source/Particles/WarpXParticleContainer.cpp b/Source/Particles/WarpXParticleContainer.cpp index c52e0a6d090..2edd3c636be 100644 --- a/Source/Particles/WarpXParticleContainer.cpp +++ b/Source/Particles/WarpXParticleContainer.cpp @@ -63,6 +63,32 @@ WarpXParticleContainer::WarpXParticleContainer (AmrCore* amr_core, int ispecies) SetParticleSize(); ReadParameters(); + // build up the map of string names to particle component numbers + particle_comps["w"] = PIdx::w; + particle_comps["ux"] = PIdx::ux; + particle_comps["uy"] = PIdx::uy; + particle_comps["uz"] = PIdx::uz; + particle_comps["Ex"] = PIdx::Ex; + particle_comps["Ey"] = PIdx::Ey; + particle_comps["Ez"] = PIdx::Ez; + particle_comps["Bx"] = PIdx::Bx; + particle_comps["By"] = PIdx::By; + particle_comps["Bz"] = PIdx::Bz; +#ifdef WARPX_RZ + particle_comps["theta"] = PIdx::theta; +#endif + + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + particle_comps["xold"] = PIdx::nattribs; + particle_comps["yold"] = PIdx::nattribs+1; + particle_comps["zold"] = PIdx::nattribs+2; + particle_comps["uxold"] = PIdx::nattribs+3; + particle_comps["uyold"] = PIdx::nattribs+4; + particle_comps["uzold"] = PIdx::nattribs+5; + + } + // Initialize temporary local arrays for charge/current deposition int num_threads = 1; #ifdef _OPENMP @@ -204,6 +230,15 @@ WarpXParticleContainer::AddNParticles (int lev, #endif p.pos(1) = z[i]; #endif + + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); + particle_tile.push_back_real(particle_comps["xold"], x[i]); + particle_tile.push_back_real(particle_comps["yold"], y[i]); + particle_tile.push_back_real(particle_comps["zold"], z[i]); + } + particle_tile.push_back(p); } @@ -214,6 +249,14 @@ WarpXParticleContainer::AddNParticles (int lev, particle_tile.push_back_real(PIdx::uy, vy + ibegin, vy + iend); particle_tile.push_back_real(PIdx::uz, vz + ibegin, vz + iend); + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + auto& particle_tile = DefineAndReturnParticleTile(0, 0, 0); + particle_tile.push_back_real(particle_comps["uxold"], vx + ibegin, vx + iend); + particle_tile.push_back_real(particle_comps["uyold"], vy + ibegin, vy + iend); + particle_tile.push_back_real(particle_comps["uzold"], vz + ibegin, vz + iend); + } + for (int comp = PIdx::uz+1; comp < PIdx::nattribs; ++comp) { #ifdef WARPX_RZ diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index 423ff649b6e..53285855666 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -436,11 +436,25 @@ WarpX::ReadParameters () if (particle_plot_vars.size() == 0) { - particle_plot_flags.resize(PIdx::nattribs, 1); + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + particle_plot_flags.resize(PIdx::nattribs + 6, 1); + } + else + { + particle_plot_flags.resize(PIdx::nattribs, 1); + } } else { - particle_plot_flags.resize(PIdx::nattribs, 0); + if (WarpX::do_boosted_frame_diagnostic && WarpX::do_boosted_frame_particles) + { + particle_plot_flags.resize(PIdx::nattribs + 6, 0); + } + else + { + particle_plot_flags.resize(PIdx::nattribs, 0); + } for (const auto& var : particle_plot_vars) { diff --git a/Tools/performance_tests/GNUmakefile_perftest b/Tools/performance_tests/GNUmakefile_perftest index bbb2cc6593c..7740b0a6433 100644 --- a/Tools/performance_tests/GNUmakefile_perftest +++ b/Tools/performance_tests/GNUmakefile_perftest @@ -6,7 +6,6 @@ DEBUG = FALSE DIM = 3 COMP=intel TINY_PROFILE = TRUE -STORE_OLD_PARTICLE_ATTRIBS=TRUE USE_OMP = TRUE EBASE = perf_tests USE_PYTHON_MAIN = FALSE