From 3c5ebd332bca524839cf6924aa4c7ac3edc33319 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Fri, 13 Sep 2024 20:28:26 -0700 Subject: [PATCH 01/13] Add base 2D Langmuir test, fix broken docs link (#5271) --- Examples/Tests/langmuir/CMakeLists.txt | 10 +++++++ .../langmuir/inputs_test_2d_langmuir_multi | 7 +++++ .../test_2d_langmuir_multi.json | 29 +++++++++++++++++++ 3 files changed, 46 insertions(+) create mode 100644 Examples/Tests/langmuir/inputs_test_2d_langmuir_multi create mode 100644 Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json diff --git a/Examples/Tests/langmuir/CMakeLists.txt b/Examples/Tests/langmuir/CMakeLists.txt index bd0cea79c7a..b259083c695 100644 --- a/Examples/Tests/langmuir/CMakeLists.txt +++ b/Examples/Tests/langmuir/CMakeLists.txt @@ -11,6 +11,16 @@ add_warpx_test( OFF # dependency ) +add_warpx_test( + test_2d_langmuir_multi # name + 2 # dims + 2 # nprocs + inputs_test_2d_langmuir_multi # inputs + analysis_2d.py # analysis + diags/diag1000080 # output + OFF # dependency +) + add_warpx_test( test_2d_langmuir_multi_mr # name 2 # dims diff --git a/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi new file mode 100644 index 00000000000..df0189acd91 --- /dev/null +++ b/Examples/Tests/langmuir/inputs_test_2d_langmuir_multi @@ -0,0 +1,7 @@ +# base input parameters +FILE = inputs_base_2d + +# test input parameters +algo.current_deposition = direct +diag1.electrons.variables = x z w ux uy uz +diag1.positrons.variables = x z w ux uy uz diff --git a/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json new file mode 100644 index 00000000000..899352c45ba --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_2d_langmuir_multi.json @@ -0,0 +1,29 @@ +{ + "lev=0": { + "Bx": 0.0, + "By": 5.726296856755232, + "Bz": 0.0, + "Ex": 3751589134191.326, + "Ey": 0.0, + "Ez": 3751589134191.332, + "jx": 1.0100623329922576e+16, + "jy": 0.0, + "jz": 1.0100623329922578e+16 + }, + "electrons": { + "particle_momentum_x": 5.668407513430198e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407513430198e-20, + "particle_position_x": 0.6553599999999999, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + }, + "positrons": { + "particle_momentum_x": 5.668407513430198e-20, + "particle_momentum_y": 0.0, + "particle_momentum_z": 5.668407513430198e-20, + "particle_position_x": 0.6553599999999999, + "particle_position_y": 0.65536, + "particle_weight": 3200000000000000.5 + } +} From b3f759b6e811fdc248343a8b36f2a8fedbdcdabe Mon Sep 17 00:00:00 2001 From: David Grote Date: Mon, 16 Sep 2024 16:19:15 -0700 Subject: [PATCH 02/13] For flux injection, improve the calculation of number of particles per cell (#5272) * Improve the calculation of number of particles per cell * Up test_rz_flux_injection benchmark This is needed since the change alters the calling of the random number --- .../test_rz_flux_injection.json | 23 ++++++++++--------- .../Particles/PhysicalParticleContainer.cpp | 5 ++-- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json b/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json index 5a80590891c..2ba80d4fb0e 100644 --- a/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json +++ b/Regression/Checksum/benchmarks_json/test_rz_flux_injection.json @@ -1,14 +1,15 @@ { - "electron": { - "particle_momentum_x": 7.168456345337534e-18, - "particle_momentum_y": 7.02290351254873e-18, - "particle_momentum_z": 9.565641373942318e-42, - "particle_position_x": 6962.988311042427, - "particle_position_y": 2034.5301680154264, - "particle_theta": 6397.068924320389, - "particle_weight": 3.215011942598676e-08 - }, "lev=0": { - "Bz": 9.526664429810971e-24 + "Bz": 9.524453851623612e-24 + }, + "electron": { + "particle_momentum_x": 7.146168286112378e-18, + "particle_momentum_y": 7.073108431229069e-18, + "particle_momentum_z": 9.282175511339672e-42, + "particle_position_x": 6978.157994231982, + "particle_position_y": 2044.6981840260364, + "particle_theta": 6298.956888689097, + "particle_weight": 3.2236798669537214e-08 } -} \ No newline at end of file +} + diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index 0617b36a273..cfd0eb0b500 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1453,8 +1453,6 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, auto lo = getCellCoords(overlap_corner, dx, {0._rt, 0._rt, 0._rt}, iv); auto hi = getCellCoords(overlap_corner, dx, {1._rt, 1._rt, 1._rt}, iv); - const int num_ppc_int = static_cast(num_ppc_real + amrex::Random(engine)); - if (flux_pos->overlapsWith(lo, hi)) { auto index = overlap_box.index(iv); @@ -1464,7 +1462,8 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, } else { r = 1; } - pcounts[index] = num_ppc_int*r; + const int num_ppc_int = static_cast(num_ppc_real*r + amrex::Random(engine)); + pcounts[index] = num_ppc_int; } amrex::ignore_unused(j,k); }); From 12bf06904e5fbefdf5ed1b6cc9b146ba386c5c17 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 16 Sep 2024 16:20:37 -0700 Subject: [PATCH 03/13] [pre-commit.ci] pre-commit autoupdate (#5276) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.6.4 → v0.6.5](https://github.com/astral-sh/ruff-pre-commit/compare/v0.6.4...v0.6.5) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index c581183703a..1b668d5931e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -69,7 +69,7 @@ repos: # Python: Ruff linter & formatter # https://docs.astral.sh/ruff/ - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.6.4 + rev: v0.6.5 hooks: # Run the linter - id: ruff From 2c7a9bee1968027773c09ec32365431aadc04b11 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 16 Sep 2024 17:01:33 -0700 Subject: [PATCH 04/13] Doc: Link to heFFTe RST Rendering in RST did not resolve, missing closing `>`. --- Docs/source/install/dependencies.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Docs/source/install/dependencies.rst b/Docs/source/install/dependencies.rst index f46dc3d1640..72c599ae2bd 100644 --- a/Docs/source/install/dependencies.rst +++ b/Docs/source/install/dependencies.rst @@ -28,7 +28,7 @@ Optional dependencies include: - `FFTW3 `__: for spectral solver (PSATD or IGF) support when running on CPU or SYCL - also needs the ``pkg-config`` tool on Unix -- `heFFTe 2.4.0+ `__: for multi-node spectral solver (IGF) support - `BLAS++ `__ and `LAPACK++ `__: for spectral solver (PSATD) support in RZ geometry - `Boost 1.66.0+ `__: for QED lookup tables generation support - `openPMD-api 0.15.1+ `__: we automatically download and compile a copy of openPMD-api for openPMD I/O support From f7dd6a9bc8bb1cfe863409fcf0451b020e99509e Mon Sep 17 00:00:00 2001 From: Johannes van de Wetering <92386744+johvandewetering@users.noreply.github.com> Date: Mon, 16 Sep 2024 19:21:59 -0700 Subject: [PATCH 05/13] [Hackathon] Ionization docs fixes (#5270) * Fixed typos in generalized Ohm's law * Fixed Testing link * Removed minus sign from BTO equation (Zhang eq 8 is wrong) * Clarify comment about difference to published equation. Co-authored-by: Axel Huebl --------- Co-authored-by: Johannes Van de Wetering Co-authored-by: Roelof Groenewald <40245517+roelof-groenewald@users.noreply.github.com> Co-authored-by: Axel Huebl --- Docs/source/theory/kinetic_fluid_hybrid_model.rst | 4 ++-- Docs/source/theory/multiphysics/ionization.rst | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Docs/source/theory/kinetic_fluid_hybrid_model.rst b/Docs/source/theory/kinetic_fluid_hybrid_model.rst index b4f494d8382..f764ce4e02b 100644 --- a/Docs/source/theory/kinetic_fluid_hybrid_model.rst +++ b/Docs/source/theory/kinetic_fluid_hybrid_model.rst @@ -46,7 +46,7 @@ integrating over velocity), also called the generalized Ohm's law, is given by: .. math:: - en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e + en_e\vec{E} = \frac{m}{e}\frac{\partial \vec{J}_e}{\partial t} + \frac{m}{e}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e where :math:`\vec{U}_e = \vec{J}_e/(en_e)` is the electron fluid velocity, :math:`{\overleftrightarrow P}_e` is the electron pressure tensor and @@ -64,7 +64,7 @@ Plugging this back into the generalized Ohm' law gives: \left(en_e +\frac{m}{e\mu_0}\nabla\times\nabla\times\right)\vec{E} =& - \frac{m}{e}\left( \frac{\partial\vec{J}_{ext}}{\partial t} + \sum_{s\neq e}\frac{\partial\vec{J}_s}{\partial t} \right) \\ - &+ \frac{m}{e^2}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. + &+ \frac{m}{e}\left( \vec{U}_e\cdot\nabla \right) \vec{J}_e - \nabla\cdot {\overleftrightarrow P}_e - \vec{J}_e\times\vec{B}+\vec{R}_e. If we now further assume electrons are inertialess (i.e. :math:`m=0`), the above equation simplifies to, diff --git a/Docs/source/theory/multiphysics/ionization.rst b/Docs/source/theory/multiphysics/ionization.rst index 11abea386c8..5003872b1a1 100644 --- a/Docs/source/theory/multiphysics/ionization.rst +++ b/Docs/source/theory/multiphysics/ionization.rst @@ -56,18 +56,18 @@ where :math:`\mathrm{d}\tau` is the simulation timestep, which is divided by the Empirical Extension to Over-the-Barrier Regime for Hydrogen ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -For hydrogen, WarpX offers the modified empirical ADK extension to the Over-the-Barrier (OTB) published in :cite:t:`mpion-zhang_empirical_2014` Eq. (8). +For hydrogen, WarpX offers the modified empirical ADK extension to the Over-the-Barrier (OTB) published in :cite:t:`mpion-zhang_empirical_2014` Eq. (8) (note there is a typo in the paper and there should not be a minus sign in Eq. 8). .. math:: - W_\mathrm{M} = \exp\left[ -\left( a_1 \frac{E^2}{E_\mathrm{b}} + a_2 \frac{E}{E_\mathrm{b}} + a_3 \right) \right] W_\mathrm{ADK} + W_\mathrm{M} = \exp\left[ a_1 \frac{E^2}{E_\mathrm{b}} + a_2 \frac{E}{E_\mathrm{b}} + a_3 \right] W_\mathrm{ADK} The parameters :math:`a_1` through :math:`a_3` are independent of :math:`E` and can be found in the same reference. :math:`E_\mathrm{b}` is the classical Barrier Suppresion Ionization (BSI) field strength :math:`E_\mathrm{b} = U_\mathrm{ion}^2 / (4 Z)` given here in atomic units (AU). For a detailed description of conversion between unit systems consider the book by :cite:t:`mpion-Mulser2010`. Testing ^^^^^^^ -* `Testing the field ionization module <../../../../Examples/Tests/field_ionization/README.rst>`_. +* `Testing the field ionization module <../../../../en/latest/usage/examples/field_ionization/README.html>`_. .. bibliography:: :keyprefix: mpion- From 55e86ded534bbadc77af7e635415004106cc750c Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Tue, 17 Sep 2024 07:37:56 -0700 Subject: [PATCH 06/13] Minor refactoring of refined injection for AddPlasmaFlux (#5274) * Minor refactoring of refined injection for AddPlasmaFlux * Additional simplification * Minor fixes * Remove unneeded variable lrrfac --- .../Particles/PhysicalParticleContainer.cpp | 37 ++++++------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/Source/Particles/PhysicalParticleContainer.cpp b/Source/Particles/PhysicalParticleContainer.cpp index cfd0eb0b500..23af57b9206 100644 --- a/Source/Particles/PhysicalParticleContainer.cpp +++ b/Source/Particles/PhysicalParticleContainer.cpp @@ -1030,7 +1030,6 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); - const amrex::IntVect lrrfac = rrfac; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { fine_overlap_box = overlap_box & amrex::shift(fine_injection_box, -shifted); @@ -1048,7 +1047,7 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int { auto index = overlap_box.index(iv); const amrex::Long r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv))? - (AMREX_D_TERM(lrrfac[0],*lrrfac[1],*lrrfac[2])) : (1); + (AMREX_D_TERM(rrfac[0],*rrfac[1],*rrfac[2])) : (1); pcounts[index] = num_ppc*r; // update pcount by checking if cell-corners or cell-center // has non-zero density @@ -1154,8 +1153,8 @@ PhysicalParticleContainer::AddPlasma (PlasmaInjector const& plasma_injector, int long ip = poffset[index] + i_part; pa_idcpu[ip] = amrex::SetParticleIDandCPU(pid+ip, cpuid); const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? - // In the refined injection region: use refinement ratio `lrrfac` - inj_pos->getPositionUnitBox(i_part, lrrfac, engine) : + // In the refined injection region: use refinement ratio `rrfac` + inj_pos->getPositionUnitBox(i_part, rrfac, engine) : // Otherwise: use 1 as the refinement ratio inj_pos->getPositionUnitBox(i_part, amrex::IntVect::TheUnitVector(), engine); auto pos = getCellCoords(overlap_corner, dx, r, iv); @@ -1441,7 +1440,6 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, Gpu::DeviceVector counts(overlap_box.numPts(), 0); Gpu::DeviceVector offset(overlap_box.numPts()); auto *pcounts = counts.data(); - const amrex::IntVect lrrfac = rrfac; const int flux_normal_axis = plasma_injector.flux_normal_axis; Box fine_overlap_box; // default Box is NOT ok(). if (refine_injection) { @@ -1450,22 +1448,21 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, amrex::ParallelForRNG(overlap_box, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { const IntVect iv(AMREX_D_DECL(i, j, k)); + amrex::ignore_unused(j,k); + auto lo = getCellCoords(overlap_corner, dx, {0._rt, 0._rt, 0._rt}, iv); auto hi = getCellCoords(overlap_corner, dx, {1._rt, 1._rt, 1._rt}, iv); if (flux_pos->overlapsWith(lo, hi)) { auto index = overlap_box.index(iv); - int r; + int r = 1; if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { - r = compute_area_weights(lrrfac, flux_normal_axis); - } else { - r = 1; + r = compute_area_weights(rrfac, flux_normal_axis); } const int num_ppc_int = static_cast(num_ppc_real*r + amrex::Random(engine)); pcounts[index] = num_ppc_int; } - amrex::ignore_unused(j,k); }); // Max number of new particles. All of them are created, @@ -1532,24 +1529,14 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, [=] AMREX_GPU_DEVICE (int i, int j, int k, amrex::RandomEngine const& engine) noexcept { const IntVect iv = IntVect(AMREX_D_DECL(i, j, k)); + amrex::ignore_unused(j,k); const auto index = overlap_box.index(iv); Real scale_fac = compute_scale_fac_area(dx, num_ppc_real, flux_normal_axis); - auto lo = getCellCoords(overlap_corner, dx, {0._rt, 0._rt, 0._rt}, iv); - auto hi = getCellCoords(overlap_corner, dx, {1._rt, 1._rt, 1._rt}, iv); - - if (flux_pos->overlapsWith(lo, hi)) - { - int r; - if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { - r = compute_area_weights(lrrfac, flux_normal_axis); - } else { - r = 1; - } - scale_fac /= r; + if (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) { + scale_fac /= compute_area_weights(rrfac, flux_normal_axis); } - amrex::ignore_unused(j,k); for (int i_part = 0; i_part < pcounts[index]; ++i_part) { @@ -1558,8 +1545,8 @@ PhysicalParticleContainer::AddPlasmaFlux (PlasmaInjector const& plasma_injector, // This assumes the flux_pos is of type InjectorPositionRandomPlane const XDim3 r = (fine_overlap_box.ok() && fine_overlap_box.contains(iv)) ? - // In the refined injection region: use refinement ratio `lrrfac` - flux_pos->getPositionUnitBox(i_part, lrrfac, engine) : + // In the refined injection region: use refinement ratio `rrfac` + flux_pos->getPositionUnitBox(i_part, rrfac, engine) : // Otherwise: use 1 as the refinement ratio flux_pos->getPositionUnitBox(i_part, amrex::IntVect::TheUnitVector(), engine); auto pos = getCellCoords(overlap_corner, dx, r, iv); From 6463b1f84878e9e0fc957b0ef9ccde53efde0859 Mon Sep 17 00:00:00 2001 From: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> Date: Tue, 17 Sep 2024 21:12:02 +0200 Subject: [PATCH 07/13] Updated visualization scripts for beam-beam collision example (#4797) * add viz scripts to beam-beam collision example * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Update plot_reduced.py * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Modified pull request * Modified README.rst * Modified README.rst * updated vizs * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * restore plot fields * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * rm EOL white spaces * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Add missing text back. * Change figure link --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Arianna Formenti Co-authored-by: Remi Lehe --- .../beam_beam_collision/README.rst | 49 +++++- .../inputs_test_3d_beam_beam_collision | 2 +- .../beam_beam_collision/plot_fields.py | 139 ++++++++++++++++++ .../beam_beam_collision/plot_reduced.py | 48 ++++++ 4 files changed, 233 insertions(+), 5 deletions(-) create mode 100644 Examples/Physics_applications/beam_beam_collision/plot_fields.py create mode 100644 Examples/Physics_applications/beam_beam_collision/plot_reduced.py diff --git a/Examples/Physics_applications/beam_beam_collision/README.rst b/Examples/Physics_applications/beam_beam_collision/README.rst index a7a06521218..28fdc1ee70e 100644 --- a/Examples/Physics_applications/beam_beam_collision/README.rst +++ b/Examples/Physics_applications/beam_beam_collision/README.rst @@ -11,7 +11,8 @@ We turn on the Quantum Synchrotron QED module for photon emission (also known as the Breit-Wheeler QED module for the generation of electron-positron pairs (also known as coherent pair generation in the collider community). To solve for the electromagnetic field we use the nodal version of the electrostatic relativistic solver. -This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). This solver accurately reproduced the subtle cancellation that occur for some component of the ``E + v x B`` terms which are crucial in simulations of relativistic particles. +This solver computes the average velocity of each species, and solves the corresponding relativistic Poisson equation (see the WarpX documentation for `warpx.do_electrostatic = relativistic` for more detail). +This solver accurately reproduces the subtle cancellation that occur for some component of ``E + v x B``, which are crucial in simulations of relativistic particles. This example is based on the following paper :cite:t:`ex-Yakimenko2019`. @@ -26,7 +27,7 @@ For `MPI-parallel `__ runs, prefix these lines with ` .. literalinclude:: inputs_test_3d_beam_beam_collision :language: ini - :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/inputs_test_3d_beam_beam_collision``. + :caption: You can copy this file from ``Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision``. Visualize @@ -34,12 +35,15 @@ Visualize The figure below shows the number of photons emitted per beam particle (left) and the number of secondary pairs generated per beam particle (right). -We compare different results: +We compare different results for the reduced diagnostics with the literature: * (red) simplified WarpX simulation as the example stored in the directory ``/Examples/Physics_applications/beam-beam_collision``; * (blue) large-scale WarpX simulation (high resolution and ad hoc generated tables ; * (black) literature results from :cite:t:`ex-Yakimenko2019`. -The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 64`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. For the large-scale simulation we have used the following options: +The small-scale simulation has been performed with a resolution of ``nx = 64, ny = 64, nz = 64`` grid cells, while the large-scale one has a much higher resolution of ``nx = 512, ny = 512, nz = 1024``. +Moreover, the large-scale simulation uses dedicated QED lookup tables instead of the builtin tables. +To generate the tables within WarpX, the code must be compiled with the flag ``-DWarpX_QED_TABLE_GEN=ON``. +For the large-scale simulation we have used the following options: .. code-block:: ini @@ -63,8 +67,45 @@ The small-scale simulation has been performed with a resolution of ``nx = 64, ny qed_bw.tab_pair_frac_how_many=512 qed_bw.save_table_in=my_bw_table.txt + .. figure:: https://gist.github.com/user-attachments/assets/2dd43782-d039-4faa-9d27-e3cf8fb17352 :alt: Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. :width: 100% Beam-beam collision benchmark against :cite:t:`ex-Yakimenko2019`. + + +Below are two visualizations scripts that provide examples to graph the field and reduced diagnostics. +They are available in the ``Examples/Physics_applications/beam-beam_collision/`` folder and can be run as simply as ``python3 plot_fields.py`` and ``python3 plot_reduced.py``. + +.. tab-set:: + + .. tab-item:: Field Diagnostics + + This script visualizes the evolution of the fields (:math:`|E|, |B|, \rho`) during the collision between the two ultra-relativistic lepton beams. + The magnitude of E and B and the charge densities of the primary beams and of the secondary pairs are sliced along either one of the two transverse coordinates (:math:`x` and :math:`y`). + + .. literalinclude:: plot_fields.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/plot_fields.py``. + + .. figure:: https://gist.github.com/user-attachments/assets/04c9c0ec-b580-446f-a11a-437c1b244a41 + :alt: Slice across :math:`x` of different fields (:math:`|E|, |B|, \rho`) at timestep 45, in the middle of the collision. + :width: 100% + + Slice across :math:`x` of different fields (:math:`|E|, |B|, \rho`) at timestep 45, in the middle of the collision. + + + .. tab-item:: Reduced Diagnostics + + A similar script to the one below was used to produce the image showing the benchmark against :cite:t:`ex-Yakimenko2019`. + + .. literalinclude:: plot_reduced.py + :language: python3 + :caption: You can copy this file from ``Examples/Physics_applications/beam-beam_collision/plot_reduced.py``. + + .. figure:: https://gist.github.com/user-attachments/assets/c280490a-f1f2-4329-ad3c-46817d245dc1 + :alt: Photon and pair production rates in time throughout the collision. + :width: 100% + + Photon and pair production rates in time throughout the collision. diff --git a/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision index e856a078003..d0cf3cd7ebf 100644 --- a/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision +++ b/Examples/Physics_applications/beam_beam_collision/inputs_test_3d_beam_beam_collision @@ -211,7 +211,7 @@ warpx.do_qed_schwinger = 0. # FULL diagnostics.diags_names = diag1 -diag1.intervals = 0 +diag1.intervals = 15 diag1.diag_type = Full diag1.write_species = 1 diag1.fields_to_plot = Ex Ey Ez Bx By Bz rho_beam1 rho_beam2 rho_ele1 rho_pos1 rho_ele2 rho_pos2 diff --git a/Examples/Physics_applications/beam_beam_collision/plot_fields.py b/Examples/Physics_applications/beam_beam_collision/plot_fields.py new file mode 100644 index 00000000000..a7ddb2d13e9 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/plot_fields.py @@ -0,0 +1,139 @@ +#!/usr/bin/env python3 + +import matplotlib.pyplot as plt +import numpy as np +from mpl_toolkits.axes_grid1.axes_divider import make_axes_locatable +from openpmd_viewer import OpenPMDTimeSeries + +plt.rcParams.update({"font.size": 16}) + +series = OpenPMDTimeSeries("./diags/diag1") +steps = series.iterations + + +for slice_axis in ["x", "y"]: # slice the fields along x and y + for n in steps: # loop through the available timesteps + fig, ax = plt.subplots( + ncols=2, nrows=2, figsize=(10, 6), dpi=300, sharex=True, sharey=True + ) + + # get E field + Ex, info = series.get_field( + field="E", coord="x", iteration=n, plot=False, slice_across=slice_axis + ) + Ey, info = series.get_field( + field="E", coord="y", iteration=n, plot=False, slice_across=slice_axis + ) + Ez, info = series.get_field( + field="E", coord="z", iteration=n, plot=False, slice_across=slice_axis + ) + # get B field + Bx, info = series.get_field( + field="B", coord="x", iteration=n, plot=False, slice_across=slice_axis + ) + By, info = series.get_field( + field="B", coord="y", iteration=n, plot=False, slice_across=slice_axis + ) + Bz, info = series.get_field( + field="B", coord="z", iteration=n, plot=False, slice_across=slice_axis + ) + # get charge densities + rho_beam1, info = series.get_field( + field="rho_beam1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_beam2, info = series.get_field( + field="rho_beam2", iteration=n, plot=False, slice_across=slice_axis + ) + rho_ele1, info = series.get_field( + field="rho_ele1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_pos1, info = series.get_field( + field="rho_pos1", iteration=n, plot=False, slice_across=slice_axis + ) + rho_ele2, info = series.get_field( + field="rho_ele2", iteration=n, plot=False, slice_across=slice_axis + ) + rho_pos2, info = series.get_field( + field="rho_pos2", iteration=n, plot=False, slice_across=slice_axis + ) + + xmin = info.z.min() + xmax = info.z.max() + xlabel = "z [m]" + + if slice_axis == "x": + ymin = info.y.min() + ymax = info.y.max() + ylabel = "y [m]" + elif slice_axis == "y": + ymin = info.x.min() + ymax = info.x.max() + ylabel = "x [m]" + + # plot E magnitude + Emag = np.sqrt(Ex**2 + Ey**2 + Ez**2) + im = ax[0, 0].imshow( + np.transpose(Emag), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=0, + vmax=np.max(np.abs(Emag)), + ) + ax[0, 0].set_title("E [V/m]") + divider = make_axes_locatable(ax[0, 0]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot B magnitude + Bmag = np.sqrt(Bx**2 + By**2 + Bz**2) + im = ax[1, 0].imshow( + np.transpose(Bmag), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=0, + vmax=np.max(np.abs(Bmag)), + ) + ax[1, 0].set_title("B [T]") + divider = make_axes_locatable(ax[1, 0]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot beam densities + rho_beams = rho_beam1 + rho_beam2 + im = ax[0, 1].imshow( + np.transpose(rho_beams), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=-np.max(np.abs(rho_beams)), + vmax=np.max(np.abs(rho_beams)), + ) + ax[0, 1].set_title(r"$\rho$ beams [C/m$^3$]") + divider = make_axes_locatable(ax[0, 1]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + # plot secondary densities + rho2 = rho_ele1 + rho_pos1 + rho_ele2 + rho_pos2 + im = ax[1, 1].imshow( + np.transpose(rho2), + cmap="seismic", + extent=[xmin, xmax, ymin, ymax], + vmin=-np.max(np.abs(rho2)), + vmax=np.max(np.abs(rho2)), + ) + ax[1, 1].set_title(r"$\rho$ secondaries [C/m$^3$]") + divider = make_axes_locatable(ax[1, 1]) + cax = divider.append_axes("right", size="5%", pad=0.05) + fig.colorbar(im, cax=cax, orientation="vertical") + + for a in ax[-1, :].reshape(-1): + a.set_xlabel(xlabel) + for a in ax[:, 0].reshape(-1): + a.set_ylabel(ylabel) + + fig.suptitle(f"Iteration = {n}, time [s] = {series.current_t}", fontsize=20) + plt.tight_layout() + + image_file_name = "FIELDS_" + slice_axis + f"_{n:03d}.png" + plt.savefig(image_file_name, dpi=100, bbox_inches="tight") + plt.close() diff --git a/Examples/Physics_applications/beam_beam_collision/plot_reduced.py b/Examples/Physics_applications/beam_beam_collision/plot_reduced.py new file mode 100644 index 00000000000..3f59f975519 --- /dev/null +++ b/Examples/Physics_applications/beam_beam_collision/plot_reduced.py @@ -0,0 +1,48 @@ +import matplotlib.pyplot as plt +import numpy as np +import pandas as pd +from scipy.constants import c, nano, physical_constants + +r_e = physical_constants["classical electron radius"][0] +my_dpi = 300 +sigmaz = 10 * nano + +fig, ax = plt.subplots( + ncols=2, nrows=1, figsize=(2000.0 / my_dpi, 1000.0 / my_dpi), dpi=my_dpi +) + +rdir = "./diags/reducedfiles/" + +df_cr = pd.read_csv(f"{rdir}" + "ColliderRelevant_beam1_beam2.txt", sep=" ", header=0) +df_pn = pd.read_csv(f"{rdir}" + "ParticleNumber.txt", sep=" ", header=0) + + +times = df_cr[[col for col in df_cr.columns if "]time" in col]].to_numpy() +steps = df_cr[[col for col in df_cr.columns if "]step" in col]].to_numpy() + +x = df_cr[[col for col in df_cr.columns if "]dL_dt" in col]].to_numpy() +coll_index = np.argmax(x) +coll_time = times[coll_index] + +# number of photons per beam particle +np1 = df_pn[[col for col in df_pn.columns if "]pho1_weight" in col]].to_numpy() +np2 = df_pn[[col for col in df_pn.columns if "]pho2_weight" in col]].to_numpy() +Ne = df_pn[[col for col in df_pn.columns if "]beam1_weight" in col]].to_numpy()[0] +Np = df_pn[[col for col in df_pn.columns if "]beam2_weight" in col]].to_numpy()[0] + +ax[0].plot((times - coll_time) / (sigmaz / c), (np1 + np2) / (Ne + Np), lw=2) +ax[0].set_title(r"photon number/beam particle") + +# number of NLBW particles per beam particle +e1 = df_pn[[col for col in df_pn.columns if "]ele1_weight" in col]].to_numpy() +e2 = df_pn[[col for col in df_pn.columns if "]ele2_weight" in col]].to_numpy() + +ax[1].plot((times - coll_time) / (sigmaz / c), (e1 + e2) / (Ne + Np), lw=2) +ax[1].set_title(r"NLBW particles/beam particle") + +for a in ax.reshape(-1): + a.set_xlabel(r"time [$\sigma_z/c$]") +image_file_name = "reduced.png" +plt.tight_layout() +plt.savefig(image_file_name, dpi=300, bbox_inches="tight") +plt.close("all") From 402d7549121b15a15aa469a2135c0753177a18ed Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 18 Sep 2024 05:31:46 -0700 Subject: [PATCH 08/13] Docs: how to use `add_subdirectory` for new tests (#5279) --- Docs/source/developers/testing.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Docs/source/developers/testing.rst b/Docs/source/developers/testing.rst index 5bbc7d0fef4..ee5c82aeea9 100644 --- a/Docs/source/developers/testing.rst +++ b/Docs/source/developers/testing.rst @@ -175,6 +175,8 @@ If you need a new Python package dependency for testing, please add it in `Regre Sometimes two or more tests share a large number of input parameters. The shared input parameters can be collected in a "base" input file that can be passed as a runtime parameter in the actual test input files through the parameter ``FILE``. +If the new test is added in a new directory that did not exist before, please add the name of that directory with the command ``add_subdirectory`` in `Physics_applications/CMakeLists.txt `__ or `Tests/CMakeLists.txt `__, depending on where the new test directory is located. + Naming conventions for automated tests -------------------------------------- From 091c8d6571d359fc88ce3086ebf4196873797e14 Mon Sep 17 00:00:00 2001 From: Remi Lehe Date: Wed, 18 Sep 2024 06:03:33 -0700 Subject: [PATCH 09/13] Copy rho from guard cells in IGF solver (#5284) * Copy rho from guard cells in IGF solver * Reset checksum --- .../test_3d_open_bc_poisson_solver.json | 24 +++++++++---------- .../fields/IntegratedGreenFunctionSolver.cpp | 2 +- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json index e4ff1fc68a8..0453481ec60 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json @@ -1,20 +1,20 @@ { "lev=0": { - "Bx": 100915933.44993827, - "By": 157610622.1855512, - "Bz": 9.717358898362187e-14, - "Ex": 4.7250652706211096e+16, - "Ey": 3.0253948990559976e+16, - "Ez": 3276573.9514776524, - "rho": 10994013582437.193 + "Bx": 100915975.15792876, + "By": 157610677.31483692, + "Bz": 2.404060922276648e-13, + "Ex": 4.725066923361703e+16, + "Ey": 3.0253961494347724e+16, + "Ez": 3276584.4383433666, + "rho": 10994013582437.197 }, "electron": { - "particle_momentum_x": 5.701277606050295e-19, - "particle_momentum_y": 3.6504516641520437e-19, + "particle_momentum_x": 5.701279599504008e-19, + "particle_momentum_y": 3.650453172860547e-19, "particle_momentum_z": 1.145432768297242e-10, - "particle_position_x": 17.314086912497864, - "particle_position_y": 0.2583691267187796, + "particle_position_x": 17.31408691249785, + "particle_position_y": 0.2583691267187801, "particle_position_z": 10066.329600000008, "particle_weight": 19969036501.910976 } -} \ No newline at end of file +} diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 0767ecfb2f3..1aeee9d81d2 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -75,7 +75,7 @@ computePhiIGF ( amrex::MultiFab const & rho, SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); // Copy from rho to tmp_rho - tmp_rho.ParallelCopy( rho, 0, 0, 1, amrex::IntVect::TheZeroVector(), amrex::IntVect::TheZeroVector() ); + tmp_rho.ParallelCopy( rho, 0, 0, 1, rho.nGrowVect(), amrex::IntVect::TheZeroVector() ); // Compute the integrated Green function { From f8f4e5783e94299160fa337e8b37a99d8958b091 Mon Sep 17 00:00:00 2001 From: Thomas Marks Date: Wed, 18 Sep 2024 13:36:08 -0400 Subject: [PATCH 10/13] CFL-limited adaptive timestepping for electrostatic solver (#5176) * Improve maxParticleVelocity * gpu fix 1 * gpu fix 2 * gpu fix 3 - change to general reduction * Remove seperate OMP bit * Add const qualifier for clang tidy * Refactor maxVelocity function * Add dt_next parameter to all Evolve + PushPX methods * Write first pass update dt function * First pass implementation works, need to add logic to only use it when ES simulation enabled * Only call update dt if no const dt supplied and solver is electrostatic * No longer errors if const_dt not specified for ES solver. still need to add CFL input to picmi * warpx_cfl option added to picmi * Address most comments from @dpgrote * Move timestep update to start of Evolve * Untrack accidentally-tracked files * Add max_dt param and handle zero-velocity cases * Fix time-centering of leapfrog push * Fix max dt picmi input * Change condition for updating dt * Fix some more picmi stuff * Add timestep diagnostic to docs * Add timestep diagnostic to picmi * Initialize dt_next on restart * Add electrostatic sphere test using adaptive timestep * Fix clang-tidy errors * ruff reformatting * begin refactoring based on feedback * finish refactor * remove unused variable in WarpX.H * Update checksum for new test * Address comments --------- Co-authored-by: Remi Lehe --- Docs/source/usage/parameters.rst | 65 ++++++++------ .../Tests/electrostatic_sphere/CMakeLists.txt | 10 +++ ...puts_test_3d_electrostatic_sphere_adaptive | 47 +++++++++++ Python/pywarpx/picmi.py | 22 ++++- ...test_3d_electrostatic_sphere_adaptive.json | 17 ++++ .../Diagnostics/ReducedDiags/CMakeLists.txt | 17 ++-- Source/Diagnostics/ReducedDiags/Make.package | 24 +++--- .../ReducedDiags/MultiReducedDiags.cpp | 24 +++--- Source/Diagnostics/ReducedDiags/Timestep.H | 35 ++++++++ Source/Diagnostics/ReducedDiags/Timestep.cpp | 72 ++++++++++++++++ Source/Evolve/WarpXComputeDt.cpp | 84 +++++++++++++------ Source/Evolve/WarpXEvolve.cpp | 55 +++++++----- Source/WarpX.H | 14 ++++ Source/WarpX.cpp | 7 ++ 14 files changed, 391 insertions(+), 102 deletions(-) create mode 100644 Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive create mode 100644 Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json create mode 100644 Source/Diagnostics/ReducedDiags/Timestep.H create mode 100644 Source/Diagnostics/ReducedDiags/Timestep.cpp diff --git a/Docs/source/usage/parameters.rst b/Docs/source/usage/parameters.rst index 86ab7594c5f..b9d82d5014a 100644 --- a/Docs/source/usage/parameters.rst +++ b/Docs/source/usage/parameters.rst @@ -2129,14 +2129,24 @@ Time step The ratio between the actual timestep that is used in the simulation and the Courant-Friedrichs-Lewy (CFL) limit. (e.g. for `warpx.cfl=1`, the timestep will be exactly equal to the CFL limit.) - This parameter will only be used with the electromagnetic solver. + For some speed v and grid spacing dx, this limits the timestep to `warpx.cfl * dx / v`. + When used with the electromagnetic solver, `v` is the speed of light. + For the electrostatic solver, `v` is the maximum speed among all particles in the domain. * ``warpx.const_dt`` (`float`) Allows direct specification of the time step size, in units of seconds. - When the electrostatic solver is being used, this must be supplied. + When the electrostatic solver is being used, this must be supplied if not using adaptive timestepping. This can be used with the electromagnetic solver, overriding ``warpx.cfl``, but it is up to the user to ensure that the CFL condition is met. +* ``warpx.dt_update_interval`` (`string`) optional (default `-1`) + How many iterations pass between timestep adaptations when using the electrostatic solver. + Must be greater than `0` to use adaptive timestepping, or else ``warpx.const_dt`` must be specified. + +* ``warpx.max_dt`` (`float`) optional + The maximum timestep permitted for the electrostatic solver, when using adaptive timestepping. + If supplied, also sets the initial timestep for these simulations, before the first timestep update. + Filtering ^^^^^^^^^ @@ -3448,39 +3458,42 @@ Reduced Diagnostics For 1D-Z, :math:`x`-related and :math:`y`-related quantities are not outputted. RZ geometry is not supported yet. -* ``DifferentialLuminosity`` - This type computes the differential luminosity between two species, defined as: + * ``DifferentialLuminosity`` + This type computes the differential luminosity between two species, defined as: - .. math:: + .. math:: + + \frac{d\mathcal{L}}{d\mathcal{E}^*}(\mathcal{E}^*, t) = \int_0^t dt'\int d\boldsymbol{x}\,d\boldsymbol{p}_1 d\boldsymbol{p}_2\; + \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(\mathcal{E}^* - \mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2)) - \frac{d\mathcal{L}}{d\mathcal{E}^*}(\mathcal{E}^*, t) = \int_0^t dt'\int d\boldsymbol{x}\,d\boldsymbol{p}_1 d\boldsymbol{p}_2\; - \sqrt{ |\boldsymbol{v}_1 - \boldsymbol{v}_2|^2 - |\boldsymbol{v}_1\times\boldsymbol{v}_2|^2/c^2} \\ f_1(\boldsymbol{x}, \boldsymbol{p}_1, t')f_2(\boldsymbol{x}, \boldsymbol{p}_2, t') \delta(\mathcal{E}^* - \mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2)) + where :math:`\mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2) = \sqrt{m_1^2c^4 + m_2^2c^4 + 2(m_1 m_2 c^4 + \gamma_1 \gamma_2 - \boldsymbol{p}_1\cdot\boldsymbol{p}_2 c^2)}` is the energy in the center-of-mass frame, + and :math:`f_i` is the distribution function of species :math:`i`. Note that, if :math:`\sigma^*(\mathcal{E}^*)` + is the center-of-mass cross-section of a given collision process, then + :math:`\int d\mathcal{E}^* \frac{d\mathcal{L}}{d\mathcal{E}^*} (\mathcal{E}^*, t)\sigma^*(\mathcal{E}^*)` + gives the total number of collisions of that process (from the beginning of the simulation up until time :math:`t`). - where :math:`\mathcal{E}^*(\boldsymbol{p}_1, \boldsymbol{p}_2) = \sqrt{m_1^2c^4 + m_2^2c^4 + 2(m_1 m_2 c^4 - \gamma_1 \gamma_2 - \boldsymbol{p}_1\cdot\boldsymbol{p}_2 c^2)}` is the energy in the center-of-mass frame, - and :math:`f_i` is the distribution function of species :math:`i`. Note that, if :math:`\sigma^*(\mathcal{E}^*)` - is the center-of-mass cross-section of a given collision process, then - :math:`\int d\mathcal{E}^* \frac{d\mathcal{L}}{d\mathcal{E}^*} (\mathcal{E}^*, t)\sigma^*(\mathcal{E}^*)` - gives the total number of collisions of that process (from the beginning of the simulation up until time :math:`t`). + The differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-1}`. For collider-relevant WarpX simulations + involving two crossing, high-energy beams of particles, the differential luminosity in :math:`\text{s}^{-1}.\text{m}^{-2}.\text{eV}^{-1}` + can be obtained by multiplying the above differential luminosity by the expected repetition rate of the beams. - The differential luminosity is given in units of :math:`\text{m}^{-2}.\text{eV}^{-1}`. For collider-relevant WarpX simulations - involving two crossing, high-energy beams of particles, the differential luminosity in :math:`\text{s}^{-1}.\text{m}^{-2}.\text{eV}^{-1}` - can be obtained by multiplying the above differential luminosity by the expected repetition rate of the beams. + In practice, the above expression of the differential luminosity is evaluated over discrete bins in energy :math:`\mathcal{E}^*`, + and by summing over macroparticles. - In practice, the above expression of the differential luminosity is evaluated over discrete bins in energy :math:`\mathcal{E}^*`, - and by summing over macroparticles. + * ``.species`` (`list of two strings`) + The names of the two species for which the differential luminosity is computed. - * ``.species`` (`list of two strings`) - The names of the two species for which the differential luminosity is computed. + * ``.bin_number`` (`int` > 0) + The number of bins in energy :math:`\mathcal{E}^*` - * ``.bin_number`` (`int` > 0) - The number of bins in energy :math:`\mathcal{E}^*` + * ``.bin_max`` (`float`, in eV) + The minimum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. - * ``.bin_max`` (`float`, in eV) - The minimum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. + * ``.bin_min`` (`float`, in eV) + The maximum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. - * ``.bin_min`` (`float`, in eV) - The maximum value of :math:`\mathcal{E}^*` for which the differential luminosity is computed. + * ``Timestep`` + This type outputs the simulation's physical timestep (in seconds) at each mesh refinement level. * ``.intervals`` (`string`) Using the `Intervals Parser`_ syntax, this string defines the timesteps at which reduced diff --git a/Examples/Tests/electrostatic_sphere/CMakeLists.txt b/Examples/Tests/electrostatic_sphere/CMakeLists.txt index 41a151b7884..3d17c4462f8 100644 --- a/Examples/Tests/electrostatic_sphere/CMakeLists.txt +++ b/Examples/Tests/electrostatic_sphere/CMakeLists.txt @@ -41,6 +41,16 @@ add_warpx_test( OFF # dependency ) +add_warpx_test( + test_3d_electrostatic_sphere_adaptive # name + 3 # dims + 2 # nprocs + inputs_test_3d_electrostatic_sphere_adaptive # inputs + analysis_electrostatic_sphere.py # analysis + diags/diag1000054 # output + OFF # dependency +) + add_warpx_test( test_rz_electrostatic_sphere # name RZ # dims diff --git a/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive new file mode 100644 index 00000000000..f64f6de08ee --- /dev/null +++ b/Examples/Tests/electrostatic_sphere/inputs_test_3d_electrostatic_sphere_adaptive @@ -0,0 +1,47 @@ +stop_time = 60e-6 +warpx.cfl = 0.2 +warpx.dt_update_interval = 10 +warpx.max_dt = 1.5e-6 +amr.n_cell = 64 64 64 +amr.max_level = 0 +amr.blocking_factor = 8 +amr.max_grid_size = 128 +geometry.dims = 3 +geometry.prob_lo = -0.5 -0.5 -0.5 +geometry.prob_hi = 0.5 0.5 0.5 +boundary.field_lo = pec pec pec +boundary.field_hi = pec pec pec +warpx.do_electrostatic = relativistic + +particles.species_names = electron + +algo.field_gathering = momentum-conserving + +# Order of particle shape factors +algo.particle_shape = 1 + +my_constants.n0 = 1.49e6 +my_constants.R0 = 0.1 + +electron.charge = -q_e +electron.mass = m_e +electron.injection_style = "NUniformPerCell" +electron.num_particles_per_cell_each_dim = 2 2 2 +electron.profile = parse_density_function +electron.density_function(x,y,z) = "(x*x + y*y + z*z < R0*R0)*n0" +electron.momentum_distribution_type = at_rest + +diagnostics.diags_names = diag1 diag2 + +diag1.intervals = 30 +diag1.diag_type = Full +diag1.fields_to_plot = Ex Ey Ez rho + +diag2.intervals = 30 +diag2.diag_type = Full +diag2.fields_to_plot = none +diag2.format = openpmd + +warpx.reduced_diags_names = timestep +timestep.intervals = 1 +timestep.type = Timestep diff --git a/Python/pywarpx/picmi.py b/Python/pywarpx/picmi.py index 0d51a8723b4..478b4d5802e 100644 --- a/Python/pywarpx/picmi.py +++ b/Python/pywarpx/picmi.py @@ -1522,8 +1522,7 @@ def solver_initialize_inputs(self): # --- Same method names are used, though mapped to lower case. pywarpx.algo.maxwell_solver = self.method - if self.cfl is not None: - pywarpx.warpx.cfl = self.cfl + pywarpx.warpx.cfl = self.cfl if self.source_smoother is not None: self.source_smoother.smoother_initialize_inputs(self) @@ -1880,6 +1879,16 @@ class ElectrostaticSolver(picmistandard.PICMI_ElectrostaticSolver): warpx_self_fields_verbosity: integer, default=2 Level of verbosity for the lab frame solver + + warpx_dt_update_interval: string, optional (default = -1) + How frequently the timestep is updated. Adaptive timestepping is disabled when this is <= 0. + + warpx_cfl: float, optional + Fraction of the CFL condition for particle velocity vs grid size, used to set the timestep when `dt_update_interval > 0`. + + warpx_max_dt: float, optional + The maximum allowable timestep when `dt_update_interval > 0`. + """ def init(self, kw): @@ -1887,6 +1896,9 @@ def init(self, kw): self.absolute_tolerance = kw.pop("warpx_absolute_tolerance", None) self.self_fields_verbosity = kw.pop("warpx_self_fields_verbosity", None) self.magnetostatic = kw.pop("warpx_magnetostatic", False) + self.cfl = kw.pop("warpx_cfl", None) + self.dt_update_interval = kw.pop("dt_update_interval", None) + self.max_dt = kw.pop("warpx_max_dt", None) def solver_initialize_inputs(self): # Open BC means FieldBoundaryType::Open for electrostatic sims, rather than perfectly-matched layer @@ -1894,6 +1906,11 @@ def solver_initialize_inputs(self): self.grid.grid_initialize_inputs() + # set adaptive timestepping parameters + pywarpx.warpx.cfl = self.cfl + pywarpx.warpx.dt_update_interval = self.dt_update_interval + pywarpx.warpx.max_dt = self.max_dt + if self.relativistic: pywarpx.warpx.do_electrostatic = "relativistic" else: @@ -3890,6 +3907,7 @@ def __init__( "ParticleNumber", "LoadBalanceCosts", "LoadBalanceEfficiency", + "Timestep", ] # The species diagnostics require a species to be provided self._species_reduced_diagnostics = [ diff --git a/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json new file mode 100644 index 00000000000..561fbf86669 --- /dev/null +++ b/Regression/Checksum/benchmarks_json/test_3d_electrostatic_sphere_adaptive.json @@ -0,0 +1,17 @@ +{ + "lev=0": { + "Ex": 5.177444767224255, + "Ey": 5.177444767224254, + "Ez": 5.177444767224256, + "rho": 2.6092568008333797e-10 + }, + "electron": { + "particle_momentum_x": 1.3215019655285216e-23, + "particle_momentum_y": 1.3215019655285214e-23, + "particle_momentum_z": 1.3215019655285217e-23, + "particle_position_x": 912.2310003741203, + "particle_position_y": 912.2310003741203, + "particle_position_z": 912.2310003741202, + "particle_weight": 6212.501525878906 + } +} diff --git a/Source/Diagnostics/ReducedDiags/CMakeLists.txt b/Source/Diagnostics/ReducedDiags/CMakeLists.txt index 4f0b05f6180..bbf1b6b65b0 100644 --- a/Source/Diagnostics/ReducedDiags/CMakeLists.txt +++ b/Source/Diagnostics/ReducedDiags/CMakeLists.txt @@ -3,26 +3,27 @@ foreach(D IN LISTS WarpX_DIMS) target_sources(lib_${SD} PRIVATE BeamRelevant.cpp + ChargeOnEB.cpp ColliderRelevant.cpp DifferentialLuminosity.cpp FieldEnergy.cpp + FieldMaximum.cpp + FieldMomentum.cpp FieldProbe.cpp FieldProbeParticleContainer.cpp - FieldMomentum.cpp + FieldReduction.cpp + FieldProbe.cpp LoadBalanceCosts.cpp LoadBalanceEfficiency.cpp MultiReducedDiags.cpp ParticleEnergy.cpp - ParticleMomentum.cpp + ParticleExtrema.cpp ParticleHistogram.cpp ParticleHistogram2D.cpp + ParticleMomentum.cpp + ParticleNumber.cpp ReducedDiags.cpp - FieldMaximum.cpp - ParticleExtrema.cpp RhoMaximum.cpp - ParticleNumber.cpp - FieldReduction.cpp - FieldProbe.cpp - ChargeOnEB.cpp + Timestep.cpp ) endforeach() diff --git a/Source/Diagnostics/ReducedDiags/Make.package b/Source/Diagnostics/ReducedDiags/Make.package index e840931f8d3..2611831a3dd 100644 --- a/Source/Diagnostics/ReducedDiags/Make.package +++ b/Source/Diagnostics/ReducedDiags/Make.package @@ -1,24 +1,24 @@ CEXE_sources += MultiReducedDiags.cpp CEXE_sources += ReducedDiags.cpp -CEXE_sources += ParticleEnergy.cpp -CEXE_sources += ParticleMomentum.cpp -CEXE_sources += FieldEnergy.cpp -CEXE_sources += FieldProbe.cpp -CEXE_sources += FieldProbeParticleContainer.cpp -CEXE_sources += FieldMomentum.cpp CEXE_sources += BeamRelevant.cpp +CEXE_sources += ChargeOnEB.cpp CEXE_sources += ColliderRelevant.cpp CEXE_sources += DifferentialLuminosity.cpp +CEXE_sources += FieldEnergy.cpp +CEXE_sources += FieldMaximum.cpp +CEXE_sources += FieldMomentum.cpp +CEXE_sources += FieldProbe.cpp +CEXE_sources += FieldProbeParticleContainer.cpp +CEXE_sources += FieldReduction.cpp CEXE_sources += LoadBalanceCosts.cpp CEXE_sources += LoadBalanceEfficiency.cpp +CEXE_sources += ParticleEnergy.cpp +CEXE_sources += ParticleExtrema.cpp CEXE_sources += ParticleHistogram.cpp CEXE_sources += ParticleHistogram2D.cpp -CEXE_sources += FieldMaximum.cpp -CEXE_sources += FieldProbe.cpp -CEXE_sources += ParticleExtrema.cpp -CEXE_sources += RhoMaximum.cpp +CEXE_sources += ParticleMomentum.cpp CEXE_sources += ParticleNumber.cpp -CEXE_sources += FieldReduction.cpp -CEXE_sources += ChargeOnEB.cpp +CEXE_sources += RhoMaximum.cpp +CEXE_sources += Timestep.cpp VPATH_LOCATIONS += $(WARPX_HOME)/Source/Diagnostics/ReducedDiags diff --git a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp index 25ea87d9f54..5035eac58a8 100644 --- a/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp +++ b/Source/Diagnostics/ReducedDiags/MultiReducedDiags.cpp @@ -12,8 +12,8 @@ #include "DifferentialLuminosity.H" #include "FieldEnergy.H" #include "FieldMaximum.H" -#include "FieldProbe.H" #include "FieldMomentum.H" +#include "FieldProbe.H" #include "FieldReduction.H" #include "LoadBalanceCosts.H" #include "LoadBalanceEfficiency.H" @@ -24,6 +24,7 @@ #include "ParticleMomentum.H" #include "ParticleNumber.H" #include "RhoMaximum.H" +#include "Timestep.H" #include "Utils/TextMsg.H" #include "Utils/WarpXProfilerWrapper.H" @@ -52,24 +53,25 @@ MultiReducedDiags::MultiReducedDiags () using CS = const std::string& ; const auto reduced_diags_dictionary = std::map(CS)>>{ + {"BeamRelevant", [](CS s){return std::make_unique(s);}}, + {"ChargeOnEB", [](CS s){return std::make_unique(s);}}, + {"ColliderRelevant", [](CS s){return std::make_unique(s);}}, + {"DifferentialLuminosity",[](CS s){return std::make_unique(s);}}, {"ParticleEnergy", [](CS s){return std::make_unique(s);}}, + {"ParticleExtrema", [](CS s){return std::make_unique(s);}}, + {"ParticleHistogram", [](CS s){return std::make_unique(s);}}, + {"ParticleHistogram2D", [](CS s){return std::make_unique(s);}}, {"ParticleMomentum", [](CS s){return std::make_unique(s);}}, + {"ParticleNumber", [](CS s){return std::make_unique(s);}}, {"FieldEnergy", [](CS s){return std::make_unique(s);}}, - {"FieldMomentum", [](CS s){return std::make_unique(s);}}, {"FieldMaximum", [](CS s){return std::make_unique(s);}}, + {"FieldMomentum", [](CS s){return std::make_unique(s);}}, {"FieldProbe", [](CS s){return std::make_unique(s);}}, {"FieldReduction", [](CS s){return std::make_unique(s);}}, - {"RhoMaximum", [](CS s){return std::make_unique(s);}}, - {"BeamRelevant", [](CS s){return std::make_unique(s);}}, - {"ColliderRelevant", [](CS s){return std::make_unique(s);}}, - {"DifferentialLuminosity",[](CS s){return std::make_unique(s);}}, {"LoadBalanceCosts", [](CS s){return std::make_unique(s);}}, {"LoadBalanceEfficiency", [](CS s){return std::make_unique(s);}}, - {"ParticleHistogram", [](CS s){return std::make_unique(s);}}, - {"ParticleHistogram2D", [](CS s){return std::make_unique(s);}}, - {"ParticleNumber", [](CS s){return std::make_unique(s);}}, - {"ParticleExtrema", [](CS s){return std::make_unique(s);}}, - {"ChargeOnEB", [](CS s){return std::make_unique(s);}} + {"RhoMaximum", [](CS s){return std::make_unique(s);}}, + {"Timestep", [](CS s){return std::make_unique(s);}} }; // loop over all reduced diags and fill m_multi_rd with requested reduced diags std::transform(m_rd_names.begin(), m_rd_names.end(), std::back_inserter(m_multi_rd), diff --git a/Source/Diagnostics/ReducedDiags/Timestep.H b/Source/Diagnostics/ReducedDiags/Timestep.H new file mode 100644 index 00000000000..bcf4fe6452f --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/Timestep.H @@ -0,0 +1,35 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Thomas Marks + * + * License: BSD-3-Clause-LBNL + */ + +#ifndef WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ +#define WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ + +#include "ReducedDiags.H" +#include + +/** + * This class contains a function for retrieving the current simulation timestep as a diagnostic. + * Useful mainly for simulations using adaptive timestepping. + */ +class Timestep: public ReducedDiags { +public: + /** + * constructor + * @param[in] rd_name reduced diags name + */ + Timestep (const std::string& rd_name); + + /** + * This function gets the current physical timestep of the simulation at all refinement levels. + * @param[in] step current time step + */ + void ComputeDiags (int step) final; +}; + +#endif //WARPX_DIAGNOSTICS_REDUCEDDIAGS_TIMESTEP_H_ diff --git a/Source/Diagnostics/ReducedDiags/Timestep.cpp b/Source/Diagnostics/ReducedDiags/Timestep.cpp new file mode 100644 index 00000000000..3474121db91 --- /dev/null +++ b/Source/Diagnostics/ReducedDiags/Timestep.cpp @@ -0,0 +1,72 @@ +/* Copyright 2024 The WarpX Community + * + * This file is part of WarpX. + * + * Authors: Thomas Marks + * + * License: BSD-3-Clause-LBNL + */ + +#include "Timestep.H" + +#include "WarpX.H" + +#include +#include +#include // TODO: remove this +#include + +#include + +using namespace amrex::literals; + +// constructor +Timestep::Timestep (const std::string& rd_name) +:ReducedDiags{rd_name} +{ + const auto& warpx = WarpX::GetInstance(); + const auto max_level = warpx.maxLevel(); + + // data size should be equal to the number of refinement levels + m_data.resize(max_level + 1, 0.0_rt); + + if (amrex::ParallelDescriptor::IOProcessor() && m_write_header) { + // open file + std::ofstream ofs{m_path + m_rd_name + "." + m_extension, std::ofstream::out}; + + // write header row + int c = 0; + ofs << "#"; + ofs << "[" << c++ << "]step()"; + ofs << m_sep; + ofs << "[" << c++ << "]time(s)"; + ofs << m_sep; + + for (int lev = 0; lev <= max_level; lev++) { + ofs << "[" << c++ << "]timestep[" << lev << "](s)"; + if (lev < max_level) { + ofs << m_sep; + } + } + + // close file + ofs << std::endl; + ofs.close(); + } +} +// end constructor + +// function to get current simulation timestep at all refinement levels +void Timestep::ComputeDiags (int step) { + // Check if diagnostic should be done + if (!m_intervals.contains(step+1)) { return; } + + const auto& warpx = WarpX::GetInstance(); + const auto max_level = warpx.maxLevel(); + const auto dt = warpx.getdt(); + + for (int lev = 0; lev <= max_level; lev++) { + m_data[lev] = dt[lev]; + } +} +// end Timestep::ComputeDiags diff --git a/Source/Evolve/WarpXComputeDt.cpp b/Source/Evolve/WarpXComputeDt.cpp index c1a87166920..b82cb6aff26 100644 --- a/Source/Evolve/WarpXComputeDt.cpp +++ b/Source/Evolve/WarpXComputeDt.cpp @@ -13,6 +13,7 @@ #else # include "FieldSolver/FiniteDifferenceSolver/FiniteDifferenceAlgorithms/CylindricalYeeAlgorithm.H" #endif +#include "Particles/MultiParticleContainer.H" #include "Utils/TextMsg.H" #include "Utils/WarpXAlgorithmSelection.H" #include "Utils/WarpXConst.H" @@ -27,29 +28,29 @@ #include #include +/** + * Compute the minimum of array x, where x has dimension AMREX_SPACEDIM + */ +AMREX_FORCE_INLINE amrex::Real +minDim (const amrex::Real* x) +{ + return std::min({AMREX_D_DECL(x[0], x[1], x[2])}); +} + /** * Determine the timestep of the simulation. */ void WarpX::ComputeDt () { // Handle cases where the timestep is not limited by the speed of light - if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None || - electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - - std::stringstream errorMsg; - if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { - errorMsg << "warpx.const_dt must be specified with the electrostatic solver."; - } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { - errorMsg << "warpx.const_dt must be specified with the hybrid-PIC solver."; - } else { - errorMsg << "warpx.const_dt must be specified when not using a field solver."; - } - WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), errorMsg.str()); - - for (int lev=0; lev<=max_level; lev++) { - dt[lev] = m_const_dt.value(); - } - return; + // and no constant timestep is provided + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::HybridPIC) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_const_dt.has_value(), "warpx.const_dt must be specified with the hybrid-PIC solver."); + } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE( + m_const_dt.has_value() || dt_update_interval.isActivated(), + "warpx.const_dt must be specified with the electrostatic solver, or warpx.dt_update_interval must be > 0." + ); } // Determine the appropriate timestep as limited by the speed of light @@ -58,16 +59,17 @@ WarpX::ComputeDt () if (m_const_dt.has_value()) { deltat = m_const_dt.value(); + } else if (electrostatic_solver_id != ElectrostaticSolverAlgo::None) { + // Set dt for electrostatic algorithm + if (m_max_dt.has_value()) { + deltat = m_max_dt.value(); + } else { + deltat = cfl * minDim(dx) / PhysConst::c; + } } else if (electromagnetic_solver_id == ElectromagneticSolverAlgo::PSATD) { // Computation of dt for spectral algorithm // (determined by the minimum cell size in all directions) -#if defined(WARPX_DIM_1D_Z) - deltat = cfl * dx[0] / PhysConst::c; -#elif defined(WARPX_DIM_XZ) || defined(WARPX_DIM_RZ) - deltat = cfl * std::min(dx[0], dx[1]) / PhysConst::c; -#else - deltat = cfl * std::min(dx[0], std::min(dx[1], dx[2])) / PhysConst::c; -#endif + deltat = cfl * minDim(dx) / PhysConst::c; } else { // Computation of dt for FDTD algorithm #ifdef WARPX_DIM_RZ @@ -99,6 +101,40 @@ WarpX::ComputeDt () } } +/** + * Determine the simulation timestep from the maximum speed of all particles + * Sets timestep so that a particle can only cross cfl*dx cells per timestep. + */ +void +WarpX::UpdateDtFromParticleSpeeds () +{ + const amrex::Real* dx = geom[max_level].CellSize(); + const amrex::Real dx_min = minDim(dx); + + const amrex::ParticleReal max_v = mypc->maxParticleVelocity(); + amrex::Real deltat_new = 0.; + + // Protections from overly-large timesteps + if (max_v == 0) { + WARPX_ALWAYS_ASSERT_WITH_MESSAGE(m_max_dt.has_value(), "Particles at rest and no constant or maximum timestep specified. Aborting."); + deltat_new = m_max_dt.value(); + } else { + deltat_new = cfl * dx_min / max_v; + } + + // Restrict to be less than user-specified maximum timestep, if present + if (m_max_dt.has_value()) { + deltat_new = std::min(deltat_new, m_max_dt.value()); + } + + // Update dt + dt[max_level] = deltat_new; + + for (int lev = max_level-1; lev >= 0; --lev) { + dt[lev] = dt[lev+1] * refRatio(lev)[0]; + } +} + void WarpX::PrintDtDxDyDz () { diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index c668eac2e26..9acbe734405 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -60,6 +60,27 @@ using namespace amrex; using ablastr::utils::SignalHandling; +void +WarpX::Synchronize () { + FillBoundaryE(guard_cells.ng_FieldGather); + FillBoundaryB(guard_cells.ng_FieldGather); + if (fft_do_time_averaging) + { + FillBoundaryE_avg(guard_cells.ng_FieldGather); + FillBoundaryB_avg(guard_cells.ng_FieldGather); + } + UpdateAuxilaryData(); + FillBoundaryAux(guard_cells.ng_UpdateAux); + for (int lev = 0; lev <= finest_level; ++lev) { + mypc->PushP(lev, 0.5_rt*dt[lev], + *Efield_aux[lev][0],*Efield_aux[lev][1], + *Efield_aux[lev][2], + *Bfield_aux[lev][0],*Bfield_aux[lev][1], + *Bfield_aux[lev][2]); + } + is_synchronized = true; +} + void WarpX::Evolve (int numsteps) { @@ -95,6 +116,18 @@ WarpX::Evolve (int numsteps) CheckLoadBalance(step); + // Update timestep for electrostatic solver if a constant dt is not provided + // This first synchronizes the position and velocity before setting the new timestep + if (electromagnetic_solver_id == ElectromagneticSolverAlgo::None && + !m_const_dt.has_value() && dt_update_interval.contains(step+1)) { + if (verbose) { + amrex::Print() << Utils::TextMsg::Info("updating timestep"); + } + Synchronize(); + UpdateDtFromParticleSpeeds(); + } + + // If position and velocity are synchronized, push velocity backward one half step if (evolve_scheme == EvolveScheme::Explicit) { ExplicitFillBoundaryEBUpdateAux(); @@ -175,25 +208,9 @@ WarpX::Evolve (int numsteps) // TODO: move out if (evolve_scheme == EvolveScheme::Explicit) { + // At the end of last step, push p by 0.5*dt to synchronize if (cur_time + dt[0] >= stop_time - 1.e-3*dt[0] || step == numsteps_max-1) { - // At the end of last step, push p by 0.5*dt to synchronize - FillBoundaryE(guard_cells.ng_FieldGather); - FillBoundaryB(guard_cells.ng_FieldGather); - if (fft_do_time_averaging) - { - FillBoundaryE_avg(guard_cells.ng_FieldGather); - FillBoundaryB_avg(guard_cells.ng_FieldGather); - } - UpdateAuxilaryData(); - FillBoundaryAux(guard_cells.ng_UpdateAux); - for (int lev = 0; lev <= finest_level; ++lev) { - mypc->PushP(lev, 0.5_rt*dt[lev], - *Efield_aux[lev][0],*Efield_aux[lev][1], - *Efield_aux[lev][2], - *Bfield_aux[lev][0],*Bfield_aux[lev][1], - *Bfield_aux[lev][2]); - } - is_synchronized = true; + Synchronize(); } } @@ -445,7 +462,7 @@ void WarpX::checkEarlyUnusedParams () void WarpX::ExplicitFillBoundaryEBUpdateAux () { WARPX_ALWAYS_ASSERT_WITH_MESSAGE(evolve_scheme == EvolveScheme::Explicit, - "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux wihtout Explicit evolve scheme set!"); + "Cannot call WarpX::ExplicitFillBoundaryEBUpdateAux without Explicit evolve scheme set!"); // At the beginning, we have B^{n} and E^{n}. // Particles have p^{n} and x^{n}. diff --git a/Source/WarpX.H b/Source/WarpX.H index 28bb6215a45..5065fa73ff9 100644 --- a/Source/WarpX.H +++ b/Source/WarpX.H @@ -117,6 +117,11 @@ public: void Evolve (int numsteps = -1); + /** Push momentum one half step forward to synchronize with position. + * Also sets is_synchronized to `true`. + */ + void Synchronize (); + // // Functions used by implicit solvers // @@ -606,6 +611,12 @@ public: /** Determine the timestep of the simulation. */ void ComputeDt (); + /** + * Determine the simulation timestep from the maximum speed of all particles + * Sets timestep so that a particle can only cross cfl*dx cells per timestep. + */ + void UpdateDtFromParticleSpeeds (); + /** Print main PIC parameters to stdout */ void PrintMainPICparameters (); @@ -1482,6 +1493,7 @@ private: amrex::Vector t_new; amrex::Vector t_old; amrex::Vector dt; + static utils::parser::IntervalsParser dt_update_interval; // How often to update the timestep when using adaptive timestepping // Particle container std::unique_ptr mypc; @@ -1643,7 +1655,9 @@ private: int num_injected_species = -1; amrex::Vector injected_plasma_species; + // Timestepping parameters std::optional m_const_dt; + std::optional m_max_dt; // Macroscopic properties std::unique_ptr m_macroscopic_properties; diff --git a/Source/WarpX.cpp b/Source/WarpX.cpp index ef1668de4c0..e9c518e8f61 100644 --- a/Source/WarpX.cpp +++ b/Source/WarpX.cpp @@ -182,6 +182,8 @@ bool WarpX::do_multi_J = false; int WarpX::do_multi_J_n_depositions; bool WarpX::safe_guard_cells = false; +utils::parser::IntervalsParser WarpX::dt_update_interval; + std::map WarpX::multifab_map; std::map WarpX::imultifab_map; @@ -775,7 +777,12 @@ WarpX::ReadParameters () pp_boundary.query("verboncoeur_axis_correction", verboncoeur_axis_correction); #endif + // Read timestepping options utils::parser::queryWithParser(pp_warpx, "const_dt", m_const_dt); + utils::parser::queryWithParser(pp_warpx, "max_dt", m_max_dt); + std::vector dt_interval_vec = {"-1"}; + pp_warpx.queryarr("dt_update_interval", dt_interval_vec); + dt_update_interval = utils::parser::IntervalsParser(dt_interval_vec); // Filter currently not working with FDTD solver in RZ geometry: turn OFF by default // (see https://github.com/ECP-WarpX/WarpX/issues/1943) From cb5d0c80a0e29d2318c61303670b581c9a484bb2 Mon Sep 17 00:00:00 2001 From: Alfred Mishi <140518333+Haavaan@users.noreply.github.com> Date: Wed, 18 Sep 2024 19:38:55 +0200 Subject: [PATCH 11/13] Integrated Green's Function Poisson Solver (#4937) * [WIP]Integrated Green Function Poisson Solver * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * updated inputs * Modified IGF.cpp file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Latest IGF.cpp * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Fixed the IGF file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * IGF final * IGF file final * Added BL_PROFILE in IGF file * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Updated IntegratedGreenFunctionSolver * Check Compilation Flags * Added Multiplication * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * removed inputs * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp Co-authored-by: Remi Lehe * change import orders * fix macros and add timers * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update checksums and changed ifdefs * mod GNUmake * fix GNUmakefile for real * heffte ctest * fix test input name * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * Apply suggestions from code review * Update Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp * Start refactoring * Continue refactoring * Continue refactoring * Finish refactoring * Fix unused variable * Fix compilation without heFFTe * Fix bugs without heFFTe * Remove unneeded ifdef * Update checksum --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Alfred Haavaan Mishi Co-authored-by: Alfred Haavaan Mishi Co-authored-by: Alfred Haavaan Mishi Co-authored-by: Arianna Formenti Co-authored-by: Remi Lehe --- .../open_bc_poisson_solver/CMakeLists.txt | 12 + ...puts_test_3d_open_bc_poisson_solver_heffte | 1 + GNUmakefile | 1 + .../test_3d_open_bc_poisson_solver.json | 20 +- .../fields/IntegratedGreenFunctionSolver.H | 31 +++ .../fields/IntegratedGreenFunctionSolver.cpp | 258 ++++++++++++------ 6 files changed, 232 insertions(+), 91 deletions(-) create mode 100644 Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte diff --git a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt index c5ec4583da1..d6141f0b4ab 100644 --- a/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt +++ b/Examples/Tests/open_bc_poisson_solver/CMakeLists.txt @@ -12,3 +12,15 @@ if(WarpX_FFT) OFF # dependency ) endif() + +if(WarpX_HEFFTE) + add_warpx_test( + test_3d_open_bc_poisson_solver_heffte # name + 3 # dims + 2 # nprocs + inputs_test_3d_open_bc_poisson_solver_heffte # inputs + analysis.py # analysis + diags/diag1000001 # output + OFF # dependency + ) +endif() diff --git a/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte new file mode 100644 index 00000000000..4f0a50df037 --- /dev/null +++ b/Examples/Tests/open_bc_poisson_solver/inputs_test_3d_open_bc_poisson_solver_heffte @@ -0,0 +1 @@ +FILE = inputs_test_3d_open_bc_poisson_solver diff --git a/GNUmakefile b/GNUmakefile index 86bdab2709f..fe10983b780 100644 --- a/GNUmakefile +++ b/GNUmakefile @@ -38,6 +38,7 @@ USE_OPENPMD = FALSE WarpxBinDir = Bin USE_FFT = FALSE +USE_HEFFTE = FALSE USE_RZ = FALSE USE_EB = FALSE diff --git a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json index 0453481ec60..0ca6bde570a 100644 --- a/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json +++ b/Regression/Checksum/benchmarks_json/test_3d_open_bc_poisson_solver.json @@ -1,20 +1,20 @@ { "lev=0": { - "Bx": 100915975.15792876, - "By": 157610677.31483692, - "Bz": 2.404060922276648e-13, - "Ex": 4.725066923361703e+16, - "Ey": 3.0253961494347724e+16, - "Ez": 3276584.4383433666, + "Bx": 100915975.15403552, + "By": 157610677.3147734, + "Bz": 1.2276713711194638e-13, + "Ex": 4.725066923359797e+16, + "Ey": 3.025396149317578e+16, + "Ez": 3276584.4383433824, "rho": 10994013582437.197 }, "electron": { - "particle_momentum_x": 5.701279599504008e-19, - "particle_momentum_y": 3.650453172860547e-19, + "particle_momentum_x": 5.701279599509506e-19, + "particle_momentum_y": 3.650453172383178e-19, "particle_momentum_z": 1.145432768297242e-10, "particle_position_x": 17.31408691249785, - "particle_position_y": 0.2583691267187801, + "particle_position_y": 0.25836912671878015, "particle_position_z": 10066.329600000008, "particle_weight": 19969036501.910976 } -} +} \ No newline at end of file diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H index 97ffdb5ac36..28885e167a3 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.H +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.H @@ -7,6 +7,8 @@ #ifndef ABLASTR_IGF_SOLVER_H #define ABLASTR_IGF_SOLVER_H +#include + #include #include #include @@ -47,6 +49,35 @@ namespace ablastr::fields return G; } + /** @brief add + * + * @param[in] x x-coordinate of given location + * @param[in] y y-coordinate of given location + * @param[in] z z-coordinate of given location + * + * @return the sum of integrated Green function G + */ + AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE + amrex::Real + SumOfIntegratedPotential (amrex::Real x, amrex::Real y, amrex::Real z, amrex::Real dx, amrex::Real dy, amrex::Real dz) + { + using namespace amrex::literals; + + + amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( + IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) + - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) + + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) + ); + + return G_value; + } + /** @brief Compute the electrostatic potential using the Integrated Green Function method * as in http://dx.doi.org/10.1103/PhysRevSTAB.9.044204 * diff --git a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp index 1aeee9d81d2..5b9aa940d6a 100644 --- a/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp +++ b/Source/ablastr/fields/IntegratedGreenFunctionSolver.cpp @@ -27,7 +27,9 @@ #include #include -#include +#if defined(ABLASTR_USE_FFT) && defined(ABLASTR_USE_HEFFTE) +#include +#endif namespace ablastr::fields { @@ -36,10 +38,16 @@ void computePhiIGF ( amrex::MultiFab const & rho, amrex::MultiFab & phi, std::array const & cell_size, - amrex::BoxArray const & ba ) + amrex::BoxArray const & ba) { using namespace amrex::literals; + BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFTs", timer_ffts); + BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: FFT plans", timer_plans); + BL_PROFILE_VAR_NS("ablastr::fields::computePhiIGF: parallel copies", timer_pcopies); + + BL_PROFILE("ablastr::fields::computePhiIGF"); + // Define box that encompasses the full domain amrex::Box domain = ba.minimalBox(); domain.surroundingNodes(); // get nodal points, since `phi` and `rho` are nodal @@ -50,41 +58,87 @@ computePhiIGF ( amrex::MultiFab const & rho, int const nz = domain.length(2); // Allocate 2x wider arrays for the convolution of rho with the Green function - // This also defines the box arrays for the global FFT: contains only one box; amrex::Box const realspace_box = amrex::Box( {domain.smallEnd(0), domain.smallEnd(1), domain.smallEnd(2)}, {2*nx-1+domain.smallEnd(0), 2*ny-1+domain.smallEnd(1), 2*nz-1+domain.smallEnd(2)}, amrex::IntVect::TheNodeVector() ); + +#if !defined(ABLASTR_USE_HEFFTE) + // Without distributed FFTs (i.e. without heFFTe): + // allocate the 2x wider array on a single box amrex::BoxArray const realspace_ba = amrex::BoxArray( realspace_box ); - amrex::Box const spectralspace_box = amrex::Box( - {0,0,0}, - {nx, 2*ny-1, 2*nz-1}, - amrex::IntVect::TheNodeVector() ); - amrex::BoxArray const spectralspace_ba = amrex::BoxArray( spectralspace_box ); // Define a distribution mapping for the global FFT, with only one box amrex::DistributionMapping dm_global_fft; dm_global_fft.define( realspace_ba ); +#elif defined(ABLASTR_USE_HEFFTE) + // With distributed FFTs (i.e. with heFFTe): + // Define a new distribution mapping which is decomposed purely along z + // and has one box per MPI rank + int const nprocs = amrex::ParallelDescriptor::NProcs(); + amrex::BoxArray realspace_ba; + amrex::DistributionMapping dm_global_fft; + { + int realspace_nx = realspace_box.length(0); + int realspace_ny = realspace_box.length(1); + int realspace_nz = realspace_box.length(2); + int minsize_z = realspace_nz / nprocs; + int nleft_z = realspace_nz - minsize_z*nprocs; + + AMREX_ALWAYS_ASSERT(realspace_nz >= nprocs); + // We are going to split realspace_box in such a way that the first + // nleft boxes has minsize_z+1 nodes and the others minsize + // nodes. We do it this way instead of BoxArray::maxSize to make + // sure there are exactly nprocs boxes and there are no overlaps. + amrex::BoxList bl(amrex::IndexType::TheNodeType()); + for (int iproc = 0; iproc < nprocs; ++iproc) { + int zlo, zhi; + if (iproc < nleft_z) { + zlo = iproc*(minsize_z+1); + zhi = zlo + minsize_z; + + } else { + zlo = iproc*minsize_z + nleft_z; + zhi = zlo + minsize_z - 1; + + } + amrex::Box tbx(amrex::IntVect(0,0,zlo),amrex::IntVect(realspace_nx-1,realspace_ny-1,zhi),amrex::IntVect(1)); + + tbx.shift(realspace_box.smallEnd()); + bl.push_back(tbx); + } + realspace_ba.define(std::move(bl)); + amrex::Vector pmap(nprocs); + std::iota(pmap.begin(), pmap.end(), 0); + dm_global_fft.define(std::move(pmap)); + } +#endif + // Allocate required arrays amrex::MultiFab tmp_rho = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); tmp_rho.setVal(0); amrex::MultiFab tmp_G = amrex::MultiFab(realspace_ba, dm_global_fft, 1, 0); tmp_G.setVal(0); - // Allocate corresponding arrays in Fourier space - using SpectralField = amrex::FabArray< amrex::BaseFab< amrex::GpuComplex< amrex::Real > > >; - SpectralField tmp_rho_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); - SpectralField tmp_G_fft = SpectralField( spectralspace_ba, dm_global_fft, 1, 0 ); - // Copy from rho to tmp_rho + BL_PROFILE_VAR_START(timer_pcopies); + // Copy from rho including its ghost cells to tmp_rho tmp_rho.ParallelCopy( rho, 0, 0, 1, rho.nGrowVect(), amrex::IntVect::TheZeroVector() ); + BL_PROFILE_VAR_STOP(timer_pcopies); + +#if !defined(ABLASTR_USE_HEFFTE) + // Without distributed FFTs (i.e. without heFFTe): + // We loop over the original box (not the 2x wider one), and the other quadrants by periodicity + amrex::BoxArray const& igf_compute_box = amrex::BoxArray( domain ); +#else + // With distributed FFTs (i.e. with heFFTe): + // We loop over the full 2x wider box, since 1 MPI rank does not necessarily own the data for the other quadrants + amrex::BoxArray const& igf_compute_box = tmp_G.boxArray(); +#endif // Compute the integrated Green function - { - BL_PROFILE("Initialize Green function"); - amrex::BoxArray const domain_ba = amrex::BoxArray( domain ); #ifdef AMREX_USE_OMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for (amrex::MFIter mfi(domain_ba, dm_global_fft,amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { + for (amrex::MFIter mfi(igf_compute_box, dm_global_fft, amrex::TilingIfNotGPU()); mfi.isValid(); ++mfi) { amrex::Box const bx = mfi.tilebox(); @@ -95,6 +149,7 @@ computePhiIGF ( amrex::MultiFab const & rho, amrex::Real const dx = cell_size[0]; amrex::Real const dy = cell_size[1]; amrex::Real const dz = cell_size[2]; + amrex::Array4 const tmp_G_arr = tmp_G.array(mfi); amrex::ParallelFor( bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept @@ -106,17 +161,9 @@ computePhiIGF ( amrex::MultiFab const & rho, amrex::Real const y = j0*dy; amrex::Real const z = k0*dz; - amrex::Real const G_value = 1._rt/(4._rt*ablastr::constant::math::pi*ablastr::constant::SI::ep0) * ( - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x+0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x+0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x-0.5_rt*dx, y+0.5_rt*dy, z-0.5_rt*dz ) - + IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z+0.5_rt*dz ) - - IntegratedPotential( x-0.5_rt*dx, y-0.5_rt*dy, z-0.5_rt*dz ) - ); - +#if !defined(ABLASTR_USE_HEFFTE) + // Without distributed FFTs (i.e. without heFFTe): + amrex::Real const G_value = SumOfIntegratedPotential(x , y , z , dx, dy, dz); tmp_G_arr(i,j,k) = G_value; // Fill the rest of the array by periodicity if (i0>0) {tmp_G_arr(hi[0]+1-i0, j , k ) = G_value;} @@ -126,71 +173,120 @@ computePhiIGF ( amrex::MultiFab const & rho, if ((j0>0)&&(k0>0)) {tmp_G_arr(i , hi[1]+1-j0, hi[2]+1-k0) = G_value;} if ((i0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, j , hi[2]+1-k0) = G_value;} if ((i0>0)&&(j0>0)&&(k0>0)) {tmp_G_arr(hi[0]+1-i0, hi[1]+1-j0, hi[2]+1-k0) = G_value;} - } - ); - } +#else + // With distributed FFTs (i.e. with heFFTe): + amrex::Real x_hi = dx*(hi[0]+2); + amrex::Real y_hi = dy*(hi[1]+2); + amrex::Real z_hi = dz*(hi[2]+2); + if ((i0< nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z , dx, dy, dz); } + if ((i0< nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z , dx, dy, dz); } + if ((i0< nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y , z_hi-z, dx, dy, dz); } + if ((i0> nx)&&(j0> ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z , dx, dy, dz); } + if ((i0< nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x , y_hi-y, z_hi-z, dx, dy, dz); } + if ((i0> nx)&&(j0< ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z_hi-z, dx, dy, dz); } + if ((i0> nx)&&(j0> ny)&&(k0> nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y_hi-y, z_hi-z, dx, dy, dz); } + if ((i0> nx)&&(j0< ny)&&(k0< nz)) { tmp_G_arr(i,j,k) = SumOfIntegratedPotential(x_hi-x, y , z , dx, dy, dz); } +#endif + } + ); } - // Perform forward FFTs - auto forward_plan_rho = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - auto forward_plan_G = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - // Loop over boxes perform FFTs - for ( amrex::MFIter mfi(realspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ - - // Note: the size of the real-space box and spectral-space box - // differ when using real-to-complex FFT. When initializing - // the FFT plan, the valid dimensions are those of the real-space box. - const amrex::IntVect fft_size = realspace_ba[mfi].length(); - - // FFT of rho - forward_plan_rho[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_rho[mfi].dataPtr(), - reinterpret_cast(tmp_rho_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(forward_plan_rho[mfi]); + // Prepare to perform global FFT + // Since there is 1 MPI rank per box, here each MPI rank obtains its local box and the associated boxid + int local_boxid = amrex::ParallelDescriptor::MyProc(); // because of how we made the DistributionMapping + if (local_boxid < realspace_ba.size()) { + // When not using heFFTe, there is only one box (the global box) + // It is taken care of my MPI rank 0 ; other ranks have no work (hence the if condition) - // FFT of G - forward_plan_G[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[mfi].dataPtr(), - reinterpret_cast(tmp_G_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(forward_plan_G[mfi]); + amrex::Box local_nodal_box = realspace_ba[local_boxid]; + amrex::Box local_box(local_nodal_box.smallEnd(), local_nodal_box.bigEnd()); + local_box.shift(-realspace_box.smallEnd()); // This simplifies the setup because the global lo is zero now + // Since we the domain decompostion is in the z-direction, setting up c_local_box is simple. + amrex::Box c_local_box = local_box; + c_local_box.setBig(0, local_box.length(0)/2+1); - } + // Allocate array in spectral space + using SpectralField = amrex::BaseFab< amrex::GpuComplex< amrex::Real > > ; + SpectralField tmp_rho_fft(c_local_box, 1, amrex::The_Device_Arena()); + SpectralField tmp_G_fft(c_local_box, 1, amrex::The_Device_Arena()); + tmp_rho_fft.shift(realspace_box.smallEnd()); + tmp_G_fft.shift(realspace_box.smallEnd()); - // Multiply tmp_G_fft and tmp_rho_fft in spectral space - // Store the result in-place in Gtmp_G_fft, to save memory - amrex::Multiply( tmp_G_fft, tmp_rho_fft, 0, 0, 1, 0); + // Create FFT plans + BL_PROFILE_VAR_START(timer_plans); +#if !defined(ABLASTR_USE_HEFFTE) + const amrex::IntVect fft_size = realspace_ba[local_boxid].length(); + ablastr::math::anyfft::FFTplan forward_plan_rho = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_rho[local_boxid].dataPtr(), + reinterpret_cast(tmp_rho_fft.dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::FFTplan forward_plan_G = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[local_boxid].dataPtr(), + reinterpret_cast(tmp_G_fft.dataPtr()), + ablastr::math::anyfft::direction::R2C, AMREX_SPACEDIM); + ablastr::math::anyfft::FFTplan backward_plan = ablastr::math::anyfft::CreatePlan( + fft_size, tmp_G[local_boxid].dataPtr(), + reinterpret_cast( tmp_G_fft.dataPtr()), + ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); +#elif defined(ABLASTR_USE_HEFFTE) +#if defined(AMREX_USE_CUDA) + heffte::fft3d_r2c fft +#elif defined(AMREX_USE_HIP) + heffte::fft3d_r2c fft +#else + heffte::fft3d_r2c fft +#endif + ({{local_box.smallEnd(0), local_box.smallEnd(1), local_box.smallEnd(2)}, + {local_box.bigEnd(0), local_box.bigEnd(1), local_box.bigEnd(2)}}, + {{c_local_box.smallEnd(0), c_local_box.smallEnd(1), c_local_box.smallEnd(2)}, + {c_local_box.bigEnd(0), c_local_box.bigEnd(1), c_local_box.bigEnd(2)}}, + 0, amrex::ParallelDescriptor::Communicator()); + using heffte_complex = typename heffte::fft_output::type; + heffte_complex* rho_fft_data = (heffte_complex*) tmp_rho_fft.dataPtr(); + heffte_complex* G_fft_data = (heffte_complex*) tmp_G_fft.dataPtr(); +#endif + BL_PROFILE_VAR_STOP(timer_plans); - // Perform inverse FFT - auto backward_plan = ablastr::math::anyfft::FFTplans(spectralspace_ba, dm_global_fft); - // Loop over boxes perform FFTs - for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ + // Perform forward FFTs + BL_PROFILE_VAR_START(timer_ffts); +#if !defined(ABLASTR_USE_HEFFTE) + ablastr::math::anyfft::Execute(forward_plan_rho); + ablastr::math::anyfft::Execute(forward_plan_G); +#elif defined(ABLASTR_USE_HEFFTE) + fft.forward(tmp_rho[local_boxid].dataPtr(), rho_fft_data); + fft.forward(tmp_G[local_boxid].dataPtr(), G_fft_data); +#endif + BL_PROFILE_VAR_STOP(timer_ffts); - // Note: the size of the real-space box and spectral-space box - // differ when using real-to-complex FFT. When initializing - // the FFT plan, the valid dimensions are those of the real-space box. - const amrex::IntVect fft_size = realspace_ba[mfi].length(); + // Multiply tmp_G_fft and tmp_rho_fft in spectral space + // Store the result in-place in Gtmp_G_fft, to save memory + tmp_G_fft.template mult(tmp_rho_fft, 0, 0, 1); + amrex::Gpu::streamSynchronize(); - // Inverse FFT: is done in-place, in the array of G - backward_plan[mfi] = ablastr::math::anyfft::CreatePlan( - fft_size, tmp_G[mfi].dataPtr(), - reinterpret_cast( tmp_G_fft[mfi].dataPtr()), - ablastr::math::anyfft::direction::C2R, AMREX_SPACEDIM); - ablastr::math::anyfft::Execute(backward_plan[mfi]); + // Perform backward FFT + BL_PROFILE_VAR_START(timer_ffts); +#if !defined(ABLASTR_USE_HEFFTE) + ablastr::math::anyfft::Execute(backward_plan); +#elif defined(ABLASTR_USE_HEFFTE) + fft.backward(G_fft_data, tmp_G[local_boxid].dataPtr()); +#endif + BL_PROFILE_VAR_STOP(timer_ffts); + +#if !defined(ABLASTR_USE_HEFFTE) + // Loop to destroy FFT plans + ablastr::math::anyfft::DestroyPlan(forward_plan_G); + ablastr::math::anyfft::DestroyPlan(forward_plan_rho); + ablastr::math::anyfft::DestroyPlan(backward_plan); +#endif } - // Normalize, since (FFT + inverse FFT) results in a factor N + + // Normalize, since (FFT + inverse FFT) results in a factor N const amrex::Real normalization = 1._rt / realspace_box.numPts(); tmp_G.mult( normalization ); + BL_PROFILE_VAR_START(timer_pcopies); // Copy from tmp_G to phi - phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect() ); - - // Loop to destroy FFT plans - for ( amrex::MFIter mfi(spectralspace_ba, dm_global_fft); mfi.isValid(); ++mfi ){ - ablastr::math::anyfft::DestroyPlan(forward_plan_G[mfi]); - ablastr::math::anyfft::DestroyPlan(forward_plan_rho[mfi]); - ablastr::math::anyfft::DestroyPlan(backward_plan[mfi]); - } + phi.ParallelCopy( tmp_G, 0, 0, 1, amrex::IntVect::TheZeroVector(), phi.nGrowVect()); + BL_PROFILE_VAR_STOP(timer_pcopies); } } // namespace ablastr::fields From 873b50636e68d0fd9688b9335b1036ffc1e19cbe Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Wed, 18 Sep 2024 11:23:34 -0700 Subject: [PATCH 12/13] Apply bug fix by @RemiLehe --- Source/Parallelization/WarpXComm.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Source/Parallelization/WarpXComm.cpp b/Source/Parallelization/WarpXComm.cpp index fd3df0bd080..8a0d74f5904 100644 --- a/Source/Parallelization/WarpXComm.cpp +++ b/Source/Parallelization/WarpXComm.cpp @@ -432,6 +432,9 @@ WarpX::UpdateAuxilaryDataSameType () m_fields.get("Bfield_cp",Direction{1},lev)->nComp(), ng); MultiFab dBz(m_fields.get("Bfield_cp",Direction{2},lev)->boxArray(), dm, m_fields.get("Bfield_cp",Direction{2},lev)->nComp(), ng); + dBx.setVal(0.0); + dBy.setVal(0.0); + dBz.setVal(0.0); // Copy Bfield_aux to the dB MultiFabs, using up to ng_src (=ng_FieldGather) guard // cells from Bfield_aux and filling up to ng (=nGrow) guard cells in the dB MultiFabs From 9a7d86441a3103c2e452b5e9ab58b81d813602aa Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Wed, 18 Sep 2024 11:56:11 -0700 Subject: [PATCH 13/13] Fix merge bug --- Source/Evolve/WarpXEvolve.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/Source/Evolve/WarpXEvolve.cpp b/Source/Evolve/WarpXEvolve.cpp index 218f14d7d8d..c9e986381ec 100644 --- a/Source/Evolve/WarpXEvolve.cpp +++ b/Source/Evolve/WarpXEvolve.cpp @@ -62,6 +62,7 @@ using ablastr::utils::SignalHandling; void WarpX::Synchronize () { + using ablastr::fields::Direction; FillBoundaryE(guard_cells.ng_FieldGather); FillBoundaryB(guard_cells.ng_FieldGather); if (fft_do_time_averaging)