Merge branch 'development' into ppm_source_tracing

CHANGES.md

@@ -1,3 +1,20 @@
+# 24.09
+
+  * Code clean-ups / clang-tidy (#2942, #2949)
+
+  * update the `hse_convergence` readme to reflect current convergence
+    (#2946)
+
+  * update the `bubble_convergence` plotting script (#2947)
+
+  * new Frontier scaling numbers (#2948)
+
+  * more GPU error printing (@3944)
+
+  * science problem updates: `flame_wave` (#2943)
+
+  * documentation updates (#2939)
+
 # 24.08
 
   * lazy QueueReduction has been enabled for the timing diagnostics

Exec/gravity_tests/hse_convergence/README.md

@@ -7,29 +7,34 @@ in the plotfiles.
 
 To run this problem, use one of the convergence scripts:
 
-  * ``convergence_plm.sh`` :
+  * `convergence_plm.sh` :
 
-    this runs CTU + PLM using the default HSE BCs and default
-    use_pslope, then with reflect BCs, then without use_pslope, and
-    finally runs with reflect instead of HSE BCs.
+    this runs CTU + PLM using:
+    1. the default HSE BCs and `use_pslope`
+    2. the HSE BCs with reflection and `use_pslope`
+    3. reflect BCs instead of HSE BCs without `use_pslope`
+    4. reflect BCs with `use_pslope`
 
-    These tests show that the best results come from HSE BCs + reflect vel
+    These tests show that the best results (by far) come from
+    `use_pslope=1` and reflecting BCs
 
   * convergence_ppm.sh :
 
     this runs CTU + PPM in a similar set of configurations as PLM above
-    (with one additional one: grav_source_type = 4)
+    1. the default HSE BCs
+    2. HSE BCs with reflection
+    3. reflecting BCs
+    4. reflecting BCs with `use_pslope`
 
-    These tests show that the best results come from HSE BCs + reflect vel
+    These tests show that the best results (by far) come from
+    reflecting BCs with `use_pslope=1`, just like the PLM case.
 
   * convergence_sdc.sh :
 
-    this uses the TRUE_SDC integration, first with SDC-2 + PLM  and reflecting BCs,
-    the SDC-2 + PPM and reflecting BCs, then the same but HSE BCs, and finally
-    SDC-4 + reflect
+    this uses the TRUE_SDC integration, first with SDC-2 + PLM and
+    reflecting BCs, the SDC-2 + PPM and reflecting BCs, then the same
+    but HSE BCs, and finally SDC-4 + reflect
 
     These tests show that the PLM + reflect (which uses the
     well-balanced use_pslope) and the SDC-4 + reflect give the lowest
-    errors and expected (or better) convergence:
-
-
+    errors and expected (or better) convergence.

Exec/gravity_tests/hse_convergence/convergence_plm.sh

@@ -58,43 +58,15 @@ pfile=`ls -t | grep -i hse_512_plt | head -1`
 fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
 
 
-## plm + hse reflect + no pslope
-
-ofile=plm-hsereflect-nopslope.converge.out
-
-RUNPARAMS="
-castro.ppm_type=0
-castro.use_pslope=0
-castro.hse_interp_temp=1
-castro.hse_reflect_vels=1
-"""
-
-${EXEC} inputs.ppm.64 ${RUNPARAMS} >& 64.out
-pfile=`ls -t | grep -i hse_64_plt | head -1`
-fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel > ${ofile}
-
-${EXEC} inputs.ppm.128 ${RUNPARAMS} >& 128.out
-pfile=`ls -t | grep -i hse_128_plt | head -1`
-fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
-
-${EXEC} inputs.ppm.256 ${RUNPARAMS} >& 256.out
-pfile=`ls -t | grep -i hse_256_plt | head -1`
-fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
-
-${EXEC} inputs.ppm.512 ${RUNPARAMS} >& 512.out
-pfile=`ls -t | grep -i hse_512_plt | head -1`
-fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
-
-
-## plm + reflect
+## plm + reflect + nopslope
 
-ofile=plm-reflect.converge.out
+ofile=plm-reflect-nopslope.converge.out
 
 RUNPARAMS="
 castro.ppm_type=0
-castro.use_pslope=1
 castro.lo_bc=3
 castro.hi_bc=3
+castro.use_pslope=0
 """
 
 ${EXEC} inputs.ppm.64 ${RUNPARAMS} >& 64.out
@@ -114,16 +86,15 @@ pfile=`ls -t | grep -i hse_512_plt | head -1`
 fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
 
 
+## plm + reflect + pslope
 
-## plm + reflect + nopslope
-
-ofile=plm-reflect-nopslope.converge.out
+ofile=plm-reflect-pslope.converge.out
 
 RUNPARAMS="
 castro.ppm_type=0
 castro.lo_bc=3
 castro.hi_bc=3
-castro.use_pslope=0
+castro.use_pslope=1
 """
 
 ${EXEC} inputs.ppm.64 ${RUNPARAMS} >& 64.out

Exec/gravity_tests/hse_convergence/convergence_ppm.sh

@@ -50,12 +50,13 @@ pfile=`ls -t | grep -i hse_512_plt | head -1`
 fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
 
 
-## ppm + grav_source_type = 4
+## ppm + reflect
 
-ofile=ppm-grav4.converge.out
+ofile=ppm-reflect.converge.out
 
 RUNPARAMS="
-castro.grav_source_type=4
+castro.lo_bc=3
+castro.hi_bc=3
 """
 
 ${EXEC} inputs.ppm.64 ${RUNPARAMS} >& 64.out
@@ -75,13 +76,14 @@ pfile=`ls -t | grep -i hse_512_plt | head -1`
 fextrema.gnu.ex -v magvel ${pfile} | grep -i magvel >> ${ofile}
 
 
-## ppm + reflect
+## ppm + reflect + pslope
 
-ofile=ppm-reflect.converge.out
+ofile=ppm-reflect-pslope.converge.out
 
 RUNPARAMS="
 castro.lo_bc=3
 castro.hi_bc=3
+castro.use_pslope=1
 """
 
 ${EXEC} inputs.ppm.64 ${RUNPARAMS} >& 64.out

Exec/hydro_tests/gresho_vortex/problem_initialize_state_data.H

@@ -73,7 +73,7 @@ void problem_initialize_state_data (int i, int j, int k,
                 }
 
                 u_tot += u_phi;
-                reint += p/(gamma_const - 1.0_rt);
+                reint += p/(eos_rp::eos_gamma - 1.0_rt);
             }
         }
     }

Exec/radiation_tests/Rad2Tshock/problem_initialize.H

@@ -14,8 +14,8 @@ void problem_initialize ()
 
     eos_state.rho = problem::rho0;
     eos_state.T = problem::T0;
-    for (int n = 0; n < NumSpec; n++) {
-        eos_state.xn[n] = 0.0_rt;
+    for (auto & X : eos_state.xn) {
+        X = 0.0_rt;
     }
     eos_state.xn[0]  = 1.0_rt;
 

Exec/radiation_tests/Rad2Tshock/problem_initialize_rad_data.H

@@ -12,6 +12,9 @@ void problem_initialize_rad_data (int i, int j, int k,
                                   const GeometryData& geomdata)
 {
 
+    amrex::ignore_unused(nugroup);
+    amrex::ignore_unused(dnugroup);
+
     const Real* dx = geomdata.CellSize();
     const Real* problo = geomdata.ProbLo();
 

Exec/radiation_tests/Rad2Tshock/problem_initialize_state_data.H

@@ -16,7 +16,7 @@ void problem_initialize_state_data (int i, int j, int k,
     // Provides the simulation to be run in the x,y,or z direction
     // where length direction is the length side in a square prism
 
-    Real length_cell;
+    Real length_cell{};
     if (problem::idir == 1) {
         length_cell = problo[0] + dx[0] * (static_cast<Real>(i) + 0.5_rt);
     } else if (problem::idir == 2) {

Exec/reacting_tests/bubble_convergence/analysis/slice_multi.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python3
 
+import matplotlib
+matplotlib.use('agg')
+
 import os
 import sys
 import yt
@@ -26,35 +29,37 @@
 fig = plt.figure()
 fig.set_size_inches(12.0, 9.0)
 
-grid = ImageGrid(fig, 111, nrows_ncols=(2, 2), axes_pad=0.75, cbar_pad="2%",
+grid = ImageGrid(fig, 111, nrows_ncols=(2, 2),
+                 axes_pad=0.75, cbar_pad="2%",
                  label_mode="L", cbar_mode="each")
 
 
-fields = ["Temp", "magvel", "X(C12)", "rho_enuc"]
+fields = ["Temp", "magvel", "X(C12)", "enuc"]
 
 for i, f in enumerate(fields):
 
-    sp = yt.SlicePlot(ds, "z", f, center=[xctr, yctr, 0.0], width=[L_x, L_y, 0.0], fontsize="12")
+    sp = yt.SlicePlot(ds, "z", f, center=[xctr, yctr, 0.0*cm],
+                      width=[L_x, L_y, 0.0*cm], fontsize="12")
     sp.set_buff_size((2000,2000))
 
     if f == "X(C12)":
         sp.set_log(f, True)
-        sp.set_cmap(f, "plasma")
-        sp.set_zlim(f, 1.e-8, 2.e-4)
+        sp.set_cmap(f, "magma")
+        sp.set_zlim(f, 1.e-8, 1.e-4)
 
     elif f == "magvel":
         sp.set_log(f, False)
         #sp.set_zlim(f, 1.e-3, 2.5e-2)
-        sp.set_cmap(f, "magma")
+        sp.set_cmap(f, "cividis")
 
     elif f == "Temp":
-        sp.set_log(f, False)
-        #sp.set_zlim(f, 1.e-3, 2.5e-2)
-
-    elif f == "rho_enuc":
         sp.set_log(f, True)
+        sp.set_zlim(f, 5.e7, 2.e8)
+
+    elif f == "enuc":
+        sp.set_log(f, True, linthresh=1.e11)
+        sp.set_zlim(f, 1.e11, 1.e14)
         sp.set_cmap(f, "plasma")
-        #sp.set_zlim(f, 1.e-3, 2.5e-2)
 
     sp.set_axes_unit("cm")
 
@@ -71,5 +76,4 @@
 
 fig.set_size_inches(8.0, 8.0)
 plt.tight_layout()
-plt.savefig("{}_slice.pdf".format(os.path.basename(plotfile)))
-
+plt.savefig("{}_slice.png".format(os.path.basename(plotfile)))

Exec/science/flame_wave/ci-benchmarks/job_info_params.txt

@@ -94,7 +94,7 @@
     castro.sdc_quadrature = 0
     castro.sdc_extra = 0
     castro.sdc_solver = 1
-    castro.use_axisymmetric_geom_source = 1
+    castro.use_geom_source = 1
     castro.add_sdc_react_source_to_advection = 1
     castro.hydro_memory_footprint_ratio = -1
     castro.fixed_dt = -1

Exec/science/flame_wave/scaling/frontier/frontier-scaling-2024-08-21.txt

@@ -0,0 +1,39 @@
+# new scaling numbers for the 3D XRB
+# using the same inputs.He.25cm.static.1000Hz as previously
+# modules:
+#
+#   module load PrgEnv-gnu
+#   module load cray-mpich/8.1.28
+#   module load craype-accel-amd-gfx90a
+#   module load amd-mixed/6.1.3
+#   module unload darshan-runtime
+#
+# build info:
+#
+# EOS: /ccs/home/zingale/Microphysics/EOS/helmholtz
+# NETWORK: /ccs/home/zingale/Microphysics/networks/iso7
+# CONDUCTIVITY: /ccs/home/zingale/Microphysics/conductivity/stellar
+# INTEGRATOR: VODE
+# SCREENING: screen5
+#
+# Castro       git describe: 24.08-3-g15327db6b
+# AMReX        git describe: 24.08-25-g6dcaa1223
+# Microphysics git describe: 24.08-2-g8ce3375a
+
+# nodes  rocm      mag_grid_size   avg time /   std dev
+#                          step
+  48      6.1.3          128           59.0711   0.2525
+  64      6.1.3          128           42.6938 0.285659
+ 128      6.1.3          128           24.5353 1.36496
+ 256      6.1.3          128           13.3647 0.108731
+ 512      6.1.3          128            7.88166 0.0856889
+1024      6.1.3          128            5.54221 0.0979851
+2048      6.1.3          128            4.55679 0.0528629
+
+
+# note that the 2048 run uses a blocking factor of 16)
+
+# in contrast to the previous run, we've disabled all inlining with
+# ROCm to get around some compiler bugs, so that might explain some
+# slight slowdown here.
+

Exec/science/subch_planar/Problem_Derive.cpp

@@ -23,9 +23,8 @@ void ca_dergradpoverp(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /*nco
 
 #if AMREX_SPACEDIM == 3
     amrex::Error("3D not supported");
-#endif
-#if AMREX_SPACEDIM == 1
-    amrex::Error("1D not supported");
+#elif AMREX_SPACEDIM == 1
+    return; // Skip for 1D
 #endif
 
     Real dxinv = 1.0_rt / dx[0];
@@ -226,9 +225,8 @@ void ca_dergradpoverp1(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /*nc
 
 #if AMREX_SPACEDIM == 3
     amrex::Error("3D not supported");
-#endif
-#if AMREX_SPACEDIM == 1
-    amrex::Error("1D not supported");
+#elif AMREX_SPACEDIM == 1
+    return; // Skip for 1D
 #endif
 
     Real dxinv = 1.0_rt / dx[0];
@@ -434,9 +432,8 @@ void ca_dergradpx(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /*ncomp*/
 
 #if AMREX_SPACEDIM == 3
     amrex::Error("3D not supported");
-#endif
-#if AMREX_SPACEDIM == 1
-    amrex::Error("1D not supported");
+#elif AMREX_SPACEDIM == 1
+    return; // Skip for 1D
 #endif
 
     Real dxinv = 1.0_rt / dx[0];
@@ -545,9 +542,8 @@ void ca_dergradpy(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /*ncomp*/
 
 #if AMREX_SPACEDIM == 3
     amrex::Error("3D not supported");
-#endif
-#if AMREX_SPACEDIM == 1
-    amrex::Error("1D not supported");
+#elif AMREX_SPACEDIM == 1
+    return; // Skip for 1D
 #endif
 
     Real dxinv = 1.0_rt / dx[0];
@@ -658,9 +654,8 @@ void ca_dergradrhooverrho(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /
 
 #if AMREX_SPACEDIM == 3
     amrex::Error("3D not supported");
-#endif
-#if AMREX_SPACEDIM == 1
-    amrex::Error("1D not supported");
+#elif AMREX_SPACEDIM == 1
+    return; // Skip for 1D
 #endif
 
     Real dxinv = 1.0_rt / dx[0];
@@ -719,4 +714,4 @@ void ca_dergradrhooverrho(const Box& bx, FArrayBox& derfab, int /*dcomp*/, int /
 
     });
 
-}
+}