Merge branch 'development' into ppm_full_well_balancing

CHANGES.md

@@ -1,3 +1,20 @@
+# 24.09
+
+  * Code clean-ups / clang-tidy (#2942, #2949)
+
+  * update the `hse_convergence` readme to reflect current convergence
+    (#2946)
+
+  * update the `bubble_convergence` plotting script (#2947)
+
+  * new Frontier scaling numbers (#2948)
+
+  * more GPU error printing (@3944)
+
+  * science problem updates: `flame_wave` (#2943)
+
+  * documentation updates (#2939)
+
 # 24.08
 
   * lazy QueueReduction has been enabled for the timing diagnostics

Exec/reacting_tests/bubble_convergence/analysis/slice_multi.py

@@ -1,5 +1,8 @@
 #!/usr/bin/env python3
 
+import matplotlib
+matplotlib.use('agg')
+
 import os
 import sys
 import yt
@@ -26,35 +29,37 @@
 fig = plt.figure()
 fig.set_size_inches(12.0, 9.0)
 
-grid = ImageGrid(fig, 111, nrows_ncols=(2, 2), axes_pad=0.75, cbar_pad="2%",
+grid = ImageGrid(fig, 111, nrows_ncols=(2, 2),
+                 axes_pad=0.75, cbar_pad="2%",
                  label_mode="L", cbar_mode="each")
 
 
-fields = ["Temp", "magvel", "X(C12)", "rho_enuc"]
+fields = ["Temp", "magvel", "X(C12)", "enuc"]
 
 for i, f in enumerate(fields):
 
-    sp = yt.SlicePlot(ds, "z", f, center=[xctr, yctr, 0.0], width=[L_x, L_y, 0.0], fontsize="12")
+    sp = yt.SlicePlot(ds, "z", f, center=[xctr, yctr, 0.0*cm],
+                      width=[L_x, L_y, 0.0*cm], fontsize="12")
     sp.set_buff_size((2000,2000))
 
     if f == "X(C12)":
         sp.set_log(f, True)
-        sp.set_cmap(f, "plasma")
-        sp.set_zlim(f, 1.e-8, 2.e-4)
+        sp.set_cmap(f, "magma")
+        sp.set_zlim(f, 1.e-8, 1.e-4)
 
     elif f == "magvel":
         sp.set_log(f, False)
         #sp.set_zlim(f, 1.e-3, 2.5e-2)
-        sp.set_cmap(f, "magma")
+        sp.set_cmap(f, "cividis")
 
     elif f == "Temp":
-        sp.set_log(f, False)
-        #sp.set_zlim(f, 1.e-3, 2.5e-2)
-
-    elif f == "rho_enuc":
         sp.set_log(f, True)
+        sp.set_zlim(f, 5.e7, 2.e8)
+
+    elif f == "enuc":
+        sp.set_log(f, True, linthresh=1.e11)
+        sp.set_zlim(f, 1.e11, 1.e14)
         sp.set_cmap(f, "plasma")
-        #sp.set_zlim(f, 1.e-3, 2.5e-2)
 
     sp.set_axes_unit("cm")
 
@@ -71,5 +76,4 @@
 
 fig.set_size_inches(8.0, 8.0)
 plt.tight_layout()
-plt.savefig("{}_slice.pdf".format(os.path.basename(plotfile)))
-
+plt.savefig("{}_slice.png".format(os.path.basename(plotfile)))

Exec/science/flame_wave/scaling/frontier/frontier-scaling-2024-08-21.txt

@@ -0,0 +1,39 @@
+# new scaling numbers for the 3D XRB
+# using the same inputs.He.25cm.static.1000Hz as previously
+# modules:
+#
+#   module load PrgEnv-gnu
+#   module load cray-mpich/8.1.28
+#   module load craype-accel-amd-gfx90a
+#   module load amd-mixed/6.1.3
+#   module unload darshan-runtime
+#
+# build info:
+#
+# EOS: /ccs/home/zingale/Microphysics/EOS/helmholtz
+# NETWORK: /ccs/home/zingale/Microphysics/networks/iso7
+# CONDUCTIVITY: /ccs/home/zingale/Microphysics/conductivity/stellar
+# INTEGRATOR: VODE
+# SCREENING: screen5
+#
+# Castro       git describe: 24.08-3-g15327db6b
+# AMReX        git describe: 24.08-25-g6dcaa1223
+# Microphysics git describe: 24.08-2-g8ce3375a
+
+# nodes  rocm      mag_grid_size   avg time /   std dev
+#                          step
+  48      6.1.3          128           59.0711   0.2525
+  64      6.1.3          128           42.6938 0.285659
+ 128      6.1.3          128           24.5353 1.36496
+ 256      6.1.3          128           13.3647 0.108731
+ 512      6.1.3          128            7.88166 0.0856889
+1024      6.1.3          128            5.54221 0.0979851
+2048      6.1.3          128            4.55679 0.0528629
+
+
+# note that the 2048 run uses a blocking factor of 16)
+
+# in contrast to the previous run, we've disabled all inlining with
+# ROCm to get around some compiler bugs, so that might explain some
+# slight slowdown here.
+

Exec/science/wdmerger/Prob.cpp

@@ -604,7 +604,7 @@ Castro::update_relaxation(Real time, Real dt) {
 
         const Real ldt = new_time - old_time;
 
-        force[lev].reset(new MultiFab(getLevel(lev).grids, getLevel(lev).dmap, NUM_STATE, 0));
+        force[lev] = std::make_unique<MultiFab>(getLevel(lev).grids, getLevel(lev).dmap, NUM_STATE, 0);
         force[lev]->setVal(0.0);
 
         MultiFab& S_new = getLevel(lev).get_new_data(State_Type);

Exec/science/wdmerger/Problem.H

@@ -27,11 +27,13 @@ void volInBoundary (amrex::Real time, amrex::Real& vol_p, amrex::Real& vol_s, am
 
 // Computes standard dot product of two three-vectors.
 
+static
 amrex::Real dot_product(const amrex::Real a[], const amrex::Real b[]);
 
 
 // Computes norm of a three vector.
 
+static
 amrex::Real norm(const amrex::Real a[]);
 
 // Problem post-initialization routine.
@@ -48,6 +50,7 @@ void problem_post_timestep();
 
 // Write out the git hashes for the various parts of the code.
 
+static
 void writeGitHashes(std::ostream& log);
 
 // Update relaxation process.

Exec/science/wdmerger/problem_checkpoint.H

@@ -7,7 +7,7 @@
 #include <prob_parameters.H>
 
 AMREX_INLINE
-void problem_checkpoint (std::string checkpoint_dir)
+void problem_checkpoint (const std::string& checkpoint_dir)
 {
     std::ofstream com;
     com.open(checkpoint_dir + "/COM", std::ios::out);

Exec/science/wdmerger/problem_restart.H

@@ -5,7 +5,7 @@
 #include <fstream>
 
 AMREX_INLINE
-void problem_restart (std::string checkpoint_dir)
+void problem_restart (const std::string& checkpoint_dir)
 {
     std::ifstream com;
     com.open(checkpoint_dir + "/COM", std::ios::in);

Exec/science/wdmerger/wdmerger_util.cpp

@@ -658,7 +658,7 @@ void binary_setup ()
 
             Real v_P_r, v_S_r, v_P_phi, v_S_phi;
 
-            kepler_third_law(problem::radius_P, problem::mass_P, problem::radius_S, problem::mass_S,
+            kepler_third_law(problem::radius_P, problem::mass_P, problem::radius_S, problem::mass_S,  // NOLINT(readability-suspicious-call-argument)
                              castro::rotational_period, problem::orbital_eccentricity, problem::orbital_angle,
                              problem::a, problem::r_P_initial, problem::r_S_initial, v_P_r, v_S_r, v_P_phi, v_S_phi);
 

Source/gravity/Gravity.cpp

@@ -2777,12 +2777,12 @@ Gravity::fill_direct_sum_BCs(int crse_level, int fine_level, const Vector<MultiF
     BL_ASSERT(nPtsXZ <= std::numeric_limits<int>::max());
     BL_ASSERT(nPtsYZ <= std::numeric_limits<int>::max());
 
-    ParallelDescriptor::ReduceRealSum(bcXYLo.dataPtr(), nPtsXY);
-    ParallelDescriptor::ReduceRealSum(bcXYHi.dataPtr(), nPtsXY);
-    ParallelDescriptor::ReduceRealSum(bcXZLo.dataPtr(), nPtsXZ);
-    ParallelDescriptor::ReduceRealSum(bcXZHi.dataPtr(), nPtsXZ);
-    ParallelDescriptor::ReduceRealSum(bcYZLo.dataPtr(), nPtsYZ);
-    ParallelDescriptor::ReduceRealSum(bcYZHi.dataPtr(), nPtsYZ);
+    ParallelDescriptor::ReduceRealSum(bcXYLo.dataPtr(), static_cast<int>(nPtsXY));
+    ParallelDescriptor::ReduceRealSum(bcXYHi.dataPtr(), static_cast<int>(nPtsXY));
+    ParallelDescriptor::ReduceRealSum(bcXZLo.dataPtr(), static_cast<int>(nPtsXZ));
+    ParallelDescriptor::ReduceRealSum(bcXZHi.dataPtr(), static_cast<int>(nPtsXZ));
+    ParallelDescriptor::ReduceRealSum(bcYZLo.dataPtr(), static_cast<int>(nPtsYZ));
+    ParallelDescriptor::ReduceRealSum(bcYZHi.dataPtr(), static_cast<int>(nPtsYZ));
 
 #ifdef _OPENMP
 #pragma omp parallel

Source/gravity/binary.H

@@ -67,7 +67,7 @@ void lagrange_iterate (Real& r, Real mass_1, Real mass_2, Real r1, Real r2, Real
     const Real tolerance = 1.0e-8_rt;
     const int max_iters = 200;
 
-    Real rm, rp;
+    Real rm{}, rp{};
 
     if (r_min == 0.0_rt && r_max == 0.0_rt) {
         amrex::Abort("Lagrange point iteration must have at least one non-zero bound provided.");