Merge branch 'development' into make_1d_cart_mom_have_p

.codespell-ignore-words

@@ -11,3 +11,4 @@ thi
 nd
 ue
 bion
+aas

.github/workflows/c-linter.yml

@@ -24,6 +24,16 @@ jobs:
           sudo apt-get update -y -qq
           sudo apt-get -qq -y install curl clang-tidy cmake jq clang cppcheck clang-format bear g++>=9.3.0 gfortran>=9.3.0
 
+      - name: Install hypre
+        run: |
+          wget -q https://github.com/hypre-space/hypre/archive/refs/tags/v2.28.0.tar.gz
+          tar xfz v2.28.0.tar.gz
+          cd hypre-2.28.0/src
+          ./configure --with-cxxstandard=17 --without-MPI
+          make -j 2
+          make install
+          cd ../../
+
       - name: Get cpp linter repo
         run: |
           cd external
@@ -38,6 +48,7 @@ jobs:
 
       - name: Run cpp linter
         run: |
+          export AMREX_HYPRE_HOME=${PWD}/hypre-2.28.0/src/hypre
           python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} \
           -ignore-files="amrex|Microphysics" \
           -config-file="${GITHUB_WORKSPACE}/.clang-tidy" \

.github/workflows/gpu_action.yml

@@ -31,7 +31,17 @@ jobs:
           sudo apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/3bf863cc.pub
           sudo add-apt-repository "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/ /"
           sudo apt-get update
-          sudo apt-get -y install cuda-toolkit
+          sudo apt-get -y install cuda-toolkit-11-8
+
+      - name: Install hypre
+        run: |
+          wget -q https://github.com/hypre-space/hypre/archive/refs/tags/v2.28.0.tar.gz
+          tar xfz v2.28.0.tar.gz
+          cd hypre-2.28.0/src
+          CUDA_HOME=/usr/local/cuda HYPRE_CUDA_SM=60 ./configure --with-cxxstandard=17 --with-cuda --enable-unified-memory --without-MPI
+          make -j 2
+          make install
+          cd ../../
 
       - name: Get cpp linter repo
         run: |
@@ -48,4 +58,5 @@ jobs:
       - name: Compile problems for GPU
         run: |
           export PATH=$PATH:/usr/local/cuda/bin
+          export AMREX_HYPRE_HOME=${PWD}/hypre-2.28.0/src/hypre
           python3 external/cpp-linter-action/run_on_changed_files.py ${{ github.event.pull_request.base.sha }} ${{ github.event.pull_request.head.sha }} -header-filter=Castro -ignore-files="amrex|Microphysics" -gpu

.github/workflows/mhd-compare.yml

@@ -0,0 +1,45 @@
+name: OrszagTang
+
+on: [pull_request]
+jobs:
+  OrszagTang-3d:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get submodules
+        run: |
+          git submodule update --init
+          cd external/Microphysics
+          git fetch; git checkout development
+          cd ../amrex
+          git fetch; git checkout development
+          cd ../..
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -y -qq
+          sudo apt-get -qq -y install curl cmake jq clang g++>=9.3.0 libopenmpi-dev openmpi-bin
+
+      - name: Compile OrszagTang
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          make USE_MPI=TRUE -j 2
+
+      - name: Run OrszagTang-3d
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          mpirun -np 2 ./Castro3d.gnu.MPI.ex inputs.test amr.plot_files_output=1
+
+      - name: Build the fextrema tool
+        run: |
+          cd external/amrex/Tools/Plotfile
+          make programs=fextrema -j 2
+
+      - name: Check the extrema
+        run: |
+          cd Exec/mhd_tests/OrszagTang
+          ../../../external/amrex/Tools/Plotfile/fextrema.gnu.ex plt00010 > fextrema.out
+          diff fextrema.out ci-benchmarks/OrszagTang-3d.out

CHANGES.md

@@ -1,5 +1,12 @@
 # 23.07
 
+  * The parameter castro.state_nghost, which allowed State_Type to have ghost
+    zones, has been removed. (#2502)
+
+  * The additional ghost zone in State_Type, used when radiation is enabled,
+    has been removed. The checkpoint version number has been updated to avoid
+    restarting from a checkpoint with the wrong number of ghost zones. (#2495)
+
   * The parameter gravity.no_composite was removed (#2483)
 
   * The parameter spherical_star was removed (#2482)

Docs/source/rotation.rst

@@ -224,7 +224,7 @@ a rotating frame
 
 
 The kinetic energy equation can be obtained from :eq:`eq:v-rot` by
-mulitplying by :math:`\rho\vbt`:
+multiplying by :math:`\rho\vbt`:
 
 .. math::
    \begin{align}

Docs/source/software.rst

@@ -256,10 +256,7 @@ The current ``StateData`` names Castro carries are:
       simply be advected, but we will allow rotation (in particular,
       the Coriolis force) to affect them.
 
-   ``State_Type`` ``MultiFab`` s have no ghost cells by default for
-   pure hydro and a single ghost cell by default when ``RADIATION``
-   is enabled. There is an option to force them to have ghost cells by
-   setting the parameter ``castro.state_nghost`` at runtime.
+   ``State_Type`` ``MultiFab`` s have no ghost cells.
 
    Note that the prediction of the hydrodynamic state to the interface
    will require 4 ghost cells. This accommodated by creating a separate

Exec/mhd_tests/OrszagTang/ci-benchmarks/OrszagTang-3d.out

@@ -0,0 +1,19 @@
+ plotfile = plt00010
+ time = 0.00017983829222995601
+              variables          minimum value          maximum value
+ density                         0.22099460191          0.22100305489
+ xmom                            -0.2210626652           0.2210626652
+ ymom                            -0.2211744105           0.2211744105
+ zmom                        -1.3394504115e-20       1.2796480966e-20
+ rho_E                           0.19805685456          0.46260143162
+ rho_e                           0.19794567403          0.19795835472
+ Temp                         7.2178022365e-09       7.2179887773e-09
+ rho_X                           0.22099460191          0.22100305489
+ pressure                         0.1326236016          0.13263209766
+ x_velocity                      -1.0002839593           1.0002839593
+ y_velocity                      -1.0007881112           1.0007881112
+ z_velocity                  -6.0608661064e-20       5.7902777063e-20
+ B_x                            -0.28206060038          0.28206060038
+ B_y                            -0.28220293547          0.28220293547
+ B_z                         -1.5570588641e-18        1.630216695e-18
+

Exec/reacting_tests/reacting_bubble/inputs_2d_test

@@ -23,10 +23,6 @@ castro.yr_ext_bc_type = 1
 
 castro.hse_interp_temp = 1
 
-
-# this is here just for test suite coverage
-castro.state_nghost = 1
-
 # WHICH PHYSICS
 castro.do_hydro = 1
 castro.do_react = 1

Exec/reacting_tests/reacting_bubble/inputs_3d_test

@@ -27,9 +27,6 @@ castro.fill_ambient_bc = 1
 castro.ambient_fill_dir = 2
 castro.ambient_outflow_vel = 1
 
-# this is here just for test suite coverage
-castro.state_nghost = 1
-
 # WHICH PHYSICS
 castro.do_hydro = 1
 castro.do_react = 1

Source/driver/Castro.H

@@ -429,16 +429,8 @@ public:
 /// @param time     the current simulation time
 /// @param dt       the timestep to advance (e.g., go from time to
 ///                    time + dt)
-/// @param amr_iteration    where we are in the current AMR subcycle.  Each
-///                    level will take a number of steps to reach the
-///                    final time of the coarser level below it.  This
-///                    counter starts at 1
-/// @param amr_ncycle   the number of subcycles at this level
 ///
-    advance_status do_advance_ctu(amrex::Real time,
-                                  amrex::Real dt,
-                                  int  amr_iteration,
-                                  int  amr_ncycle);
+    advance_status do_advance_ctu (amrex::Real time, amrex::Real dt);
 
 
 #ifndef AMREX_USE_GPU

Source/driver/Castro_advance_ctu.cpp

@@ -13,15 +13,8 @@
 using namespace amrex;
 
 advance_status
-Castro::do_advance_ctu(Real time,
-                       Real dt,
-                       int  amr_iteration,
-                       int  amr_ncycle)
+Castro::do_advance_ctu (Real time, Real dt)
 {
-
-    amrex::ignore_unused(amr_iteration);
-    amrex::ignore_unused(amr_ncycle);
-
     // this routine will advance the old state data (called Sborder here)
     // to the new time, for a single level.  The new data is called
     // S_new here.  The update includes reactions (if we are not doing
@@ -36,21 +29,6 @@ Castro::do_advance_ctu(Real time,
     const Real prev_time = state[State_Type].prevTime();
     const Real  cur_time = state[State_Type].curTime();
 
-    MultiFab& S_new = get_new_data(State_Type);
-
-    MultiFab& old_source = get_old_data(Source_Type);
-    MultiFab& new_source = get_new_data(Source_Type);
-
-#ifdef MHD
-    MultiFab& Bx_old = get_old_data(Mag_Type_x);
-    MultiFab& By_old = get_old_data(Mag_Type_y);
-    MultiFab& Bz_old = get_old_data(Mag_Type_z);
-
-    MultiFab& Bx_new = get_new_data(Mag_Type_x);
-    MultiFab& By_new = get_new_data(Mag_Type_y);
-    MultiFab& Bz_new = get_new_data(Mag_Type_z);
-#endif 
-
     // Perform initialization steps.
 
     status = initialize_do_advance(time, dt);
@@ -59,98 +37,74 @@ Castro::do_advance_ctu(Real time,
         return status;
     }
 
-    // If we are Strang splitting the reactions, do the old-time contribution now
+    // Perform all pre-advance operations and then initialize
+    // the new-time state with the output of those operators.
 
-#ifdef REACTIONS
-    status = do_old_reactions(time, dt);
+    status = pre_advance_operators(prev_time, dt);
 
     if (status.success == false) {
         return status;
     }
-#endif
-
-    // Initialize the new-time data. This copy needs to come after the
-    // reactions.
-
-    MultiFab::Copy(S_new, Sborder, 0, 0, NUM_STATE, S_new.nGrow());
 
-    // Construct the old-time sources from Sborder.  This will already
-    // be applied to S_new (with full dt weighting), to be correctly
-    // later.  Note -- this does not affect the prediction of the
-    // interface state, an explicit source will be traced there as
+    // Construct the old-time sources from Sborder. This will already
+    // be applied to S_new (with full dt weighting), to be corrected
+    // later. Note that this does not affect the prediction of the
+    // interface state; an explicit source will be traced there as
     // needed.
 
-#ifdef GRAVITY
-    construct_old_gravity(amr_iteration, amr_ncycle, prev_time);
-#endif
+    status = do_old_sources(prev_time, dt);
 
-    do_old_sources(
-#ifdef MHD
-                   Bx_old, By_old, Bz_old,
-#endif                
-                   old_source, Sborder, S_new, prev_time, dt);
+    if (status.success == false) {
+        return status;
+    }
 
-#ifdef SIMPLIFIED_SDC
-#ifdef REACTIONS
-    // the SDC reactive source ghost cells on coarse levels might not
-    // be in sync due to any average down done, so fill them here
+    // Perform any operations that occur after the sources but before the hydro.
 
-    MultiFab& react_src = get_new_data(Simplified_SDC_React_Type);
+    status = pre_hydro_operators(prev_time, dt);
 
-    AmrLevel::FillPatch(*this, react_src, react_src.nGrow(), cur_time, Simplified_SDC_React_Type, 0, react_src.nComp());
-#endif
-#endif
+    if (status.success == false) {
+        return status;
+    }
 
-    // Do the hydro update.  We build directly off of Sborder, which
-    // is the state that has already seen the burn
+    // Do the hydro update. We build directly off of Sborder, which
+    // is the state that has already seen the burn.
 
-    if (do_hydro)
-    {
 #ifndef MHD
-        status = construct_ctu_hydro_source(time, dt);
+    status = construct_ctu_hydro_source(prev_time, dt);
 #else
-        status = construct_ctu_mhd_source(time, dt);
+    status = construct_ctu_mhd_source(prev_time, dt);
 #endif
 
-        if (status.success == false) {
-            return status;
-        }
+    if (status.success == false) {
+        return status;
     }
 
-    // Construct and apply new-time source terms.
+    // Perform any operations that occur after the hydro but before
+    // the corrector sources.
 
-#ifdef GRAVITY
-    construct_new_gravity(amr_iteration, amr_ncycle, cur_time);
-#endif
+    status = post_hydro_operators(cur_time, dt);
 
-    do_new_sources(
-#ifdef MHD
-                    Bx_new, By_new, Bz_new,
-#endif  
-                    new_source, Sborder, S_new, cur_time, dt);
+    if (status.success == false) {
+        return status;
+    }
 
-    // If the state has ghost zones, sync them up now
-    // since the hydro source only works on the valid zones.
+    // Construct and apply new-time source terms.
 
-    if (S_new.nGrow() > 0) {
-      clean_state(
-#ifdef MHD
-                  Bx_new, By_new, Bz_new,
-#endif                
-                  S_new, cur_time, 0);
+    status = do_new_sources(cur_time, dt);
 
-      expand_state(S_new, cur_time, S_new.nGrow());
+    if (status.success == false) {
+        return status;
     }
 
     // Do the second half of the reactions for Strang, or the full burn for simplified SDC.
 
-#ifdef REACTIONS
-    status = do_new_reactions(cur_time, dt);
+    status = post_advance_operators(cur_time, dt);
 
     if (status.success == false) {
         return status;
     }
-#endif // REACTIONS
+
+    // Perform finalization steps.
 
     status = finalize_do_advance(cur_time, dt);
 
@@ -252,7 +206,6 @@ Castro::retry_advance_ctu(Real dt, const advance_status& status)
     }
 
     return do_retry;
-
 }
 
 
@@ -404,7 +357,7 @@ Castro::subcycle_advance_ctu(const Real time, const Real dt, int amr_iteration,
 
             // We do the hydro advance here, and record whether we completed it.
 
-            status = do_advance_ctu(subcycle_time, dt_subcycle, amr_iteration, amr_ncycle);
+            status = do_advance_ctu(subcycle_time, dt_subcycle);
 
             if (in_retry) {
                 in_retry = false;

Source/driver/Castro_advance_sdc.cpp

@@ -91,7 +91,7 @@ Castro::do_advance_sdc (Real time,
 
       // TODO: this is not using the density at the current stage
 #ifdef GRAVITY
-      construct_old_gravity(amr_iteration, amr_ncycle, prev_time);
+      construct_old_gravity(prev_time);
 #endif
 
       if (apply_sources()) {