Add PeleMP cases for soot and spray

.github/workflows/ci.yml

@@ -273,6 +273,7 @@ jobs:
             -DPELEC_ENABLE_MPI:BOOL=ON \
             -DPELEC_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
             -DPELEC_ENABLE_MASA:BOOL=ON \
+            -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
             -DMASA_ROOT:PATH=${{runner.workspace}}/deps/install/MASA \
             -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
             ${{github.workspace}}; \
@@ -365,6 +366,7 @@ jobs:
             -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
             -DPELEC_ENABLE_MPI:BOOL=OFF \
             -DPELEC_ENABLE_CUDA:BOOL=ON \
+            -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
             -DAMReX_CUDA_ERROR_CAPTURE_THIS:BOOL=ON \
             -DCMAKE_CUDA_ARCHITECTURES:STRING=70 \
             -DCMAKE_CUDA_COMPILER_LAUNCHER:STRING=ccache \
@@ -430,6 +432,7 @@ jobs:
           -DCMAKE_CXX_STANDARD:STRING=17 \
           -DPELEC_ENABLE_MPI:BOOL=OFF \
           -DPELEC_ENABLE_HIP:BOOL=ON \
+          -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
           -DAMReX_AMD_ARCH:STRING=gfx90a \
           -DCMAKE_HIP_ARCHITECTURES:STRING=gfx90a \
           -DAMDGPU_TARGETS:STRING=gfx90a \
@@ -502,6 +505,7 @@ jobs:
             -DPELEC_ENABLE_MASA:BOOL=OFF \
             -DPELEC_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
             -DPELEC_ENABLE_SYCL:BOOL=ON \
+            -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
             -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
             ${{github.workspace}}
       - name: Build
@@ -589,6 +593,7 @@ jobs:
           -DPELEC_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
           -DPELEC_ENABLE_MASA:BOOL=OFF \
           -DPELEC_ENABLE_CLANG_TIDY:BOOL=ON \
+          -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
           -DCMAKE_CXX_COMPILER_LAUNCHER:STRING=ccache \
           ${{github.workspace}}
       - name: Check
@@ -638,6 +643,7 @@ jobs:
           -DPELEC_ENABLE_MPI:BOOL=OFF \
           -DPELEC_ENABLE_FCOMPARE_FOR_TESTS:BOOL=OFF \
           -DPELEC_ENABLE_MASA:BOOL=OFF \
+          -DPELEC_EXCLUDE_BUILD_IN_CI:BOOL=ON \
           -DPELEC_ENABLE_CLANG_TIDY:BOOL=OFF \
           ${{github.workspace}}
       - name: Build

CMakeLists.txt

@@ -24,6 +24,7 @@ option(PELEC_ENABLE_TINY_PROFILE "Enable tiny profiler in AMReX" OFF)
 option(PELEC_ENABLE_HDF5 "Enable plot file output using HDF5" OFF)
 option(PELEC_ENABLE_HDF5_ZFP "Enable ZFP compression in HDF5" OFF)
 option(PELEC_ENABLE_ASCENT "Enable Ascent in-situ visualization" OFF)
+option(PELEC_EXCLUDE_BUILD_IN_CI "Exclude some builds when running in the CI" OFF)
 set(PELEC_PRECISION "DOUBLE" CACHE STRING "Floating point precision SINGLE or DOUBLE")
 
 #Options for performance

Exec/RegTests/CMakeLists.txt

@@ -12,6 +12,14 @@ add_subdirectory(MassCons)
 
 if(PELEC_ENABLE_AMREX_PARTICLES)
   add_subdirectory(Spray-Conv)
+  if(NOT PELEC_EXCLUDE_BUILD_IN_CI)
+    add_subdirectory(Spray-A-Wbreakup)
+    add_subdirectory(Spray-EB)
+    add_subdirectory(Spray-Evaporation)
+    add_subdirectory(Spray-Jet)
+    add_subdirectory(Spray-Multijet)
+    add_subdirectory(Spray-TG)
+  endif()
 endif()
 
 add_subdirectory(EB-C3)
@@ -22,7 +30,10 @@ add_subdirectory(EB-C11)
 add_subdirectory(EB-C12)
 #add_subdirectory(EB-C14)
 add_subdirectory(EB-ConvergingNozzle)
-add_subdirectory(Soot-Flame)
+add_subdirectory(Soot-ZeroD)
+if(NOT PELEC_EXCLUDE_BUILD_IN_CI)
+  add_subdirectory(Soot-Flame)
+endif()
 if(PELEC_DIM GREATER 2)
   add_subdirectory(EB-C7)
   add_subdirectory(EB-C8)

Exec/RegTests/Soot-Flame/GNUmakefile

@@ -5,7 +5,7 @@ PRECISION = DOUBLE
 
 # Profiling
 PROFILE = FALSE
-TINY_PROFILE = TRUE
+TINY_PROFILE = FALSE
 COMM_PROFILE = FALSE
 TRACE_PROFILE = FALSE
 MEM_PROFILE = FALSE
@@ -16,6 +16,7 @@ USE_MPI = FALSE
 USE_OMP = FALSE
 USE_CUDA = FALSE
 USE_HIP = FALSE
+USE_SYCL = FALSE
 
 # Debugging
 DEBUG = FALSE
@@ -32,7 +33,6 @@ Transport_Model := Simple
 
 # Soot
 USE_SOOT = TRUE
-# If this is changed, must run a make clean and rerun make
 NUM_SOOT_MOMENTS = 3
 #NUM_SOOT_MOMENTS = 6
 

Exec/RegTests/Soot-Flame/soot-flame.inp

@@ -25,7 +25,6 @@ pelec.diffuse_spec = 1
 pelec.do_react = 1
 pelec.chem_integrator = "ReactorCvode"
 pelec.do_mol = 0
-pelec.do_grav = 0
 pelec.allow_negative_energy = 1
 
 # TIME STEP CONTROL
@@ -61,7 +60,6 @@ amr.plot_file       = plt # root name of plotfile
 # SOOT MODELING
 soot.incept_pah = A2 # Soot inception species
 soot.v = 0
-soot.max_dt_rate = 0.01
 soot.num_subcycles = 1
 soot.temp_cutoff = 350.
 soot.conserve_mass = false

Exec/RegTests/Soot-ZeroD/CMakeLists.txt

@@ -0,0 +1,7 @@
+set(PELEC_ENABLE_PARTICLES OFF)
+set(PELEC_ENABLE_SOOT ON)
+set(PELE_NUM_SOOT_MOMENTS 3)
+set(PELEC_EOS_MODEL Fuego)
+set(PELEC_CHEMISTRY_MODEL SootReaction)
+set(PELEC_TRANSPORT_MODEL Simple)
+include(BuildExeAndLib)

Exec/RegTests/Soot-ZeroD/GNUmakefile

@@ -0,0 +1,43 @@
+# AMReX
+DIM = 2
+COMP = gnu
+PRECISION = DOUBLE
+
+# Profiling
+PROFILE = FALSE
+TINY_PROFILE = FALSE
+COMM_PROFILE = FALSE
+TRACE_PROFILE = FALSE
+MEM_PROFILE = FALSE
+USE_GPROF = FALSE
+
+# Performance
+USE_MPI = FALSE
+USE_OMP = FALSE
+USE_CUDA = FALSE
+USE_HIP = FALSE
+USE_SYCL = FALSE
+
+# Debugging
+DEBUG = FALSE
+FSANITIZER = FALSE
+THREAD_SANITIZER = FALSE
+
+# PeleC
+PELE_CVODE_FORCE_YCORDER = FALSE
+PELE_USE_MAGMA = FALSE
+PELE_COMPILE_AJACOBIAN = FALSE
+Eos_Model := Fuego
+Chemistry_Model := SootReaction
+Transport_Model := Simple
+
+# Soot
+USE_SOOT = TRUE
+NUM_SOOT_MOMENTS = 3
+#NUM_SOOT_MOMENTS = 6
+
+# GNU Make
+Bpack := ./Make.package
+Blocs := .
+PELEC_HOME := ../../..
+include $(PELEC_HOME)/Exec/Make.PeleC

Exec/RegTests/Soot-ZeroD/Make.package

@@ -0,0 +1,3 @@
+CEXE_sources += prob.cpp
+CEXE_headers += prob.H
+CEXE_headers += prob_parm.H

Exec/RegTests/Soot-ZeroD/example.inp

@@ -0,0 +1,80 @@
+# ------------------  INPUTS TO MAIN PROGRAM  -------------------
+max_step = 1000000
+stop_time = 0.022
+
+# PROBLEM SIZE & GEOMETRY
+geometry.is_periodic = 1 1 1
+geometry.coord_sys   = 0  # 0 => cart, 1 => RZ  2=>spherical
+geometry.prob_lo     = 0. 0. 0.
+geometry.prob_hi     = 1. 1. 1.
+amr.n_cell           = 4 4 4
+
+pelec.lo_bc =  "Interior" "Interior" "Interior"
+pelec.hi_bc =  "Interior" "Interior" "Interior"
+
+amrex.fpe_trap_invalid = 1
+amrex.fpe_trap_zero = 1
+amrex.fpe_trap_overflow = 1
+
+cvode.solve_type = dense_direct
+
+# WHICH PHYSICS
+pelec.do_hydro = 1
+pelec.diffuse_vel = 0
+pelec.diffuse_temp = 0
+pelec.diffuse_enth = 0
+pelec.diffuse_spec = 0
+pelec.do_react = 1
+pelec.chem_integrator = "ReactorCvode"
+pelec.do_mol = 1
+pelec.allow_negative_energy = 1
+
+# TIME STEP CONTROL
+pelec.cfl            = 0.3     # cfl number for hyperbolic system
+pelec.init_shrink    = 0.1     # scale back initial timestep
+pelec.change_max     = 1.1     # max time step growth
+pelec.dt_cutoff      = 5.e-20  # level 0 timestep below which we halt
+pelec.fixed_dt       = 1.E-7
+
+# DIAGNOSTICS & VERBOSITY
+pelec.sum_interval   = -1      # timesteps between computing mass
+pelec.v              = 0       # verbosity in Castro.cpp
+amr.v                = 1       # verbosity in Amr.cpp
+
+# REFINEMENT / REGRIDDING
+amr.max_level       = 0       # maximum level number allowed
+amr.ref_ratio       = 2 2 2 2 # refinement ratio
+amr.regrid_int      = 2 2 2 2 # how often to regrid
+amr.blocking_factor = 4       # block factor in grid generation
+amr.max_grid_size   = 32
+amr.n_error_buf     = 2 2 2 2 # number of buffer cells in error est
+
+# CHECKPOINT FILES
+amr.checkpoint_files_output = 0
+amr.check_file      = chk        # root name of checkpoint file
+amr.check_int       = 1000
+
+# PLOTFILES
+amr.plot_file       = plt # root name of plotfile
+amr.plot_per        = 2.5E-5
+
+# SOOT MODELING
+soot.incept_pah = A2 # Soot inception species
+soot.v = 0
+soot.conserve_mass = true
+
+pelec.add_soot_src = 1
+amr.derive_plot_vars = x_velocity y_velocity pressure soot_vars soot_large_particles
+pelec.plot_rhoy = 0
+pelec.plot_massfrac = 1
+amr.file_name_digits = 7
+
+pelec.plot_soot = 1
+
+# PROBLEM INPUT
+prob.init_T = 1300.
+prob.init_p = 1.01325E6
+prob.init_N2 = 0.6686465670855919
+prob.init_O2 = 0.20313062439007742
+prob.init_fuel = 0.12822280852433066
+prob.fuel_name = N-C7H16

Exec/RegTests/Soot-ZeroD/prob.H

@@ -0,0 +1,94 @@
+#ifndef PROB_H
+#define PROB_H
+
+#include <AMReX_Print.H>
+#include <AMReX_ParmParse.H>
+#include <AMReX_Geometry.H>
+#include <AMReX_FArrayBox.H>
+
+#include "mechanism.H"
+
+#include "IndexDefines.H"
+#include "Constants.H"
+#include "PelePhysics.H"
+#include "Tagging.H"
+#include "Transport.H"
+#include "ProblemDerive.H"
+#include "prob_parm.H"
+#include "SootModel.H"
+
+AMREX_GPU_DEVICE
+AMREX_FORCE_INLINE
+void
+pc_initdata(
+  int i,
+  int j,
+  int k,
+  amrex::Array4<amrex::Real> const& state,
+  amrex::GeometryData const& /*geomdata*/,
+  ProbParmDevice const& prob_parm)
+{
+
+  // Geometry
+  // const amrex::Real* prob_lo = geomdata.ProbLo();
+  // const amrex::Real* prob_hi = geomdata.ProbHi();
+  // const amrex::Real* dx = geomdata.CellSize();
+  // const amrex::Real x = prob_lo[0] + (i + 0.5) * dx[0];
+  // const amrex::Real y = prob_lo[1] + (j + 0.5) * dx[1];
+  // const amrex::Real z = prob_lo[2] + (k + 0.5) * dx[2];
+
+  // TG functions
+  amrex::Real u[3] = {0.0};
+  const amrex::Real p = prob_parm.p0;
+  amrex::Real rho, eint;
+  amrex::Real massfrac[NUM_SPECIES] = {0.0};
+  massfrac[O2_ID] = prob_parm.Y_O2;
+  massfrac[N2_ID] = prob_parm.Y_N2;
+  massfrac[prob_parm.fuelIndx] = prob_parm.Y_Fuel;
+  amrex::Real Tval = prob_parm.T0;
+  auto eos = pele::physics::PhysicsType::eos();
+  eos.PYT2RE(p, massfrac, Tval, rho, eint);
+
+  // Set the state
+  state(i, j, k, URHO) = rho;
+  state(i, j, k, UMX) = rho * u[0];
+  state(i, j, k, UMY) = rho * u[1];
+  state(i, j, k, UMZ) = rho * u[2];
+  state(i, j, k, UEINT) = rho * eint;
+  state(i, j, k, UEDEN) =
+    rho * (eint + 0.5 * (u[0] * u[0] + u[1] * u[1] + u[2] * u[2]));
+  state(i, j, k, UTEMP) = Tval;
+  for (int n = 0; n < NUM_SPECIES; n++) {
+    state(i, j, k, UFS + n) = rho * massfrac[n];
+  }
+  for (int n = 0; n < NUM_SOOT_MOMENTS + 1; n++) {
+    state(i, j, k, UFSOOT + n) = prob_parm.soot_vals[n];
+  }
+}
+
+AMREX_GPU_DEVICE
+AMREX_FORCE_INLINE
+void
+bcnormal(
+  const amrex::Real* /*x*/,
+  const amrex::Real s_int[NVAR],
+  amrex::Real s_ext[NVAR],
+  const int /*idir*/,
+  const int /*sgn*/,
+  const amrex::Real /*time*/,
+  amrex::GeometryData const& /*geomdata*/,
+  ProbParmDevice const& /*prob_parm*/,
+  const amrex::GpuArray<amrex::Real, AMREX_SPACEDIM>& /*turb_fluc*/)
+{
+  // This sets the values at the boundary equal to the
+  // adjacent interior values
+  for (int n = 0; n < NVAR; n++) {
+    s_ext[n] = s_int[n];
+  }
+}
+
+void pc_prob_close();
+
+using ProblemTags = EmptyProbTagStruct;
+using ProblemDerives = EmptyProbDeriveStruct;
+#endif