diff --git a/Exec/RegTests/DensityCurrent/ERF_prob.cpp b/Exec/RegTests/DensityCurrent/ERF_prob.cpp index bc40daece..33c1d763b 100644 --- a/Exec/RegTests/DensityCurrent/ERF_prob.cpp +++ b/Exec/RegTests/DensityCurrent/ERF_prob.cpp @@ -48,6 +48,7 @@ Problem::init_custom_pert( const SolverChoice& sc) { const bool use_moisture = (sc.moisture_type != MoistureType::None); + const bool is_anelastic = (sc.anelastic[0] == 1); const Real l_x_r = parms.x_r; //const Real l_x_r = parms.x_r * mf_u(0,0,0); //used to validate constant msf @@ -70,6 +71,7 @@ Problem::init_custom_pert( Real L = std::sqrt( std::pow((x - l_x_c)/l_x_r, 2) + std::pow((z - l_z_c)/l_z_r, 2)); + if (L <= 1.0) { Real dT = l_Tpt * (std::cos(PI*L) + 1.0)/2.0; @@ -77,8 +79,13 @@ Problem::init_custom_pert( // Note: dT is a perturbation in temperature, theta_perturbed is base state + perturbation Real theta_perturbed = (Tbar_hse+dT)*std::pow(p_0/p_hse(i,j,k), rdOcp); + Real theta_0 = (Tbar_hse )*std::pow(p_0/p_hse(i,j,k), rdOcp); - state_pert(i, j, k, Rho_comp) = getRhoThetagivenP(p_hse(i,j,k)) / theta_perturbed - r_hse(i,j,k); + if (is_anelastic) { + state_pert(i, j, k, RhoTheta_comp) = r_hse(i,j,k) * (theta_perturbed - theta_0); + } else { + state_pert(i, j, k, Rho_comp) = getRhoThetagivenP(p_hse(i,j,k)) / theta_perturbed - r_hse(i,j,k); + } } // Set scalar = 0 everywhere @@ -109,8 +116,13 @@ Problem::init_custom_pert( // Note: dT is a perturbation in temperature, theta_perturbed is base state + perturbation Real theta_perturbed = (Tbar_hse+dT)*std::pow(p_0/p_hse(i,j,k), rdOcp); + Real theta_0 = (Tbar_hse )*std::pow(p_0/p_hse(i,j,k), rdOcp); - state_pert(i, j, k, Rho_comp) = getRhoThetagivenP(p_hse(i,j,k)) / theta_perturbed - r_hse(i,j,k); + if (is_anelastic) { + state_pert(i, j, k, RhoTheta_comp) = r_hse(i,j,k) * (theta_perturbed - theta_0); + } else { + state_pert(i, j, k, Rho_comp) = getRhoThetagivenP(p_hse(i,j,k)) / theta_perturbed - r_hse(i,j,k); + } } // Set scalar = 0 everywhere diff --git a/Exec/RegTests/DensityCurrent/inputs_anelastic b/Exec/RegTests/DensityCurrent/inputs_anelastic new file mode 100644 index 000000000..d4373fa8c --- /dev/null +++ b/Exec/RegTests/DensityCurrent/inputs_anelastic @@ -0,0 +1,74 @@ +# ------------------ INPUTS TO MAIN PROGRAM ------------------- +max_step = 999999 +stop_time = 900.0 + +erf.anelastic = 1 +erf.no_substepping = 1 +erf.buoyancy_type = 2 + +erf.use_terrain = true +erf.use_terrain = false + +amrex.fpe_trap_invalid = 1 + +fabarray.mfiter_tile_size = 1024 1024 1024 + +# PROBLEM SIZE & GEOMETRY + +#geometry.prob_lo = -12800. 0. 0. +#geometry.prob_hi = 12800. 1600. 6400. +#xlo.type = "Outflow" + +geometry.prob_lo = 0. 0. 0. +geometry.prob_hi = 25600. 1600. 6400. +xlo.type = "Symmetry" + +xhi.type = "Outflow" + +geometry.is_periodic = 0 1 0 + +amr.n_cell = 256 16 64 # dx=dy=dz=100 m, Straka et al 1993 / Xue et al 2000 + +zlo.type = "SlipWall" +zhi.type = "SlipWall" + +# TIME STEP CONTROL +erf.fixed_dt = 1.0 # fixed time step [s] -- Straka et al 1993 +erf.fixed_fast_dt = 0.25 # fixed time step [s] -- Straka et al 1993 + +# DIAGNOSTICS & VERBOSITY +erf.sum_interval = 1 # timesteps between computing mass +erf.v = 0 # verbosity in ERF.cpp +erf.mg_v = 1 # verbosity in ERF.cpp +amr.v = 1 # verbosity in Amr.cpp + +# REFINEMENT / REGRIDDING +amr.max_level = 0 # maximum level number allowed + +# CHECKPOINT FILES +erf.check_file = chk # root name of checkpoint file +erf.check_int = -57600 # number of timesteps between checkpoints + +# PLOTFILES +erf.plot_file_1 = plta # prefix of plotfile name +erf.plot_int_1 = 100 # number of timesteps between plotfiles +erf.plot_vars_1 = density x_velocity y_velocity z_velocity pressure theta pres_hse dens_hse pert_pres pert_dens + +# SOLVER CHOICE +erf.use_gravity = true +erf.use_coriolis = false + +erf.les_type = "None" +# +# diffusion coefficient from Straka, K = 75 m^2/s +# +erf.molec_diff_type = "ConstantAlpha" +erf.rho0_trans = 1.0 # [kg/m^3], used to convert input diffusivities +erf.dynamicViscosity = 75.0 # [kg/(m-s)] ==> nu = 75.0 m^2/s +erf.alpha_T = 0.0 # [m^2/s] + +erf.c_p = 1004.0 + +# PROBLEM PARAMETERS (optional) +prob.T_0 = 300.0 +prob.U_0 = 0.0 diff --git a/Exec/RegTests/DensityCurrent/inputs_crse_halfdomain b/Exec/RegTests/DensityCurrent/inputs_crse_halfdomain index 75bb9b815..b24573027 100644 --- a/Exec/RegTests/DensityCurrent/inputs_crse_halfdomain +++ b/Exec/RegTests/DensityCurrent/inputs_crse_halfdomain @@ -1,7 +1,6 @@ # ------------------ INPUTS TO MAIN PROGRAM ------------------- max_step = 999999 stop_time = 900.0 -max_step = 100 erf.use_terrain = true erf.use_terrain = false @@ -11,16 +10,19 @@ amrex.fpe_trap_invalid = 1 fabarray.mfiter_tile_size = 1024 1024 1024 # PROBLEM SIZE & GEOMETRY +#geometry.prob_lo = -12800. 0. 0. +#geometry.prob_hi = 12800. 1600. 6400. +#xlo.type = "Outflow" + geometry.prob_lo = 0. 0. 0. -geometry.prob_hi = 25600. 400. 6400. +geometry.prob_hi = 25600. 1600. 6400. +xlo.type = "Symmetry" -amr.n_cell = 256 4 64 # dx=dy=dz=100 m, Straka et al 1993 / Xue et al 2000 +xhi.type = "Outflow" -# periodic in x to match WRF setup -# - as an alternative, could use symmetry at x=0 and outflow at x=25600 geometry.is_periodic = 0 1 0 -xlo.type = "Symmetry" -xhi.type = "Outflow" + +amr.n_cell = 256 16 64 # dx=dy=dz=100 m, Straka et al 1993 / Xue et al 2000 zlo.type = "SlipWall" zhi.type = "SlipWall" @@ -42,8 +44,8 @@ erf.check_file = chk # root name of checkpoint file erf.check_int = -57600 # number of timesteps between checkpoints # PLOTFILES -erf.plot_file_1 = plt # prefix of plotfile name -erf.plot_int_1 = 100 # number of timesteps between plotfiles +erf.plot_file_1 = pltc # prefix of plotfile name +erf.plot_int_1 = 10 # number of timesteps between plotfiles erf.plot_vars_1 = density x_velocity y_velocity z_velocity pressure theta pres_hse dens_hse pert_pres pert_dens # SOLVER CHOICE diff --git a/Exec/RegTests/WitchOfAgnesi/ERF_prob.cpp b/Exec/RegTests/WitchOfAgnesi/ERF_prob.cpp index 2c315c0de..d30a98cc7 100644 --- a/Exec/RegTests/WitchOfAgnesi/ERF_prob.cpp +++ b/Exec/RegTests/WitchOfAgnesi/ERF_prob.cpp @@ -48,9 +48,6 @@ Problem::init_custom_pert ( Array4 const& /*mf_v*/, const SolverChoice& sc) { - //const int khi = geomdata.Domain().bigEnd()[2]; - //AMREX_ALWAYS_ASSERT(bx.length()[2] == khi+1); - const bool use_moisture = (sc.moisture_type != MoistureType::None); ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept diff --git a/Source/DataStructs/ERF_DataStruct.H b/Source/DataStructs/ERF_DataStruct.H index 41b896286..1a1a17a4d 100644 --- a/Source/DataStructs/ERF_DataStruct.H +++ b/Source/DataStructs/ERF_DataStruct.H @@ -197,7 +197,19 @@ struct SolverChoice { for (int i = 0; i <= max_level; ++i) anelastic.push_back(anelastic_in[0]); } - pp.query("constant_density", constant_density); + bool any_anelastic = false; + for (int i = 0; i <= max_level; ++i) { + if (anelastic[i] == 1) any_anelastic = true; + } + + // If anelastic at all, we do not advect rho -- it is always == rho0 + if (any_anelastic == 1) { + constant_density = true; + } else { + constant_density = false; // We default to false but allow the user to set it + pp.query("constant_density", constant_density); + } + pp.query("project_every_stage", project_every_stage); pp.query("ncorr", ncorr); pp.query("poisson_abstol", poisson_abstol); diff --git a/Source/ERF.cpp b/Source/ERF.cpp index e39b5056c..1325baf74 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -1627,17 +1627,23 @@ ERF::MakeHorizontalAverages () auto domain = geom[0].Domain(); bool use_moisture = (solverChoice.moisture_type != MoistureType::None); + bool is_anelastic = (solverChoice.anelastic[lev] == 1); for (MFIter mfi(mf); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); auto fab_arr = mf.array(mfi); + auto const hse_arr = base_state[lev].const_array(mfi); auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi); ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { Real dens = cons_arr(i, j, k, Rho_comp); fab_arr(i, j, k, 0) = dens; fab_arr(i, j, k, 1) = cons_arr(i, j, k, RhoTheta_comp) / dens; if (!use_moisture) { - fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp)); + if (is_anelastic) { + fab_arr(i,j,k,2) = hse_arr(i,j,k,1); + } else { + fab_arr(i,j,k,2) = getPgivenRTh(cons_arr(i,j,k,RhoTheta_comp)); + } } }); } @@ -1647,13 +1653,18 @@ ERF::MakeHorizontalAverages () for (MFIter mfi(mf); mfi.isValid(); ++mfi) { const Box& bx = mfi.validbox(); auto fab_arr = mf.array(mfi); + auto const hse_arr = base_state[lev].const_array(mfi); auto const cons_arr = vars_new[lev][Vars::cons].const_array(mfi); auto const qv_arr = qmoist[lev][0]->const_array(mfi); int ncomp = vars_new[lev][Vars::cons].nComp(); ParallelFor(bx, [=] AMREX_GPU_DEVICE (int i, int j, int k) { Real dens = cons_arr(i, j, k, Rho_comp); - fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv_arr(i,j,k)); + if (is_anelastic) { + fab_arr(i,j,k,2) = hse_arr(i,j,k,1); + } else { + fab_arr(i, j, k, 2) = getPgivenRTh(cons_arr(i, j, k, RhoTheta_comp), qv_arr(i,j,k)); + } fab_arr(i, j, k, 3) = (ncomp > RhoQ1_comp ? cons_arr(i, j, k, RhoQ1_comp) / dens : 0.0); fab_arr(i, j, k, 4) = (ncomp > RhoQ2_comp ? cons_arr(i, j, k, RhoQ2_comp) / dens : 0.0); }); diff --git a/Source/IO/ERF_Plotfile.cpp b/Source/IO/ERF_Plotfile.cpp index 087ba9284..7b9f76def 100644 --- a/Source/IO/ERF_Plotfile.cpp +++ b/Source/IO/ERF_Plotfile.cpp @@ -345,47 +345,61 @@ ERF::WritePlotFile (int which, Vector plot_var_names) if (containerHasElement(plot_var_names, "pressure")) { + if (solverChoice.anelastic[lev] == 1) { + MultiFab::Copy(mf[lev], p_hse, 0, mf_comp, 1, 0); + } else #ifdef _OPENMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - const Array4& derdat = mf[lev].array(mfi); - const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); - const int ncomp = vars_new[lev][Vars::cons].nComp(); - - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0; - const Real rhotheta = S_arr(i,j,k,RhoTheta_comp); - derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p); - }); - } + const Box& bx = mfi.tilebox(); + const Array4& derdat = mf[lev].array(mfi); + const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); + const int ncomp = vars_new[lev][Vars::cons].nComp(); + + ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0; + const Real rhotheta = S_arr(i,j,k,RhoTheta_comp); + derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p); + }); + } + } // not anelastic mf_comp += 1; - } + } // pressure + if (containerHasElement(plot_var_names, "pert_pres")) { +#ifdef ERF_USE_POISSON_SOLVE + if (solverChoice.anelastic[lev] == 1) { + MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0); + } else +#endif #ifdef _OPENMP #pragma omp parallel if (amrex::Gpu::notInLaunchRegion()) #endif - for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) { - const Box& bx = mfi.tilebox(); - const Array4& derdat = mf[lev].array(mfi); - const Array4& p0_arr = p_hse.const_array(mfi); - const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); - const int ncomp = vars_new[lev][Vars::cons].nComp(); - - ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + for ( MFIter mfi(mf[lev],TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0; - const Real rhotheta = S_arr(i,j,k,RhoTheta_comp); - derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p) - p0_arr(i,j,k); - }); - } + const Box& bx = mfi.tilebox(); + const Array4& derdat = mf[lev].array(mfi); + const Array4& p0_arr = p_hse.const_array(mfi); + const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); + const int ncomp = vars_new[lev][Vars::cons].nComp(); + + ParallelFor(bx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept + { + Real qv_for_p = (use_moisture && (ncomp > RhoQ1_comp)) ? S_arr(i,j,k,RhoQ1_comp)/S_arr(i,j,k,Rho_comp) : 0; + const Real rhotheta = S_arr(i,j,k,RhoTheta_comp); + derdat(i, j, k, mf_comp) = getPgivenRTh(rhotheta,qv_for_p) - p0_arr(i,j,k); + }); + } + } // not anelastic mf_comp += 1; } + if (containerHasElement(plot_var_names, "pert_dens")) { #ifdef _OPENMP @@ -475,11 +489,18 @@ ERF::WritePlotFile (int which, Vector plot_var_names) { // First define pressure on grown box const Box& gbx = mfi.growntilebox(1); - const Array4 & p_arr = pres.array(mfi); + const Array4 & p_arr = pres.array(mfi); + const Array4 & hse_arr = base_state[lev].const_array(mfi); const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); - ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp)); - }); + if (solverChoice.anelastic[lev] == 1) { + ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { + p_arr(i,j,k) = hse_arr(i,j,k,1); + }); + } else { + ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { + p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp)); + }); + } } pres.FillBoundary(geom[lev].periodicity()); @@ -562,11 +583,18 @@ ERF::WritePlotFile (int which, Vector plot_var_names) { // First define pressure on grown box const Box& gbx = mfi.growntilebox(1); - const Array4 & p_arr = pres.array(mfi); + const Array4 & p_arr = pres.array(mfi); + const Array4 & hse_arr = base_state[lev].const_array(mfi); const Array4& S_arr = vars_new[lev][Vars::cons].const_array(mfi); - ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { - p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp)); - }); + if (solverChoice.anelastic[lev] == 1) { + ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { + p_arr(i,j,k) = hse_arr(i,j,k,1); + }); + } else { + ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { + p_arr(i,j,k) = getPgivenRTh(S_arr(i,j,k,RhoTheta_comp)); + }); + } } pres.FillBoundary(geom[lev].periodicity()); @@ -1076,13 +1104,6 @@ ERF::WritePlotFile (int which, Vector plot_var_names) } #endif -#ifdef ERF_USE_POISSON_SOLVE - if (containerHasElement(plot_var_names, "pp_inc")) { - MultiFab::Copy(mf[lev], pp_inc[lev], 0, mf_comp, 1, 0); - mf_comp += 1; - } -#endif - #ifdef ERF_COMPUTE_ERROR // Next, check for error in velocities and if desired, output them -- note we output none or all, not just some if (containerHasElement(plot_var_names, "xvel_err") || diff --git a/Source/Utils/ERF_PoissonSolve.cpp b/Source/Utils/ERF_PoissonSolve.cpp index c0dc4e1ad..f68326806 100644 --- a/Source/Utils/ERF_PoissonSolve.cpp +++ b/Source/Utils/ERF_PoissonSolve.cpp @@ -1,6 +1,6 @@ #include "ERF.H" #include -#include +#include #include "ERF_Utils.H" #ifdef ERF_USE_POISSON_SOLVE @@ -48,84 +48,30 @@ bool ERF::projection_has_dirichlet (Array bcs) const * Project the single-level velocity field to enforce incompressibility * Note that the level may or may not be level 0. */ -void ERF::project_velocities (int lev, Real l_dt, Vector& vmf, MultiFab& pmf) +void ERF::project_velocities (int lev, Real l_dt, Vector& mom_mf, MultiFab& pmf) { BL_PROFILE("ERF::project_velocities()"); AMREX_ALWAYS_ASSERT(!solverChoice.use_terrain); // Make sure the solver only sees the levels over which we are solving LPInfo info; - Vector ba_tmp; ba_tmp.push_back(vmf[Vars::cons].boxArray()); - Vector dm_tmp; dm_tmp.push_back(vmf[Vars::cons].DistributionMap()); + Vector ba_tmp; ba_tmp.push_back(mom_mf[Vars::cons].boxArray()); + Vector dm_tmp; dm_tmp.push_back(mom_mf[Vars::cons].DistributionMap()); Vector geom_tmp; geom_tmp.push_back(geom[lev]); - MLABecLaplacian mlabec(geom_tmp, ba_tmp, dm_tmp, info); - - // - // This will hold (1/rho) on faces - // - Array inv_rho; - - // - // The operator is (alpha A - beta del dot B grad) phi = RHS - // Here we set alpha to 0 and beta to -1 - // Then b is (dt/rho) - // - mlabec.setScalars(0.0, -1.0); - inv_rho[0].define(vmf[Vars::xvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - inv_rho[1].define(vmf[Vars::yvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - inv_rho[2].define(vmf[Vars::zvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - - MultiFab density(vmf[Vars::cons], make_alias, Rho_comp, 1); - density.FillBoundary(geom_tmp[0].periodicity()); + MLPoisson mlpoisson(geom_tmp, ba_tmp, dm_tmp, info); MultiFab r_hse(base_state[lev], make_alias, 0, 1); // r_0 is first component -#ifdef _OPENMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(density,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& rho_arr = density.const_array(mfi); - Array4 const& rho_0_arr = r_hse.const_array(mfi); - - Box const& bxx = mfi.nodaltilebox(0); - Array4 const& inv_rhox_arr = inv_rho[0].array(mfi); - ParallelFor(bxx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho_edge = Real(0.5) * (rho_arr(i,j,k) + rho_arr(i-1,j,k)); - inv_rhox_arr(i,j,k) = l_dt * rho_0_arr(i,j,k) / rho_edge; - }); - - Box const& bxy = mfi.nodaltilebox(1); - Array4 const& inv_rhoy_arr = inv_rho[1].array(mfi); - ParallelFor(bxy, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho_edge = Real(0.5) * (rho_arr(i,j,k) + rho_arr(i,j-1,k)); - inv_rhoy_arr(i,j,k) = l_dt * rho_0_arr(i,j,k) / rho_edge; - }); - - Box const& bxz = mfi.nodaltilebox(2); - Array4 const& inv_rhoz_arr = inv_rho[2].array(mfi); - ParallelFor(bxz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho_edge = Real(0.5) * (rho_arr(i,j,k) + rho_arr(i,j,k-1)); - Real rho_0_edge = Real(0.5) * (rho_0_arr(i,j,k) + rho_0_arr(i,j,k-1)); - inv_rhoz_arr(i,j,k) = l_dt * rho_0_edge / rho_edge; - }); - } // mfi - - mlabec.setBCoeffs(0, GetArrOfConstPtrs(inv_rho)); - auto bclo = get_projection_bc(Orientation::low); auto bchi = get_projection_bc(Orientation::high); bool need_adjust_rhs = (projection_has_dirichlet(bclo) || projection_has_dirichlet(bchi)) ? false : true; - mlabec.setDomainBC(bclo, bchi); + mlpoisson.setDomainBC(bclo, bchi); if (lev > 0) { - mlabec.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann); + mlpoisson.setCoarseFineBC(nullptr, ref_ratio[lev-1], LinOpBCType::Neumann); } - mlabec.setLevelBC(0, nullptr); + mlpoisson.setLevelBC(0, nullptr); Vector rhs; Vector phi; @@ -144,51 +90,13 @@ void ERF::project_velocities (int lev, Real l_dt, Vector& vmf, MultiFa fluxes[0][idim].define(convert(ba_tmp[0], IntVect::TheDimensionVector(idim)), dm_tmp[0], 1, 0); } - Array rho0_u; - rho0_u[0].define(vmf[Vars::xvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - rho0_u[1].define(vmf[Vars::yvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - rho0_u[2].define(vmf[Vars::zvel].boxArray(),dm_tmp[0],1,0,MFInfo()); - -#ifdef _OPENMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(density,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& rho0_arr = r_hse.const_array(mfi); - - Box const& bxx = mfi.nodaltilebox(0); - Array4 const& u_arr = vmf[Vars::xvel].const_array(mfi); - Array4 const& rho0_u_arr = rho0_u[0].array(mfi); - ParallelFor(bxx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - rho0_u_arr(i,j,k) = u_arr(i,j,k) * rho0_arr(i,j,k); - }); - - Box const& bxy = mfi.nodaltilebox(1); - Array4 const& v_arr = vmf[Vars::yvel].const_array(mfi); - Array4 const& rho0_v_arr = rho0_u[1].array(mfi); - ParallelFor(bxy, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - rho0_v_arr(i,j,k) = v_arr(i,j,k) * rho0_arr(i,j,k); - }); - - Box const& bxz = mfi.nodaltilebox(2); - Array4 const& w_arr = vmf[Vars::zvel].const_array(mfi); - Array4 const& rho0_w_arr = rho0_u[2].array(mfi); - ParallelFor(bxz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho0_edge = Real(0.5) * (rho0_arr(i,j,k) + rho0_arr(i,j,k-1)); - rho0_w_arr(i,j,k) = w_arr(i,j,k) * rho0_edge; - }); - } // mfi - Array rho0_u_const; - rho0_u_const[0] = &rho0_u[0]; - rho0_u_const[1] = &rho0_u[1]; - rho0_u_const[2] = &rho0_u[2]; + rho0_u_const[0] = &mom_mf[IntVars::xmom]; + rho0_u_const[1] = &mom_mf[IntVars::ymom]; + rho0_u_const[2] = &mom_mf[IntVars::zmom]; computeDivergence(rhs[0], rho0_u_const, geom_tmp[0]); - Print() << "Max norm of divergence after at level " << lev << " : " << rhs[0].norm0() << std::endl; + Print() << "Max norm of divergence before at level " << lev << " : " << rhs[0].norm0() << std::endl; // If all Neumann BCs, adjust RHS to make sure we can converge if (need_adjust_rhs) @@ -201,7 +109,10 @@ void ERF::project_velocities (int lev, Real l_dt, Vector& vmf, MultiFa // Initialize phi to 0 phi[0].setVal(0.0); - MLMG mlmg(mlabec); + // Divide rhs by dt so that solution will actually be delta pressure, not delta pressure / time + rhs[0].mult(l_dt); + + MLMG mlmg(mlpoisson); int max_iter = 100; mlmg.setMaxIter(max_iter); @@ -220,83 +131,95 @@ void ERF::project_velocities (int lev, Real l_dt, Vector& vmf, MultiFa pmf.FillBoundary(geom[lev].periodicity()); // Subtract (dt rho0/rho) grad(phi) from the rho0-weighted velocity components - // MultiFab::Add(vmf[Vars::xvel], fluxes[0][0], 0,0,1,0); - // MultiFab::Add(vmf[Vars::yvel], fluxes[0][1], 0,0,1,0); - // MultiFab::Add(vmf[Vars::zvel], fluxes[0][2], 0,0,1,0); - MultiFab::Add(rho0_u[0], fluxes[0][0], 0,0,1,0); - MultiFab::Add(rho0_u[1], fluxes[0][1], 0,0,1,0); - MultiFab::Add(rho0_u[2], fluxes[0][2], 0,0,1,0); + MultiFab::Add(mom_mf[IntVars::xmom],fluxes[0][0],0,0,1,0); + MultiFab::Add(mom_mf[IntVars::ymom],fluxes[0][1],0,0,1,0); + MultiFab::Add(mom_mf[IntVars::zmom],fluxes[0][2],0,0,1,0); + + auto const dom_lo = lbound(geom[lev].Domain()); + auto const dom_hi = ubound(geom[lev].Domain()); #ifdef _OPENMP #pragma omp parallel if (Gpu::notInLaunchRegion()) #endif - for (MFIter mfi(density,TilingIfNotGPU()); mfi.isValid(); ++mfi) + for (MFIter mfi(mom_mf[Vars::cons],TilingIfNotGPU()); mfi.isValid(); ++mfi) { - Array4 const& rho0_arr = r_hse.const_array(mfi); - - Box const& bxx = mfi.nodaltilebox(0); - Array4 const& u_arr = vmf[Vars::xvel].array(mfi); - Array4 const& rho0_u_arr = rho0_u[0].array(mfi); - ParallelFor(bxx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - u_arr(i,j,k) = rho0_u_arr(i,j,k) / rho0_arr(i,j,k); - }); - - Box const& bxy = mfi.nodaltilebox(1); - Array4 const& v_arr = vmf[Vars::yvel].array(mfi); - Array4 const& rho0_v_arr = rho0_u[1].array(mfi); - ParallelFor(bxy, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - v_arr(i,j,k) = rho0_v_arr(i,j,k) / rho0_arr(i,j,k); - }); - - Box const& bxz = mfi.nodaltilebox(2); - Array4 const& w_arr = vmf[Vars::zvel].array(mfi); - Array4 const& rho0_w_arr = rho0_u[2].array(mfi); - ParallelFor(bxz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho0_edge = Real(0.5) * (rho0_arr(i,j,k) + rho0_arr(i,j,k-1)); - w_arr(i,j,k) = rho0_w_arr(i,j,k) / rho0_edge; - }); + Array4 const& pp_arr = pmf.array(mfi); + Box const& bx = mfi.tilebox(); + auto const bx_lo = lbound(bx); + auto const bx_hi = ubound(bx); + if (bx_lo.x == dom_lo.x) { + if (bclo[0] == LinOpBCType::Dirichlet) { + ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i-1,j,k) = -pp_arr(i,j,k); + }); + } else { + ParallelFor(makeSlab(bx,0,dom_lo.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i-1,j,k) = pp_arr(i,j,k); + }); + } + } + if (bx_lo.y == dom_lo.y) { + if (bclo[1] == LinOpBCType::Dirichlet) { + ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j-1,k) = -pp_arr(i,j,k); + }); + } else { + ParallelFor(makeSlab(bx,1,dom_lo.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j-1,k) = pp_arr(i,j,k); + }); + } + } + if (bx_lo.z == dom_lo.z) { + ParallelFor(makeSlab(bx,2,dom_lo.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j,k-1) = pp_arr(i,j,k); + }); + } + if (bx_hi.x == dom_hi.x) { + if (bchi[0] == LinOpBCType::Dirichlet) { + ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i+1,j,k) = -pp_arr(i,j,k); + }); + } else { + ParallelFor(makeSlab(bx,0,dom_hi.x), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i+1,j,k) = pp_arr(i,j,k); + }); + } + } + if (bx_hi.y == dom_hi.y) { + if (bchi[1] == LinOpBCType::Dirichlet) { + ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j+1,k) = -pp_arr(i,j,k); + }); + } else { + ParallelFor(makeSlab(bx,1,dom_hi.y), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j+1,k) = pp_arr(i,j,k); + }); + } + } + if (bx_hi.z == dom_hi.z) { + ParallelFor(makeSlab(bx,2,dom_hi.z), [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept + { + pp_arr(i,j,k+1) = pp_arr(i,j,k); + }); + } } // mfi + // Now overwrite with periodic fill outside domain and fine-fine fill inside + pmf.FillBoundary(geom[lev].periodicity()); + #if 0 // // BELOW IS SIMPLY VERIFYING THE DIVERGENCE AFTER THE SOLVE // -#ifdef _OPENMP -#pragma omp parallel if (Gpu::notInLaunchRegion()) -#endif - for (MFIter mfi(density,TilingIfNotGPU()); mfi.isValid(); ++mfi) - { - Array4 const& rho0_arr = r_hse.const_array(mfi); - - Box const& bxx = mfi.nodaltilebox(0); - Array4 const& u_arr = vmf[Vars::xvel].const_array(mfi); - Array4 const& rho0_u_arr = rho0_u[0].array(mfi); - ParallelFor(bxx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - rho0_u_arr(i,j,k) = u_arr(i,j,k) * rho0_arr(i,j,k); - }); - - Box const& bxy = mfi.nodaltilebox(1); - Array4 const& v_arr = vmf[Vars::yvel].const_array(mfi); - Array4 const& rho0_v_arr = rho0_u[1].array(mfi); - ParallelFor(bxy, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - rho0_v_arr(i,j,k) = v_arr(i,j,k) * rho0_arr(i,j,k); - }); - - Box const& bxz = mfi.nodaltilebox(2); - Array4 const& w_arr = vmf[Vars::zvel].const_array(mfi); - Array4 const& rho0_w_arr = rho0_u[2].array(mfi); - ParallelFor(bxz, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept - { - Real rho0_edge = Real(0.5) * (rho0_arr(i,j,k) + rho0_arr(i,j,k-1)); - rho0_w_arr(i,j,k) = w_arr(i,j,k) * rho0_edge; - }); - } // mfi - computeDivergence(rhs[0], rho0_u_const, geom_tmp[0]); Print() << "Max norm of divergence after solve at level " << lev << " : " << rhs[0].norm0() << std::endl; #endif