diff --git a/Source/ERF.H b/Source/ERF.H index 1c3ffd66e..ab20060a6 100644 --- a/Source/ERF.H +++ b/Source/ERF.H @@ -402,8 +402,7 @@ public: const amrex::Real& dt_advance, amrex::MultiFab& cons_in, amrex::MultiFab& U_old, amrex::MultiFab& V_old, amrex::MultiFab& W_old, - amrex::MultiFab& mf_vars_windfarm, const amrex::MultiFab& mf_Nturb, - const amrex::MultiFab& mf_SMark); + amrex::MultiFab& mf_vars_windfarm, const amrex::MultiFab& mf_Nturb); #endif #ifdef ERF_USE_EB @@ -432,7 +431,7 @@ private: void update_diffusive_arrays (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm); void init_zphys (int lev, amrex::Real time); - void remake_zphys (int lev, amrex::Real time, std::unique_ptr& temp_zphys_nd); + void remake_zphys (int lev, std::unique_ptr& temp_zphys_nd); void update_terrain_arrays (int lev); void Construct_ERFFillPatchers (int lev); @@ -744,7 +743,7 @@ private: amrex::Vector> SFS_q2fx3_lev; // Terrain / grid stretching - amrex::Vector zlevels_stag; // nominal height levels + amrex::Vector> zlevels_stag; // nominal height levels amrex::Vector> z_phys_nd; amrex::Vector> z_phys_cc; @@ -1048,7 +1047,7 @@ private: // amrex::MultiFab fine_mask; - amrex::Real dz_min; + amrex::Vector dz_min; static AMREX_FORCE_INLINE int diff --git a/Source/ERF.cpp b/Source/ERF.cpp index 3093a7383..e39b5056c 100644 --- a/Source/ERF.cpp +++ b/Source/ERF.cpp @@ -146,21 +146,24 @@ ERF::ERF_shared () // Initialize staggered vertical levels for grid stretching or terrain, and // to simplify Rayleigh damping layer calculations. + zlevels_stag.resize(max_level+1); init_zlevels(zlevels_stag, - geom[0], + geom, + refRatio(), solverChoice.grid_stretching_ratio, solverChoice.zsurf, solverChoice.dz0); + if (solverChoice.use_terrain) { int nz = geom[0].Domain().length(2) + 1; // staggered - if (std::fabs(zlevels_stag[nz-1]-geom[0].ProbHi(2)) > 1.0e-4) { + if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) { Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2) - << " does not match highest requested z level " << zlevels_stag[nz-1] + << " does not match highest requested z level " << zlevels_stag[0][nz-1] << std::endl; } - if (std::fabs(zlevels_stag[0]-geom[0].ProbLo(2)) > 1.0e-4) { + if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) { Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2) - << " does not match lowest requested level " << zlevels_stag[0] + << " does not match lowest requested level " << zlevels_stag[0][0] << std::endl; } @@ -777,7 +780,7 @@ ERF::InitData () h_u_geos[lev], d_u_geos[lev], h_v_geos[lev], d_v_geos[lev], geom[lev], - zlevels_stag); + zlevels_stag[0]); } } } @@ -871,6 +874,7 @@ ERF::InitData () } } #endif + // Copy from new into old just in case for (int lev = 0; lev <= finest_level; ++lev) { @@ -885,10 +889,14 @@ ERF::InitData () MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0, 1,lev_new[Vars::zvel].nGrowVect()); } - // Compute the minimum dz in the domain (to be used for setting the timestep) - dz_min = geom[0].CellSize(2); - if ( solverChoice.use_terrain ) { - dz_min *= (*detJ_cc[0]).min(0); + // Compute the minimum dz in the domain at each level (to be used for setting the timestep) + dz_min.resize(max_level+1); + for (int lev = 0; lev <= finest_level; ++lev) + { + dz_min[lev] = geom[lev].CellSize(2); + if ( solverChoice.use_terrain ) { + dz_min[lev] *= (*detJ_cc[lev]).min(0); + } } ComputeDt(); diff --git a/Source/ERF_Tagging.cpp b/Source/ERF_Tagging.cpp index 3bbb06aa4..a2c028eee 100644 --- a/Source/ERF_Tagging.cpp +++ b/Source/ERF_Tagging.cpp @@ -160,7 +160,7 @@ ERF::refinement_criteria_setup () khi = domain.smallEnd(2) - 1; for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) { - if (zlevels_stag[k] > rbox_lo[2]) { + if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) { klo = k-1; break; } @@ -168,7 +168,7 @@ ERF::refinement_criteria_setup () AMREX_ASSERT(klo >= domain.smallEnd(2)); for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) { - if (zlevels_stag[k] > rbox_hi[2]) { + if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) { khi = k-1; break; } diff --git a/Source/ERF_make_new_arrays.cpp b/Source/ERF_make_new_arrays.cpp index 345b52670..14593569b 100644 --- a/Source/ERF_make_new_arrays.cpp +++ b/Source/ERF_make_new_arrays.cpp @@ -441,26 +441,24 @@ ERF::update_diffusive_arrays (int lev, const BoxArray& ba, const DistributionMap void ERF::init_zphys (int lev, Real time) { - if (solverChoice.use_terrain) { - // - // First interpolate from coarser level if there is one - // + if (solverChoice.use_terrain) + { if (lev > 0) { - Vector fmf = {z_phys_nd[lev].get(), z_phys_nd[lev].get()}; - Vector ftime = {t_old[lev], t_new[lev]}; - Vector cmf = {z_phys_nd[lev-1].get(), z_phys_nd[lev-1].get()}; - Vector ctime = {t_old[lev-1], t_new[lev-1]}; - // - // First we fill z_phys_nd at lev>0 through interpolation + // First interpolate from coarser level if there is one + // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab + // have been pre-filled - this includes ghost cells both inside and outside + // the domain // - Interpolater* mapper = &node_bilinear_interp; - PhysBCFunctNoOp null_bc; - InterpFromCoarseLevel(*z_phys_nd[lev], time, *z_phys_nd[lev-1], - 0, 0, 1, + InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(), + IntVect(0,0,0), // do not fill ghost cells outside the domain + *z_phys_nd[lev-1], 0, 0, 1, geom[lev-1], geom[lev], - null_bc, 0, null_bc, 0, refRatio(lev-1), - mapper, domain_bcs_type, 0); + refRatio(lev-1), &node_bilinear_interp, domain_bcs_type); + + // This recomputes the fine values using the bottom terrain at the fine resolution, + // and also fills values of z_phys_nd outside the domain + init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type); } // @@ -478,35 +476,38 @@ ERF::init_zphys (int lev, Real time) { z_phys_nd[lev]->setVal(-1.e23); prob->init_custom_terrain(geom[lev],*z_phys_nd[lev],time); - init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag,phys_bc_type); + init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type); Real zmax = z_phys_nd[0]->max(0,0,false); - Real rel_diff = (zmax - zlevels_stag[zlevels_stag.size()-1]) / zmax; + Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax; AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!"); } // init_type + z_phys_nd[lev]->FillBoundary(geom[lev].periodicity()); } // lev == 0 - } + } // terrain } void -ERF::remake_zphys (int lev, Real time, std::unique_ptr& temp_zphys_nd) +ERF::remake_zphys (int lev, std::unique_ptr& temp_zphys_nd) { - if (solverChoice.use_terrain && lev > 0) { - - Vector fmf = {z_phys_nd[lev].get(), z_phys_nd[lev].get()}; - Vector cmf = {z_phys_nd[lev-1].get(), z_phys_nd[lev-1].get()}; - Vector ftime = {time, time}; - Vector ctime = {time, time}; - - PhysBCFunctNoOp null_bc; - Interpolater* mapper = &node_bilinear_interp; - - FillPatchTwoLevels(*temp_zphys_nd, time, - cmf, ctime, fmf, ftime, - 0, 0, 1, geom[lev-1], geom[lev], - null_bc, 0, null_bc, 0, refRatio(lev-1), - mapper, domain_bcs_type, 0); + if (solverChoice.use_terrain && lev > 0) + { + // + // First interpolate from coarser level + // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab + // have been pre-filled - this includes ghost cells both inside and outside + // the domain + // + InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(), + IntVect(0,0,0), // do not fill ghost cells outside the domain + *z_phys_nd[lev-1], 0, 0, 1, + geom[lev-1], geom[lev], + refRatio(lev-1), &node_bilinear_interp, domain_bcs_type); + + // This recomputes the fine values using the bottom terrain at the fine resolution, + // and also fills values of z_phys_nd outside the domain + init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type); std::swap(temp_zphys_nd, z_phys_nd[lev]); diff --git a/Source/ERF_make_new_level.cpp b/Source/ERF_make_new_level.cpp index ee71e71e6..8e91f49f6 100644 --- a/Source/ERF_make_new_level.cpp +++ b/Source/ERF_make_new_level.cpp @@ -313,21 +313,16 @@ ERF::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, // ******************************************************************************************** // Update the base state at this level by interpolation from coarser level // ******************************************************************************************** - // Interp all three components: rho, p, pi - int icomp = 0; int bccomp = 0; int ncomp = 3; - - PhysBCFunctNoOp null_bc; - Interpolater* mapper = &cell_cons_interp; - - Vector fmf = {&base_state[lev ], &base_state[lev ]}; - Vector cmf = {&base_state[lev-1], &base_state[lev-1]}; - Vector ftime = {time, time}; - Vector ctime = {time, time}; - InterpFromCoarseLevel(base_state[lev], time, base_state[lev-1], - icomp, icomp, ncomp, + // + // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab + // have been pre-filled - this includes ghost cells both inside and outside + // the domain + // + InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(), + IntVect(0,0,0), // do not fill ghost cells outside the domain + base_state[lev-1], 0, 0, 3, geom[lev-1], geom[lev], - null_bc, 0, null_bc, 0, refRatio(lev-1), - mapper, domain_bcs_type, bccomp); + refRatio(lev-1), &cell_cons_interp, domain_bcs_type); initHSE(lev); @@ -375,7 +370,7 @@ ERF::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba, void ERF::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMapping& dm) { - amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl; + // amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl; AMREX_ALWAYS_ASSERT(lev > 0); AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::Moving); @@ -406,7 +401,7 @@ ERF::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMapp // ******************************************************************************************** // Build the data structures for terrain-related quantities // ******************************************************************************************** - remake_zphys(lev, time, temp_zphys_nd); + remake_zphys(lev, temp_zphys_nd); update_terrain_arrays(lev); // diff --git a/Source/IO/ERF_Write1DProfiles.cpp b/Source/IO/ERF_Write1DProfiles.cpp index 43c473d90..450515a7a 100644 --- a/Source/IO/ERF_Write1DProfiles.cpp +++ b/Source/IO/ERF_Write1DProfiles.cpp @@ -66,8 +66,8 @@ ERF::write_1D_profiles (Real time) // Write the quantities at this time for (int k = 0; k < hu_size; k++) { Real z; - if (zlevels_stag.size() > 1) { - z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]); + if (zlevels_stag[0].size() > 1) { + z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]); } else { z = (k + 0.5)* dx[2]; } @@ -89,8 +89,8 @@ ERF::write_1D_profiles (Real time) // Write the perturbational quantities at this time for (int k = 0; k < hu_size; k++) { Real z; - if (zlevels_stag.size() > 1) { - z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]); + if (zlevels_stag[0].size() > 1) { + z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]); } else { z = (k + 0.5)* dx[2]; } @@ -144,8 +144,8 @@ ERF::write_1D_profiles (Real time) // Write the average stresses for (int k = 0; k < hu_size; k++) { Real z; - if (zlevels_stag.size() > 1) { - z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]); + if (zlevels_stag[0].size() > 1) { + z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]); } else { z = (k + 0.5)* dx[2]; } diff --git a/Source/IO/ERF_Write1DProfiles_stag.cpp b/Source/IO/ERF_Write1DProfiles_stag.cpp index 693da0e26..81fa117f5 100644 --- a/Source/IO/ERF_Write1DProfiles_stag.cpp +++ b/Source/IO/ERF_Write1DProfiles_stag.cpp @@ -73,7 +73,7 @@ ERF::write_1D_profiles_stag (Real time) if (data_log1.good()) { // Write the quantities at this time for (int k = 0; k < unstag_size; k++) { - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2]; data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " " << std::setw(datwidth) << std::setprecision(datprecision) << z << " " << h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " " @@ -84,7 +84,7 @@ ERF::write_1D_profiles_stag (Real time) << std::endl; } // loop over z // Write top face values - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2]; data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " " << std::setw(datwidth) << std::setprecision(datprecision) << z << " " << 0 << " " << 0 << " " << h_avg_w[unstag_size+1] << " " @@ -139,7 +139,7 @@ ERF::write_1D_profiles_stag (Real time) // For internal values, interpolate scalar quantities to faces for (int k = 1; k < unstag_size; k++) { - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2]; Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]); Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]); Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]); @@ -207,7 +207,7 @@ ERF::write_1D_profiles_stag (Real time) Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2]; Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2]; Real thvface = thface * (1 + 0.61*qvface - qcface - qrface); - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2]; data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " " << std::setw(datwidth) << std::setprecision(datprecision) << z << " " << 0 << " " // u'u' @@ -243,7 +243,7 @@ ERF::write_1D_profiles_stag (Real time) if (data_log3.good()) { // Write the average stresses for (int k = 0; k < unstag_size; k++) { - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2]; data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " " << std::setw(datwidth) << std::setprecision(datprecision) << z << " " << h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " " @@ -255,7 +255,7 @@ ERF::write_1D_profiles_stag (Real time) } // loop over z // Write top face values Real NANval = 0.0; - Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2]; + Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2]; data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " " << std::setw(datwidth) << std::setprecision(datprecision) << z << " " << NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " " diff --git a/Source/Initialization/ERF_init1d.cpp b/Source/Initialization/ERF_init1d.cpp index 6acefbed3..bb80de73b 100644 --- a/Source/Initialization/ERF_init1d.cpp +++ b/Source/Initialization/ERF_init1d.cpp @@ -21,7 +21,7 @@ void ERF::initRayleigh () { const int khi = geom[0].Domain().bigEnd(2); - solverChoice.rayleigh_ztop = (solverChoice.use_terrain) ? zlevels_stag[khi+1] : geom[0].ProbHi(2); + solverChoice.rayleigh_ztop = (solverChoice.use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2); h_rayleigh_ptrs.resize(max_level+1); d_rayleigh_ptrs.resize(max_level+1); @@ -67,7 +67,7 @@ ERF::setRayleighRefFromSounding (bool restarting) if (restarting) { input_sounding_data.resize_arrays(); for (int n = 0; n < input_sounding_data.n_sounding_files; n++) { - input_sounding_data.read_from_file(geom[0], zlevels_stag, n); + input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n); } } @@ -85,8 +85,8 @@ ERF::setRayleighRefFromSounding (bool restarting) const int Nz = khi - klo + 1; Vector zcc(Nz); - Vector zlevels_sub(zlevels_stag.begin()+klo/refine_fac, - zlevels_stag.begin()+khi/refine_fac+2); + Vector zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac, + zlevels_stag[0].begin()+khi/refine_fac+2); expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true); #if 0 amrex::AllPrint() << "lev="< 0); - const Real zbot = (use_terrain) ? zlevels_stag[klo] : geom[lev].ProbLo(2); - const Real ztop = (use_terrain) ? zlevels_stag[khi+1] : geom[lev].ProbHi(2); + const bool use_terrain = (zlevels_stag[0].size() > 0); + const Real zbot = (use_terrain) ? zlevels_stag[0][klo] : geom[lev].ProbLo(2); + const Real ztop = (use_terrain) ? zlevels_stag[0][khi+1] : geom[lev].ProbHi(2); // Size of Nz (domain grid) Vector zcc_inp(Nz ); @@ -646,9 +646,9 @@ void ERF::init_Dirichlet_bc_data (const std::string input_file) // z_inp_tmp[N-1] >= ztop. Now, interpolate to grid level 0 heights const int Ninp = z_inp_tmp.size(); for (int k(0); k 0) { - // Interp all three components: rho, p, pi - int icomp = 0; int bccomp = 0; int ncomp = 3; - - PhysBCFunctNoOp null_bc; - Interpolater* mapper = &cell_cons_interp; - - Real time = 0.; - - Vector fmf = {&base_state[lev ], &base_state[lev ]}; - Vector cmf = {&base_state[lev-1], &base_state[lev-1]}; - Vector ftime = {time, time}; - Vector ctime = {time, time}; - FillPatchTwoLevels(base_state[lev], time, - cmf, ctime, fmf, ftime, - icomp, icomp, ncomp, geom[lev-1], geom[lev], - null_bc, 0, null_bc, 0, refRatio(lev-1), - mapper, domain_bcs_type, bccomp); + if (lev > 0) + { + base_state[lev-1].FillBoundary(geom[lev-1].periodicity()); + // + // NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab + // have been pre-filled - this includes ghost cells both inside and outside + // the domain + // + InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(), + IntVect(0,0,0), // do not fill ghost cells outside the domain + base_state[lev-1], 0, 0, 3, + geom[lev-1], geom[lev], + refRatio(lev-1), &cell_cons_interp, domain_bcs_type); } } diff --git a/Source/Initialization/ERF_init_from_metgrid.cpp b/Source/Initialization/ERF_init_from_metgrid.cpp index d693ee3ab..9c58ef537 100644 --- a/Source/Initialization/ERF_init_from_metgrid.cpp +++ b/Source/Initialization/ERF_init_from_metgrid.cpp @@ -138,7 +138,7 @@ ERF::init_from_metgrid (int lev) } // mf // This defines all the z(i,j,k) values given z(i,j,0) from above. - init_terrain_grid(lev, geom[lev], *z_phys, zlevels_stag, phys_bc_type); + init_terrain_grid(lev, geom[lev], *z_phys, zlevels_stag[lev], phys_bc_type); // Copy LATITUDE, LONGITUDE, SST and LANDMASK data into MF and iMF data structures auto& ba = lev_new[Vars::cons].boxArray(); diff --git a/Source/Initialization/ERF_input_sponge.cpp b/Source/Initialization/ERF_input_sponge.cpp index 433c72805..19e4219a1 100644 --- a/Source/Initialization/ERF_input_sponge.cpp +++ b/Source/Initialization/ERF_input_sponge.cpp @@ -26,6 +26,6 @@ ERF::input_sponge (int lev) // this will interpolate the input profiles to the nominal height levels // (ranging from 0 to the domain top) - input_sponge_data.read_from_file(geom[lev], zlevels_stag); + input_sponge_data.read_from_file(geom[lev], zlevels_stag[lev]); } } diff --git a/Source/TimeIntegration/ERF_ComputeTimestep.cpp b/Source/TimeIntegration/ERF_ComputeTimestep.cpp index 1bb725ce7..2fd44769a 100644 --- a/Source/TimeIntegration/ERF_ComputeTimestep.cpp +++ b/Source/TimeIntegration/ERF_ComputeTimestep.cpp @@ -59,8 +59,12 @@ ERF::estTimeStep (int level, long& dt_fast_ratio) const Real estdt_comp = 1.e20; Real estdt_lowM = 1.e20; + // We intentionally use the level 0 domain to compute whether to use this direction in the dt calculation + const int nxc = geom[0].Domain().length(0); + const int nyc = geom[0].Domain().length(1); + auto const dxinv = geom[level].InvCellSizeArray(); - auto const dzinv = 1.0 / dz_min; + auto const dzinv = 1.0 / dz_min[level]; MultiFab const& S_new = vars_new[level][Vars::cons]; @@ -111,15 +115,33 @@ ERF::estTimeStep (int level, long& dt_fast_ratio) const // If we are not doing the acoustic substepping, then the z-direction contributes // to the computation of the time step if (l_no_substepping) { - new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), - ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), - ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt); + if (nxc > 1 && nyc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), + ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), + ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt); + } else if (nxc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), + ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt); + } else if (nyc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), + ((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt); + } else { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,2))+c)*dzinv ), new_comp_dt); + } // If we are doing the acoustic substepping, then the z-direction does not contribute // to the computation of the time step } else { - new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), - ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt); + if (nxc > 1 && nyc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), + ((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt); + } else if (nxc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,0))+c)*dxinv[0]), new_comp_dt); + } else if (nyc > 1) { + new_comp_dt = amrex::max(((amrex::Math::abs(u(i,j,k,1))+c)*dxinv[1]), new_comp_dt); + } else { + amrex::Abort("Not sure how to compute dt for this case"); + } } } }); @@ -149,7 +171,8 @@ ERF::estTimeStep (int level, long& dt_fast_ratio) const if (verbose) { if (fixed_dt[level] <= 0.0) { - Print() << "Using cfl = " << cfl << std::endl; + Print() << "Using cfl = " << cfl << " and dx/dy/dz_min = " << + 1.0/dxinv[0] << " " << 1.0/dxinv[1] << " " << dz_min[level] << std::endl; Print() << "Compressible dt at level " << level << ": " << estdt_comp << std::endl; if (estdt_lowM_inv > 0.0_rt) { Print() << "Anelastic dt at level " << level << ": " << estdt_lowM << std::endl; diff --git a/Source/TimeIntegration/ERF_TI_fast_rhs_fun.H b/Source/TimeIntegration/ERF_TI_fast_rhs_fun.H index d8fc1bd26..f61dba333 100644 --- a/Source/TimeIntegration/ERF_TI_fast_rhs_fun.H +++ b/Source/TimeIntegration/ERF_TI_fast_rhs_fun.H @@ -45,13 +45,13 @@ auto fast_rhs_fun = [&](int fast_step, int /*n_sub*/, int nrk, // Make "old" fast geom -- store in z_phys_nd for convenience if (verbose) Print() << "Making geometry at start of substep time: " << old_substep_time << std::endl; prob->init_custom_terrain(fine_geom,*z_phys_nd[level],old_substep_time); - init_terrain_grid (level,fine_geom,*z_phys_nd[level], zlevels_stag, phys_bc_type); + init_terrain_grid (level,fine_geom,*z_phys_nd[level], zlevels_stag[level], phys_bc_type); make_J (fine_geom,*z_phys_nd[level], *detJ_cc[level]); // Make "new" fast geom if (verbose) Print() << "Making geometry for end of substep time :" << new_substep_time << std::endl; prob->init_custom_terrain(fine_geom,*z_phys_nd_new[level],new_substep_time); - init_terrain_grid (level,fine_geom,*z_phys_nd_new[level], zlevels_stag, phys_bc_type); + init_terrain_grid (level,fine_geom,*z_phys_nd_new[level], zlevels_stag[level], phys_bc_type); make_J (fine_geom,*z_phys_nd_new[level], *detJ_cc_new[level]); make_areas (fine_geom,*z_phys_nd_new[level], *ax_new[level], *ay_new[level], *az_new[level]); diff --git a/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H b/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H index d9401f2c8..784024334 100644 --- a/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H +++ b/Source/TimeIntegration/ERF_TI_slow_rhs_fun.H @@ -61,19 +61,19 @@ if (verbose) Print() << "Re-making old geometry at old time : " << old_step_time << std::endl; prob->init_custom_terrain(fine_geom,*z_phys_nd[level],old_step_time); - init_terrain_grid (level,fine_geom,*z_phys_nd[level], zlevels_stag, phys_bc_type); + init_terrain_grid (level,fine_geom,*z_phys_nd[level], zlevels_stag[level], phys_bc_type); make_J (fine_geom,*z_phys_nd[level], *detJ_cc[level]); make_areas (fine_geom,*z_phys_nd[level], *ax[level], *ay[level], *az[level]); if (verbose) Print() << "Making src geometry at old_stage_time: " << old_stage_time << std::endl; prob->init_custom_terrain(fine_geom,*z_phys_nd_src[level],old_stage_time); - init_terrain_grid (level,fine_geom,*z_phys_nd_src[level], zlevels_stag, phys_bc_type); + init_terrain_grid (level,fine_geom,*z_phys_nd_src[level], zlevels_stag[level], phys_bc_type); make_J (fine_geom,*z_phys_nd_src[level], *detJ_cc_src[level]); make_areas (fine_geom,*z_phys_nd_src[level], *ax_src[level], *ay_src[level], *az_src[level]); if (verbose) Print() << "Making new geometry at new_stage_time: " << new_stage_time << std::endl; prob->init_custom_terrain(fine_geom,*z_phys_nd_new[level],new_stage_time); - init_terrain_grid (level,fine_geom,*z_phys_nd_new[level], zlevels_stag, phys_bc_type); + init_terrain_grid (level,fine_geom,*z_phys_nd_new[level], zlevels_stag[level], phys_bc_type); make_J (fine_geom,*z_phys_nd_new[level], *detJ_cc_new[level]); make_areas (fine_geom,*z_phys_nd_new[level], *ax_new[level], *ay_new[level], *az_new[level]); diff --git a/Source/TimeIntegration/ERF_slow_rhs_pre.cpp b/Source/TimeIntegration/ERF_slow_rhs_pre.cpp index 2c4a5dcb7..934543e4b 100644 --- a/Source/TimeIntegration/ERF_slow_rhs_pre.cpp +++ b/Source/TimeIntegration/ERF_slow_rhs_pre.cpp @@ -349,14 +349,15 @@ void erf_slow_rhs_pre (int level, int finest_level, ParallelFor(gbx, [=] AMREX_GPU_DEVICE (int i, int j, int k) noexcept { #ifdef AMREX_USE_GPU - if (cell_data(i,j,k,RhoTheta_comp) < 0.) AMREX_DEVICE_PRINTF("BAD THETA AT %d %d %d %e %e \n", + if (cell_data(i,j,k,RhoTheta_comp) <= 0.) AMREX_DEVICE_PRINTF("BAD THETA AT %d %d %d %e %e \n", i,j,k,cell_data(i,j,k,RhoTheta_comp),cell_data(i,j,k+1,RhoTheta_comp)); #else - if (cell_data(i,j,k,RhoTheta_comp) < 0.) printf("BAD THETA AT %d %d %d %e %e \n", + if (cell_data(i,j,k,RhoTheta_comp) <= 0.) { + printf("BAD THETA AT %d %d %d %e %e \n", i,j,k,cell_data(i,j,k,RhoTheta_comp),cell_data(i,j,k+1,RhoTheta_comp)); + amrex::Abort("Bad theta in ERF_slow_rhs_pre"); + } #endif - - AMREX_ASSERT(cell_data(i,j,k,RhoTheta_comp) > 0.); Real qv_for_p = (l_use_moisture) ? cell_data(i,j,k,RhoQ1_comp)/cell_data(i,j,k,Rho_comp) : 0.0; pptemp_arr(i,j,k) = getPgivenRTh(cell_data(i,j,k,RhoTheta_comp),qv_for_p) - p0_arr(i,j,k); }); diff --git a/Source/Utils/ERF_TerrainMetrics.H b/Source/Utils/ERF_TerrainMetrics.H index f7d38e670..9b09d61e1 100644 --- a/Source/Utils/ERF_TerrainMetrics.H +++ b/Source/Utils/ERF_TerrainMetrics.H @@ -11,8 +11,9 @@ */ // Declare functions for ERF.cpp -void init_zlevels (amrex::Vector& zlevels_stag, - const amrex::Geometry& geom, +void init_zlevels (amrex::Vector> & zlevels_stag, + amrex::Vector const& geom, + amrex::Vector const& ref_ratio, const amrex::Real grid_stretching_ratio, const amrex::Real zsurf, const amrex::Real dz0); diff --git a/Source/Utils/ERF_TerrainMetrics.cpp b/Source/Utils/ERF_TerrainMetrics.cpp index b0bbf2210..cea4c3641 100644 --- a/Source/Utils/ERF_TerrainMetrics.cpp +++ b/Source/Utils/ERF_TerrainMetrics.cpp @@ -2,40 +2,51 @@ #include #include #include +#include #include using namespace amrex; void -init_zlevels (Vector& zlevels_stag, - const Geometry& geom, +init_zlevels (Vector>& zlevels_stag, + Vector const& geom, + Vector const& ref_ratio, const Real grid_stretching_ratio, const Real zsurf, const Real dz0) { - auto dx = geom.CellSizeArray(); - const Box& domain = geom.Domain(); - int nz = domain.length(2)+1; // staggered - zlevels_stag.resize(nz); + int max_level = zlevels_stag.size()-1; - if (grid_stretching_ratio == 0) { - // This is the default for z_levels - for (int k = 0; k < nz; k++) - { - zlevels_stag[k] = k * dx[2]; - } - } else { - // Create stretched grid based on initial dz and stretching ratio - zlevels_stag[0] = zsurf; - Real dz = dz0; - Print() << "Stretched grid levels: " << zsurf; - for (int k = 1; k < nz; k++) - { - zlevels_stag[k] = zlevels_stag[k-1] + dz; - Print() << " " << zlevels_stag[k]; - dz *= grid_stretching_ratio; + for (int lev = 0; lev <= max_level; lev++) + { + auto dx = geom[lev].CellSizeArray(); + const Box& domain = geom[lev].Domain(); + int nz = domain.length(2)+1; // staggered + + zlevels_stag[lev].resize(nz); + + if (grid_stretching_ratio == 0) { + // This is the default for z_levels + for (int k = 0; k < nz; k++) + { + zlevels_stag[lev][k] = k * dx[2]; + } + } else if (lev == 0) { + // Create stretched grid based on initial dz and stretching ratio + zlevels_stag[lev][0] = zsurf; + Real dz = dz0; + Print() << "Stretched grid levels at level : " << lev << " is " << zsurf; + for (int k = 1; k < nz; k++) + { + zlevels_stag[lev][k] = zlevels_stag[lev][k-1] + dz; + Print() << " " << zlevels_stag[lev][k]; + dz *= grid_stretching_ratio; + } + Print() << std::endl; + } else if (lev > 0) { + int rr = ref_ratio[lev-1][2]; + expand_and_interpolate_1d(zlevels_stag[lev], zlevels_stag[lev-1], rr, false); } - Print() << std::endl; } // Try reading in terrain_z_levels, which allows arbitrarily spaced grid @@ -45,6 +56,7 @@ init_zlevels (Vector& zlevels_stag, int n_zlevels = pp.countval("terrain_z_levels"); if (n_zlevels > 0) { + int nz = geom[0].Domain().length(2)+1; // staggered if (n_zlevels != nz) { Print() << "You supplied " << n_zlevels << " staggered terrain_z_levels " << std::endl; Print() << "but n_cell+1 in the z-direction is " << nz << std::endl; @@ -55,21 +67,22 @@ init_zlevels (Vector& zlevels_stag, Print() << "Note: Found terrain_z_levels, ignoring grid_stretching_ratio" << std::endl; } - pp.queryarr("terrain_z_levels", zlevels_stag, 0, nz); + pp.getarr("terrain_z_levels", zlevels_stag[0], 0, nz); // These levels should range from 0 at the surface to the height of the // top of model domain (see the coordinate surface height, zeta, in // Klemp 2011) - AMREX_ALWAYS_ASSERT(zlevels_stag[0] == 0); + AMREX_ALWAYS_ASSERT(zlevels_stag[0][0] == 0); + + for (int lev = 1; lev <= max_level; lev++) { + int rr = ref_ratio[lev-1][2]; + expand_and_interpolate_1d(zlevels_stag[lev], zlevels_stag[lev-1], rr, false); + } } } /** * Computation of the terrain grid from BTF, STF, or Sullivan TF model - * - * NOTE: Multilevel is not yet working for either of these terrain-following coordinates, - * but (we think) the issue is deep in ERF and this code will work once the deeper - * problem is fixed. For now, make sure to run on a single level. -mmsanders */ void @@ -111,11 +124,13 @@ init_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, z_phys_nd.ParallelCopy(z_phys_nd_new,0,0,1,z_phys_nd.nGrowVect(),z_phys_nd.nGrowVect()); } - } else { + } else { // lev > 0 init_which_terrain_grid(lev, geom, z_phys_nd, z_levels_h); } - // Fill ghost layers and corners (including periodic) + // + // Fill ghost layers and corners (including periodic) -- no matter what level + // z_phys_nd.FillBoundary(geom.periodicity()); if (phys_bc_type[Orientation(0,Orientation::low )] == ERF_BC::symmetry || @@ -123,8 +138,8 @@ init_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, phys_bc_type[Orientation(1,Orientation::low )] == ERF_BC::symmetry || phys_bc_type[Orientation(1,Orientation::high)] == ERF_BC::symmetry) { - const auto& dom_lo = lbound(convert(geom.Domain(),IntVect(1,1,1))); - const auto& dom_hi = ubound(convert(geom.Domain(),IntVect(1,1,1))); + const auto& dom_lo = lbound(convert(domain,IntVect(1,1,1))); + const auto& dom_hi = ubound(convert(domain,IntVect(1,1,1))); for (MFIter mfi(z_phys_nd,true); mfi.isValid(); ++mfi) { const Box& bx = mfi.growntilebox(); @@ -204,7 +219,7 @@ init_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, } // init_terrain_grid void -init_which_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, +init_which_terrain_grid (int lev, Geometry const& geom, MultiFab& z_phys_nd, Vector const& z_levels_h) { // User-selected method from inputs file (BTF default) @@ -225,13 +240,13 @@ init_which_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, int domlo_y = domain.smallEnd(1); int domhi_y = domain.bigEnd(1) + 1; int domlo_z = domain.smallEnd(2); int domhi_z = domain.bigEnd(2) + 1; - int imin = domlo_x; if (geom.isPeriodic(0)) imin -= z_phys_nd.nGrowVect()[0]; - int jmin = domlo_y; if (geom.isPeriodic(1)) jmin -= z_phys_nd.nGrowVect()[1]; + int imin = domlo_x; // if (geom.isPeriodic(0)) imin -= z_phys_nd.nGrowVect()[0]; + int jmin = domlo_y; // if (geom.isPeriodic(1)) jmin -= z_phys_nd.nGrowVect()[1]; - int imax = domhi_x; if (geom.isPeriodic(0)) imax += z_phys_nd.nGrowVect()[0]; - int jmax = domhi_y; if (geom.isPeriodic(1)) jmax += z_phys_nd.nGrowVect()[1]; + int imax = domhi_x; // if (geom.isPeriodic(0)) imax += z_phys_nd.nGrowVect()[0]; + int jmax = domhi_y; // if (geom.isPeriodic(1)) jmax += z_phys_nd.nGrowVect()[1]; - int nz = domain.length(2)+1; // staggered + int nz = z_levels_h.size(); Real z_top = z_levels_h[nz-1]; Gpu::DeviceVector z_levels_d; @@ -286,8 +301,22 @@ init_which_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, z_arr(i,j,k) = ( (z_sfc - z_lev_sfc) * z_top + (z_top - z_sfc ) * z ) / (z_top - z_lev_sfc); }); + } // mfi + + z_phys_nd.FillBoundary(geom.periodicity()); + + for ( MFIter mfi(z_phys_nd, TilingIfNotGPU()); mfi.isValid(); ++mfi ) + { + // Note that this box is nodal because it is based on z_phys_nd + const Box& bx = mfi.validbox(); + Box gbx = mfi.growntilebox(ngrowVect); + + int k0 = bx.smallEnd()[2]; + + if (k0 == 0) + { + Array4 const& z_arr = z_phys_nd.array(mfi); - if (k0 == 0) { // Fill lateral boundaries below the bottom surface ParallelFor(makeSlab(gbx,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int) { @@ -370,13 +399,14 @@ init_which_terrain_grid (int lev, const Geometry& geom, MultiFab& z_phys_nd, Real zz_minus = z_lev_h[k-1]; // Hybrid attenuation profile, Klemp2011 Eqn. 9 - Real A{0.}, A_minus{0.}; - if (z_H > 1e-8) { - Real foo = cos((PI/2)*(zz/z_H)); - if(zz < z_H) { A = foo*foo*foo*foo*foo*foo; } // A controls rate of return to atm - Real foo_minus = cos((PI/2)*(zz_minus/z_H)); - if(zz_minus < z_H) { A_minus = foo_minus*foo_minus*foo_minus*foo_minus*foo_minus*foo_minus; } // A controls rate of return to atm - } + Real A; + Real foo = cos((PI/2)*(zz/z_H)); + if(zz < z_H) { A = foo*foo*foo*foo*foo*foo; } // A controls rate of return to atm + else { A = 0; } + Real foo_minus = cos((PI/2)*(zz_minus/z_H)); + Real A_minus; + if(zz_minus < z_H) { A_minus = foo_minus*foo_minus*foo_minus*foo_minus*foo_minus*foo_minus; } // A controls rate of return to atm + else { A_minus = 0; } unsigned maxIter = 50; // M_k in paper unsigned iter = 0; @@ -569,10 +599,10 @@ make_J (const Geometry& geom, Array4 z_nd = z_phys_nd.const_array(mfi); Array4 detJ = detJ_cc.array(mfi); ParallelFor(gbx, [=] AMREX_GPU_DEVICE(int i, int j, int k) noexcept { - detJ(i, j, k) = .25 * dzInv * ( - z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) - -z_nd(i,j,k ) - z_nd(i+1,j,k ) - z_nd(i,j+1,k ) - z_nd(i+1,j+1,k ) ); - }); + detJ(i, j, k) = .25 * dzInv * ( + z_nd(i,j,k+1) + z_nd(i+1,j,k+1) + z_nd(i,j+1,k+1) + z_nd(i+1,j+1,k+1) + -z_nd(i,j,k ) - z_nd(i+1,j,k ) - z_nd(i,j+1,k ) - z_nd(i+1,j+1,k ) ); + }); } detJ_cc.FillBoundary(geom.periodicity()); }