Skip to content

Commit

Permalink
Pr 1814 (#1817)
Browse files Browse the repository at this point in the history 
* make z_levels_stag for every level and use new InterpFromCoarseLevel to fill z_phys_nd and base_state

* fix dt calculation and init from input sounding

* protect against divide by 0

---------

Co-authored-by: Aaron M. Lattanzi <[email protected]>
  • Loading branch information
@asalmgren @AMLattanzi
asalmgren and AMLattanzi committed Sep 18, 2024
1 parent c2d8682 commit 68168c1
Show file tree
Hide file tree
Showing 18 changed files with 233 additions and 180 deletions.
9 changes: 4 additions & 5 deletions Source/ERF.H
View file
Original file line number Diff line number Diff line change
Expand Up @@ -402,8 +402,7 @@ public:
const amrex::Real& dt_advance,
amrex::MultiFab& cons_in,
amrex::MultiFab& U_old, amrex::MultiFab& V_old, amrex::MultiFab& W_old,
amrex::MultiFab& mf_vars_windfarm, const amrex::MultiFab& mf_Nturb,
const amrex::MultiFab& mf_SMark);
amrex::MultiFab& mf_vars_windfarm, const amrex::MultiFab& mf_Nturb);
#endif

#ifdef ERF_USE_EB
Expand Down Expand Up @@ -432,7 +431,7 @@ private:
void update_diffusive_arrays (int lev, const amrex::BoxArray& ba, const amrex::DistributionMapping& dm);

void init_zphys (int lev, amrex::Real time);
void remake_zphys (int lev, amrex::Real time, std::unique_ptr<amrex::MultiFab>& temp_zphys_nd);
void remake_zphys (int lev, std::unique_ptr<amrex::MultiFab>& temp_zphys_nd);
void update_terrain_arrays (int lev);

void Construct_ERFFillPatchers (int lev);
Expand Down Expand Up @@ -744,7 +743,7 @@ private:
amrex::Vector<std::unique_ptr<amrex::MultiFab>> SFS_q2fx3_lev;

// Terrain / grid stretching
amrex::Vector<amrex::Real> zlevels_stag; // nominal height levels
amrex::Vector<amrex::Vector<amrex::Real>> zlevels_stag; // nominal height levels
amrex::Vector<std::unique_ptr<amrex::MultiFab>> z_phys_nd;
amrex::Vector<std::unique_ptr<amrex::MultiFab>> z_phys_cc;

Expand Down Expand Up @@ -1048,7 +1047,7 @@ private:
//
amrex::MultiFab fine_mask;

amrex::Real dz_min;
amrex::Vector<amrex::Real> dz_min;

static AMREX_FORCE_INLINE
int
Expand Down
28 changes: 18 additions & 10 deletions Source/ERF.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,21 +146,24 @@ ERF::ERF_shared ()

// Initialize staggered vertical levels for grid stretching or terrain, and
// to simplify Rayleigh damping layer calculations.
zlevels_stag.resize(max_level+1);
init_zlevels(zlevels_stag,
geom[0],
geom,
refRatio(),
solverChoice.grid_stretching_ratio,
solverChoice.zsurf,
solverChoice.dz0);

if (solverChoice.use_terrain) {
int nz = geom[0].Domain().length(2) + 1; // staggered
if (std::fabs(zlevels_stag[nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
if (std::fabs(zlevels_stag[0][nz-1]-geom[0].ProbHi(2)) > 1.0e-4) {
Print() << "Note: prob_hi[2]=" << geom[0].ProbHi(2)
<< " does not match highest requested z level " << zlevels_stag[nz-1]
<< " does not match highest requested z level " << zlevels_stag[0][nz-1]
<< std::endl;
}
if (std::fabs(zlevels_stag[0]-geom[0].ProbLo(2)) > 1.0e-4) {
if (std::fabs(zlevels_stag[0][0]-geom[0].ProbLo(2)) > 1.0e-4) {
Print() << "Note: prob_lo[2]=" << geom[0].ProbLo(2)
<< " does not match lowest requested level " << zlevels_stag[0]
<< " does not match lowest requested level " << zlevels_stag[0][0]
<< std::endl;
}

Expand Down Expand Up @@ -777,7 +780,7 @@ ERF::InitData ()
h_u_geos[lev], d_u_geos[lev],
h_v_geos[lev], d_v_geos[lev],
geom[lev],
zlevels_stag);
zlevels_stag[0]);
}
}
}
Expand Down Expand Up @@ -871,6 +874,7 @@ ERF::InitData ()
}
}
#endif

// Copy from new into old just in case
for (int lev = 0; lev <= finest_level; ++lev)
{
Expand All @@ -885,10 +889,14 @@ ERF::InitData ()
MultiFab::Copy(lev_old[Vars::zvel],lev_new[Vars::zvel],0,0, 1,lev_new[Vars::zvel].nGrowVect());
}

// Compute the minimum dz in the domain (to be used for setting the timestep)
dz_min = geom[0].CellSize(2);
if ( solverChoice.use_terrain ) {
dz_min *= (*detJ_cc[0]).min(0);
// Compute the minimum dz in the domain at each level (to be used for setting the timestep)
dz_min.resize(max_level+1);
for (int lev = 0; lev <= finest_level; ++lev)
{
dz_min[lev] = geom[lev].CellSize(2);
if ( solverChoice.use_terrain ) {
dz_min[lev] *= (*detJ_cc[lev]).min(0);
}
}

ComputeDt();
Expand Down
4 changes: 2 additions & 2 deletions Source/ERF_Tagging.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -160,15 +160,15 @@ ERF::refinement_criteria_setup ()
khi = domain.smallEnd(2) - 1;

for (int k=domain.smallEnd(2); k<=domain.bigEnd(2)+1; ++k) {
if (zlevels_stag[k] > rbox_lo[2]) {
if (zlevels_stag[lev_for_box][k] > rbox_lo[2]) {
klo = k-1;
break;
}
}
AMREX_ASSERT(klo >= domain.smallEnd(2));

for (int k=klo+1; k<=domain.bigEnd(2)+1; ++k) {
if (zlevels_stag[k] > rbox_hi[2]) {
if (zlevels_stag[lev_for_box][k] > rbox_hi[2]) {
khi = k-1;
break;
}
Expand Down
71 changes: 36 additions & 35 deletions Source/ERF_make_new_arrays.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,26 +441,24 @@ ERF::update_diffusive_arrays (int lev, const BoxArray& ba, const DistributionMap
void
ERF::init_zphys (int lev, Real time)
{
if (solverChoice.use_terrain) {
//
// First interpolate from coarser level if there is one
//
if (solverChoice.use_terrain)
{
if (lev > 0) {
Vector<MultiFab*> fmf = {z_phys_nd[lev].get(), z_phys_nd[lev].get()};
Vector<Real> ftime = {t_old[lev], t_new[lev]};
Vector<MultiFab*> cmf = {z_phys_nd[lev-1].get(), z_phys_nd[lev-1].get()};
Vector<Real> ctime = {t_old[lev-1], t_new[lev-1]};

//
// First we fill z_phys_nd at lev>0 through interpolation
// First interpolate from coarser level if there is one
// NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
// have been pre-filled - this includes ghost cells both inside and outside
// the domain
//
Interpolater* mapper = &node_bilinear_interp;
PhysBCFunctNoOp null_bc;
InterpFromCoarseLevel(*z_phys_nd[lev], time, *z_phys_nd[lev-1],
0, 0, 1,
InterpFromCoarseLevel(*z_phys_nd[lev], z_phys_nd[lev]->nGrowVect(),
IntVect(0,0,0), // do not fill ghost cells outside the domain
*z_phys_nd[lev-1], 0, 0, 1,
geom[lev-1], geom[lev],
null_bc, 0, null_bc, 0, refRatio(lev-1),
mapper, domain_bcs_type, 0);
refRatio(lev-1), &node_bilinear_interp, domain_bcs_type);

// This recomputes the fine values using the bottom terrain at the fine resolution,
// and also fills values of z_phys_nd outside the domain
init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);
}

//
Expand All @@ -478,35 +476,38 @@ ERF::init_zphys (int lev, Real time)
{
z_phys_nd[lev]->setVal(-1.e23);
prob->init_custom_terrain(geom[lev],*z_phys_nd[lev],time);
init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag,phys_bc_type);
init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);

Real zmax = z_phys_nd[0]->max(0,0,false);
Real rel_diff = (zmax - zlevels_stag[zlevels_stag.size()-1]) / zmax;
Real rel_diff = (zmax - zlevels_stag[0][zlevels_stag[0].size()-1]) / zmax;
AMREX_ALWAYS_ASSERT_WITH_MESSAGE(rel_diff < 1.e-8, "Terrain is taller than domain top!");

} // init_type
z_phys_nd[lev]->FillBoundary(geom[lev].periodicity());
} // lev == 0
}
} // terrain
}

void
ERF::remake_zphys (int lev, Real time, std::unique_ptr<MultiFab>& temp_zphys_nd)
ERF::remake_zphys (int lev, std::unique_ptr<MultiFab>& temp_zphys_nd)
{
if (solverChoice.use_terrain && lev > 0) {

Vector<MultiFab*> fmf = {z_phys_nd[lev].get(), z_phys_nd[lev].get()};
Vector<MultiFab*> cmf = {z_phys_nd[lev-1].get(), z_phys_nd[lev-1].get()};
Vector<Real> ftime = {time, time};
Vector<Real> ctime = {time, time};

PhysBCFunctNoOp null_bc;
Interpolater* mapper = &node_bilinear_interp;

FillPatchTwoLevels(*temp_zphys_nd, time,
cmf, ctime, fmf, ftime,
0, 0, 1, geom[lev-1], geom[lev],
null_bc, 0, null_bc, 0, refRatio(lev-1),
mapper, domain_bcs_type, 0);
if (solverChoice.use_terrain && lev > 0)
{
//
// First interpolate from coarser level
// NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
// have been pre-filled - this includes ghost cells both inside and outside
// the domain
//
InterpFromCoarseLevel(*temp_zphys_nd, z_phys_nd[lev]->nGrowVect(),
IntVect(0,0,0), // do not fill ghost cells outside the domain
*z_phys_nd[lev-1], 0, 0, 1,
geom[lev-1], geom[lev],
refRatio(lev-1), &node_bilinear_interp, domain_bcs_type);

// This recomputes the fine values using the bottom terrain at the fine resolution,
// and also fills values of z_phys_nd outside the domain
init_terrain_grid(lev,geom[lev],*z_phys_nd[lev],zlevels_stag[lev],phys_bc_type);

std::swap(temp_zphys_nd, z_phys_nd[lev]);

Expand Down
27 changes: 11 additions & 16 deletions Source/ERF_make_new_level.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -313,21 +313,16 @@ ERF::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba,
// ********************************************************************************************
// Update the base state at this level by interpolation from coarser level
// ********************************************************************************************
// Interp all three components: rho, p, pi
int icomp = 0; int bccomp = 0; int ncomp = 3;

PhysBCFunctNoOp null_bc;
Interpolater* mapper = &cell_cons_interp;

Vector<MultiFab*> fmf = {&base_state[lev ], &base_state[lev ]};
Vector<MultiFab*> cmf = {&base_state[lev-1], &base_state[lev-1]};
Vector<Real> ftime = {time, time};
Vector<Real> ctime = {time, time};
InterpFromCoarseLevel(base_state[lev], time, base_state[lev-1],
icomp, icomp, ncomp,
//
// NOTE: this interpolater assumes that ALL ghost cells of the coarse MultiFab
// have been pre-filled - this includes ghost cells both inside and outside
// the domain
//
InterpFromCoarseLevel(base_state[lev], base_state[lev].nGrowVect(),
IntVect(0,0,0), // do not fill ghost cells outside the domain
base_state[lev-1], 0, 0, 3,
geom[lev-1], geom[lev],
null_bc, 0, null_bc, 0, refRatio(lev-1),
mapper, domain_bcs_type, bccomp);
refRatio(lev-1), &cell_cons_interp, domain_bcs_type);

initHSE(lev);

Expand Down Expand Up @@ -375,7 +370,7 @@ ERF::MakeNewLevelFromCoarse (int lev, Real time, const BoxArray& ba,
void
ERF::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMapping& dm)
{
amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;
// amrex::Print() <<" REMAKING WITH NEW BA AT LEVEL " << lev << " " << ba << std::endl;

AMREX_ALWAYS_ASSERT(lev > 0);
AMREX_ALWAYS_ASSERT(solverChoice.terrain_type != TerrainType::Moving);
Expand Down Expand Up @@ -406,7 +401,7 @@ ERF::RemakeLevel (int lev, Real time, const BoxArray& ba, const DistributionMapp
// ********************************************************************************************
// Build the data structures for terrain-related quantities
// ********************************************************************************************
remake_zphys(lev, time, temp_zphys_nd);
remake_zphys(lev, temp_zphys_nd);
update_terrain_arrays(lev);

//
Expand Down
12 changes: 6 additions & 6 deletions Source/IO/ERF_Write1DProfiles.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,8 +66,8 @@ ERF::write_1D_profiles (Real time)
// Write the quantities at this time
for (int k = 0; k < hu_size; k++) {
Real z;
if (zlevels_stag.size() > 1) {
z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]);
if (zlevels_stag[0].size() > 1) {
z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
} else {
z = (k + 0.5)* dx[2];
}
Expand All @@ -89,8 +89,8 @@ ERF::write_1D_profiles (Real time)
// Write the perturbational quantities at this time
for (int k = 0; k < hu_size; k++) {
Real z;
if (zlevels_stag.size() > 1) {
z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]);
if (zlevels_stag[0].size() > 1) {
z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
} else {
z = (k + 0.5)* dx[2];
}
Expand Down Expand Up @@ -144,8 +144,8 @@ ERF::write_1D_profiles (Real time)
// Write the average stresses
for (int k = 0; k < hu_size; k++) {
Real z;
if (zlevels_stag.size() > 1) {
z = 0.5 * (zlevels_stag[k] + zlevels_stag[k+1]);
if (zlevels_stag[0].size() > 1) {
z = 0.5 * (zlevels_stag[0][k] + zlevels_stag[0][k+1]);
} else {
z = (k + 0.5)* dx[2];
}
Expand Down
12 changes: 6 additions & 6 deletions Source/IO/ERF_Write1DProfiles_stag.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ ERF::write_1D_profiles_stag (Real time)
if (data_log1.good()) {
// Write the quantities at this time
for (int k = 0; k < unstag_size; k++) {
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
<< std::setw(datwidth) << std::setprecision(datprecision) << z << " "
<< h_avg_u[k] << " " << h_avg_v[k] << " " << h_avg_w[k] << " "
Expand All @@ -84,7 +84,7 @@ ERF::write_1D_profiles_stag (Real time)
<< std::endl;
} // loop over z
// Write top face values
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
data_log1 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
<< std::setw(datwidth) << std::setprecision(datprecision) << z << " "
<< 0 << " " << 0 << " " << h_avg_w[unstag_size+1] << " "
Expand Down Expand Up @@ -139,7 +139,7 @@ ERF::write_1D_profiles_stag (Real time)

// For internal values, interpolate scalar quantities to faces
for (int k = 1; k < unstag_size; k++) {
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
Real uface = 0.5*(h_avg_u[k] + h_avg_u[k-1]);
Real vface = 0.5*(h_avg_v[k] + h_avg_v[k-1]);
Real thface = 0.5*(h_avg_th[k] + h_avg_th[k-1]);
Expand Down Expand Up @@ -207,7 +207,7 @@ ERF::write_1D_profiles_stag (Real time)
Real uuface = 1.5*h_avg_uu[k-1] - 0.5*h_avg_uu[k-2];
Real vvface = 1.5*h_avg_vv[k-1] - 0.5*h_avg_vv[k-2];
Real thvface = thface * (1 + 0.61*qvface - qcface - qrface);
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
data_log2 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
<< std::setw(datwidth) << std::setprecision(datprecision) << z << " "
<< 0 << " " // u'u'
Expand Down Expand Up @@ -243,7 +243,7 @@ ERF::write_1D_profiles_stag (Real time)
if (data_log3.good()) {
// Write the average stresses
for (int k = 0; k < unstag_size; k++) {
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[k] : k * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][k] : k * dx[2];
data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
<< std::setw(datwidth) << std::setprecision(datprecision) << z << " "
<< h_avg_tau11[k] << " " << h_avg_tau12[k] << " " << h_avg_tau13[k] << " "
Expand All @@ -255,7 +255,7 @@ ERF::write_1D_profiles_stag (Real time)
} // loop over z
// Write top face values
Real NANval = 0.0;
Real z = (zlevels_stag.size() > 1) ? zlevels_stag[unstag_size] : unstag_size * dx[2];
Real z = (zlevels_stag[0].size() > 1) ? zlevels_stag[0][unstag_size] : unstag_size * dx[2];
data_log3 << std::setw(datwidth) << std::setprecision(timeprecision) << time << " "
<< std::setw(datwidth) << std::setprecision(datprecision) << z << " "
<< NANval << " " << NANval << " " << h_avg_tau13[unstag_size] << " "
Expand Down
10 changes: 5 additions & 5 deletions Source/Initialization/ERF_init1d.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ void
ERF::initRayleigh ()
{
const int khi = geom[0].Domain().bigEnd(2);
solverChoice.rayleigh_ztop = (solverChoice.use_terrain) ? zlevels_stag[khi+1] : geom[0].ProbHi(2);
solverChoice.rayleigh_ztop = (solverChoice.use_terrain) ? zlevels_stag[0][khi+1] : geom[0].ProbHi(2);

h_rayleigh_ptrs.resize(max_level+1);
d_rayleigh_ptrs.resize(max_level+1);
Expand Down Expand Up @@ -67,7 +67,7 @@ ERF::setRayleighRefFromSounding (bool restarting)
if (restarting) {
input_sounding_data.resize_arrays();
for (int n = 0; n < input_sounding_data.n_sounding_files; n++) {
input_sounding_data.read_from_file(geom[0], zlevels_stag, n);
input_sounding_data.read_from_file(geom[0], zlevels_stag[0], n);
}
}

Expand All @@ -85,8 +85,8 @@ ERF::setRayleighRefFromSounding (bool restarting)
const int Nz = khi - klo + 1;

Vector<Real> zcc(Nz);
Vector<Real> zlevels_sub(zlevels_stag.begin()+klo/refine_fac,
zlevels_stag.begin()+khi/refine_fac+2);
Vector<Real> zlevels_sub(zlevels_stag[0].begin()+klo/refine_fac,
zlevels_stag[0].begin()+khi/refine_fac+2);
expand_and_interpolate_1d(zcc, zlevels_sub, refine_fac, true);
#if 0
amrex::AllPrint() << "lev="<<lev<<" : (refine_fac="<<refine_fac<<",klo="<<klo<<",khi="<<khi<<") ";
Expand Down Expand Up @@ -159,7 +159,7 @@ ERF::setSpongeRefFromSounding (bool restarting)
// so we need to read it here
// TODO: should we store this information in the checkpoint file instead?
if (restarting) {
input_sponge_data.read_from_file(geom[0], zlevels_stag);
input_sponge_data.read_from_file(geom[0], zlevels_stag[0]);
}

const Real* z_inp_sponge = input_sponge_data.z_inp_sponge.dataPtr();
Expand Down
Loading

0 comments on commit 68168c1

Please sign in to comment.