Skip to content

Commit

Permalink
add mask_rho, mask_u, mask_v
Browse files Browse the repository at this point in the history
  • Loading branch information
@asalmgren
asalmgren committed Jan 13, 2024
1 parent 069bc35 commit 07cfc2a
Show file tree
Hide file tree
Showing 7 changed files with 137 additions and 107 deletions.
24 changes: 24 additions & 0 deletions Source/IO/Checkpoint.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,6 +148,18 @@ REMORA::WriteCheckpointFile () const
MultiFab::Copy(mf_vbar,*(vec_vbar[lev]),0,0,3,NGROW);
VisMF::Write(mf_vbar, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "YBar"));

MultiFab mf_mskr(ba2d,dmap[lev],1,NGROW);
MultiFab::Copy(mf_mskr,*(vec_mskr[lev]),0,0,3,NGROW);
VisMF::Write(mf_mskr, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Mskr"));

MultiFab mf_msku(ba2d,dmap[lev],1,NGROW);
MultiFab::Copy(mf_msku,*(vec_msku[lev]),0,0,3,NGROW);
VisMF::Write(mf_msku, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Msku"));

MultiFab mf_mskv(ba2d,dmap[lev],1,NGROW);
MultiFab::Copy(mf_mskv,*(vec_mskv[lev]),0,0,3,NGROW);
VisMF::Write(mf_mskv, amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "Mskv"));

VisMF::Write(*(vec_rufrc[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rufrc"));
VisMF::Write(*(vec_rvfrc[lev]), amrex::MultiFabFileFullPrefix(lev, checkpointname, "Level_", "rvfrc"));

Expand Down Expand Up @@ -359,6 +371,18 @@ REMORA::ReadCheckpointFile ()
VisMF::Read(mf_vbar, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "YBar"));
MultiFab::Copy(*(vec_vbar[lev]),mf_vbar,0,0,3,(vec_vbar[lev])->nGrowVect());

MultiFab mf_mskr(ba2d,dmap[lev],3,NGROW);
VisMF::Read(mf_mskr, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Mskr"));
MultiFab::Copy(*(vec_mskr[lev]),mf_mskr,0,0,1,(vec_mskr[lev])->nGrowVect());

MultiFab mf_msku(ba2d,dmap[lev],3,NGROW);
VisMF::Read(mf_msku, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Msku"));
MultiFab::Copy(*(vec_msku[lev]),mf_msku,0,0,1,(vec_msku[lev])->nGrowVect());

MultiFab mf_mskv(ba2d,dmap[lev],3,NGROW);
VisMF::Read(mf_mskv, amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "Mskv"));
MultiFab::Copy(*(vec_mskv[lev]),mf_mskv,0,0,1,(vec_mskv[lev])->nGrowVect());

VisMF::Read(*(vec_rufrc[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rufrc"));
VisMF::Read(*(vec_rvfrc[lev]), amrex::MultiFabFileFullPrefix(lev, restart_chkfile, "Level_", "rvfrc"));

Expand Down
35 changes: 0 additions & 35 deletions Source/IO/NCPlotFile.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,39 +96,11 @@ void
REMORA::WriteNCPlotFile_which(int which_step, int lev, int which_subdomain,
bool write_header, ncutils::NCFile& ncf) const
{
int iproc = amrex::ParallelContext::MyProcAll();
int nproc = amrex::ParallelDescriptor::NProcs();

// Number of cells in this "domain" at this level
std::vector<int> n_cells;

int nblocks = grids[lev].size();

// Number of points in each block at this level
std::vector<int> offset_s;
std::vector<int> offset_u;
std::vector<int> offset_v;
offset_s.reserve(nproc);
offset_u.reserve(nproc);
offset_v.reserve(nproc);

for (auto n = 0; n < nproc; n++) {
offset_s[n] = 0;
offset_u[n] = 0;
offset_v[n] = 0;
}

for (auto ib=0; ib<nblocks; ib++) {
Box tmp_bx_s(grids[lev][ib]); tmp_bx_s.grow(IntVect(1,1,0));
offset_s[dmap[lev][ib]] += tmp_bx_s.numPts();

Box tmp_bx_u(grids[lev][ib]); tmp_bx_u.surroundingNodes(0); tmp_bx_u.grow(IntVect(0,1,0));
offset_u[dmap[lev][ib]] += tmp_bx_u.numPts();

Box tmp_bx_v(grids[lev][ib]); tmp_bx_v.surroundingNodes(1); tmp_bx_v.grow(IntVect(1,0,0));
offset_v[dmap[lev][ib]] += tmp_bx_v.numPts();
}

// We only do single-level writes when using NetCDF format
int flev = lev;

Expand Down Expand Up @@ -337,7 +309,6 @@ REMORA::WriteNCPlotFile_which(int which_step, int lev, int which_subdomain,
} // end if write_header

size_t nbox_per_proc = 0;
long unsigned numpts = 0;

long unsigned local_start_x = 0;
long unsigned local_start_y = 0;
Expand All @@ -353,8 +324,6 @@ REMORA::WriteNCPlotFile_which(int which_step, int lev, int which_subdomain,
auto bx = mfi.validbox();
if (subdomain.contains(bx))
{
// for(auto ip = 1; ip <= iproc; ++ip) {diff += offset_s[ip-1];}

//
// We only include one grow cell at subdomain boundaries, not internal grid boundaries
//
Expand Down Expand Up @@ -428,8 +397,6 @@ REMORA::WriteNCPlotFile_which(int which_step, int lev, int which_subdomain,

if (subdomain.contains(bx))
{
// for (auto ip = 0; ip < iproc; ++ip) { diff_u += offset_u[ip]; }

//
// We only include one grow cell at subdomain boundaries, not internal grid boundaries
//
Expand Down Expand Up @@ -479,8 +446,6 @@ REMORA::WriteNCPlotFile_which(int which_step, int lev, int which_subdomain,

if (subdomain.contains(bx))
{
// for (auto ip = 0; ip <= iproc-1; ++ip) { diff_v += offset_v[ip]; }

//
// We only include one grow cell at subdomain boundaries, not internal grid boundaries
//
Expand Down
21 changes: 13 additions & 8 deletions Source/IO/ReadFromInitNetcdf.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,21 +12,26 @@ read_data_from_netcdf (int /*lev*/,
FArrayBox& NC_temp_fab, FArrayBox& NC_salt_fab,
FArrayBox& NC_xvel_fab, FArrayBox& NC_yvel_fab,
FArrayBox& NC_ubar_fab, FArrayBox& NC_vbar_fab,
FArrayBox& NC_zeta_fab)
FArrayBox& NC_zeta_fab,
FArrayBox& NC_mskr_fab, FArrayBox& NC_msku_fab,
FArrayBox& NC_mskv_fab)
{
amrex::Print() << "Loading initial solution data from NetCDF file " << fname << std::endl;

Vector<FArrayBox*> NC_fabs;
Vector<std::string> NC_names;
Vector<enum NC_Data_Dims_Type> NC_dim_types;

NC_fabs.push_back(&NC_temp_fab); NC_names.push_back("temp"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 0
NC_fabs.push_back(&NC_salt_fab); NC_names.push_back("salt"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 1
NC_fabs.push_back(&NC_xvel_fab); NC_names.push_back("u"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 2
NC_fabs.push_back(&NC_yvel_fab); NC_names.push_back("v"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 3
NC_fabs.push_back(&NC_ubar_fab), NC_names.push_back("ubar"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 4
NC_fabs.push_back(&NC_vbar_fab); NC_names.push_back("vbar"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 5
NC_fabs.push_back(&NC_zeta_fab); NC_names.push_back("zeta"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 6
NC_fabs.push_back(&NC_temp_fab); NC_names.push_back("temp"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 0
NC_fabs.push_back(&NC_salt_fab); NC_names.push_back("salt"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 1
NC_fabs.push_back(&NC_xvel_fab); NC_names.push_back("u"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 2
NC_fabs.push_back(&NC_yvel_fab); NC_names.push_back("v"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_BT_SN_WE); // 3
NC_fabs.push_back(&NC_ubar_fab), NC_names.push_back("ubar"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 4
NC_fabs.push_back(&NC_vbar_fab); NC_names.push_back("vbar"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 5
NC_fabs.push_back(&NC_zeta_fab); NC_names.push_back("zeta"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 6
NC_fabs.push_back(&NC_mskr_fab); NC_names.push_back("mask_rho"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 7
NC_fabs.push_back(&NC_msku_fab); NC_names.push_back("mask_u"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 8
NC_fabs.push_back(&NC_mskv_fab); NC_names.push_back("mask_v"); NC_dim_types.push_back(NC_Data_Dims_Type::Time_SN_WE); // 9

// Read the netcdf file and fill these FABs
BuildFABsFromNetCDFFile<FArrayBox,Real>(domain, fname, NC_names, NC_dim_types, NC_fabs);
Expand Down
82 changes: 48 additions & 34 deletions Source/Initialization/REMORA_init.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ REMORA::set_2darrays (int lev)
const Box& bx = mfi.growntilebox();
const auto & geomdata = Geom(lev).data();
Gpu::synchronize();
amrex::ParallelFor(amrex::makeSlab(bx,2,0),
ParallelFor(amrex::makeSlab(bx,2,0),
[=] AMREX_GPU_DEVICE (int i, int j, int )
{
const auto prob_lo = geomdata.ProbLo();
Expand All @@ -110,56 +110,70 @@ REMORA::set_2darrays (int lev)
MultiFab* V_old = yvel_new[lev];
std::unique_ptr<MultiFab>& mf_ubar = vec_ubar[lev];
std::unique_ptr<MultiFab>& mf_vbar = vec_vbar[lev];
std::unique_ptr<MultiFab>& mf_mskr = vec_mskr[lev];
std::unique_ptr<MultiFab>& mf_msku = vec_msku[lev];
std::unique_ptr<MultiFab>& mf_mskv = vec_mskv[lev];
std::unique_ptr<MultiFab>& mf_Hz = vec_Hz[lev];
int nstp = 0;
int kstp = 0;
int knew = 0;

for ( MFIter mfi(*cons_new[lev], TilingIfNotGPU()); mfi.isValid(); ++mfi )
{
Array4<Real> const& ubar = (mf_ubar)->array(mfi);
Array4<Real> const& vbar = (mf_vbar)->array(mfi);

Array4<Real> const& mskr = (mf_mskr)->array(mfi);
Array4<Real> const& msku = (mf_msku)->array(mfi);
Array4<Real> const& mskv = (mf_mskv)->array(mfi);

Array4<const Real> const& Hz = mf_Hz->const_array(mfi);
Array4<const Real> const& u = U_old->const_array(mfi);
Array4<const Real> const& v = V_old->const_array(mfi);

Box bx2 = mfi.tilebox() ; bx2.grow(IntVect(NGROW ,NGROW ,0)); // cell-centered, grown by 2
Box ubx2 = mfi.nodaltilebox(0); ubx2.grow(IntVect(NGROW ,NGROW ,0)); // x-face-centered, grown by 2
Box vbx2 = mfi.nodaltilebox(1); vbx2.grow(IntVect(NGROW ,NGROW ,0)); // y-face-centered, grown by 2

amrex::ParallelFor(makeSlab(ubx2,2,0),
[=] AMREX_GPU_DEVICE (int i, int j, int )
{
Real CF = 0.;
Real sum_of_hz = 0.;

for (int k=0; k<=N; k++) {
Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i-1,j,k));
sum_of_hz += avg_hz;
CF += avg_hz*u(i,j,k,nstp);
}
ubar(i,j,0,kstp) = CF / sum_of_hz;
ubar(i,j,0,knew) = CF / sum_of_hz;
});

amrex::ParallelFor(makeSlab(vbx2,2,0),
[=] AMREX_GPU_DEVICE (int i, int j, int )
{
Real CF = 0.;
Real sum_of_hz = 0.;

for(int k=0; k<=N; k++) {
Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i,j-1,k));
sum_of_hz += avg_hz;
CF += avg_hz*v(i,j,k,nstp);
}
vbar(i,j,0,kstp) = CF / sum_of_hz;
vbar(i,j,0,knew) = CF / sum_of_hz;
});
ParallelFor(makeSlab(bx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
{
mskr(i,j,0,0) = 1.0;
});

ParallelFor(makeSlab(ubx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
{
Real CF = 0.;
Real sum_of_hz = 0.;

for (int k=0; k<=N; k++) {
Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i-1,j,k));
sum_of_hz += avg_hz;
CF += avg_hz*u(i,j,k,nstp);
}
ubar(i,j,0,0) = CF / sum_of_hz;

msku(i,j,0,0) = 1.0;
});

ParallelFor(makeSlab(vbx2,2,0), [=] AMREX_GPU_DEVICE (int i, int j, int )
{
Real CF = 0.;
Real sum_of_hz = 0.;

for(int k=0; k<=N; k++) {
Real avg_hz = 0.5*(Hz(i,j,k)+Hz(i,j-1,k));
sum_of_hz += avg_hz;
CF += avg_hz*v(i,j,k,nstp);
}
vbar(i,j,0,0) = CF / sum_of_hz;

mskv(i,j,0,0) = 1.0;
});
}

// DEBUGGING NOTE -- DoublyPeriodic fails if these are commented out
const Real time = 0.0;
FillPatch(lev,time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
FillPatch(lev,time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);
FillPatch(lev, time, *vec_ubar[lev], GetVecOfPtrs(vec_ubar), BdyVars::ubar);
FillPatch(lev, time, *vec_vbar[lev], GetVecOfPtrs(vec_vbar), BdyVars::vbar);

FillPatch(lev, time, *vec_msku[lev], GetVecOfPtrs(vec_ubar));
FillPatch(lev, time, *vec_mskv[lev], GetVecOfPtrs(vec_vbar));
}
Loading

0 comments on commit 07cfc2a

Please sign in to comment.