Skip to content

Commit

Permalink
Merge pull request #193 from lanl/jmm/selected-sparse-packs-1
Browse files Browse the repository at this point in the history 
Selected sparse packs 1
  • Loading branch information
@Yurlungur
Yurlungur authored Dec 14, 2023
2 parents 44f0637 + 7574c38 commit 7b56701
Show file tree
Hide file tree
Showing 9 changed files with 130 additions and 121 deletions.
2 changes: 1 addition & 1 deletion external/parthenon
Submodule parthenon updated from a540d3 to 007203
8 changes: 5 additions & 3 deletions inputs/linear_modes.pin
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@ variables = p.density, &
p.energy, &
p.bfield, &
pressure, &
g.n.coord, &
g.c.coord, &
cs

file_type = hdf5 # Tabular data dump
Expand Down Expand Up @@ -63,9 +65,9 @@ num_threads = 1 # maximum number of OMP threads
<phoebus/mesh>
#bc_vars = primitive

# <parthenon/meshblock>
# nx1 = 128
# nx2 = 128
<parthenon/meshblock>
nx1 = 64
nx2 = 64
# nx3 = 1

<parthenon/refinement0>
Expand Down
2 changes: 1 addition & 1 deletion scripts/python/linmode.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -282,7 +282,7 @@ def get_mode(x, y, t, var):
dump = phdf.phdf(dumps[-1])
tf_soln = mode["tf"]
t = dump.Time
if (np.fabs(t - tf_soln) / tf_soln) > 0.05:
if not args.use_initial and (np.fabs(t - tf_soln) / tf_soln) > 0.05:
print("Mismatch in expected solution times!")
print(" Code: ", t)
print(" Soln: ", tf_soln)
Expand Down
5 changes: 4 additions & 1 deletion scripts/python/movie1d.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,10 @@ def addPath():

def read(filename, nGhost=0):
""" Read the parthenon hdf file """
from phdf import phdf
try:
from parthenon_tools.phdf import phdf
except ModuleNotFoundError:
from phdf import phdf

f = phdf(filename)
return f
Expand Down
160 changes: 74 additions & 86 deletions src/fluid/fluid.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
#include "tmunu.hpp"

#include <globals.hpp>
#include <interface/sparse_pack.hpp>
#include <kokkos_abstraction.hpp>
#include <parthenon/package.hpp>
#include <utils/error_checking.hpp>
Expand Down Expand Up @@ -350,43 +351,31 @@ TaskStatus PrimitiveToConservedRegion(MeshBlockData<Real> *rc, const IndexRange
namespace p = fluid_prim;
namespace c = fluid_cons;
namespace impl = internal_variables;
auto *pmb = rc->GetParentPointer();
using parthenon::MakePackDescriptor;

Mesh *pmesh = rc->GetMeshPointer();
auto &resolved_pkgs = pmesh->resolved_packages;
const int ndim = pmesh->ndim;

static auto desc =
MakePackDescriptor<p::density, c::density, p::velocity, c::momentum, p::energy,
c::energy, p::bfield, c::bfield, p::ye, c::ye, p::pressure,
p::gamma1, impl::cell_signal_speed>(resolved_pkgs.get());
auto v = desc.GetPack(rc);

const std::vector<std::string> vars(
{p::density::name(), c::density::name(), p::velocity::name(), c::momentum::name(),
p::energy::name(), c::energy::name(), p::bfield::name(), c::bfield::name(),
p::ye::name(), c::ye::name(), p::pressure::name(), p::gamma1::name(),
impl::cell_signal_speed::name()});

PackIndexMap imap;
auto v = rc->PackVariables(vars, imap);

const int prho = imap[p::density::name()].first;
const int crho = imap[c::density::name()].first;
const int pvel_lo = imap[p::velocity::name()].first;
const int pvel_hi = imap[p::velocity::name()].second;
const int cmom_lo = imap[c::momentum::name()].first;
const int cmom_hi = imap[c::momentum::name()].second;
const int peng = imap[p::energy::name()].first;
const int ceng = imap[c::energy::name()].first;
const int prs = imap[p::pressure::name()].first;
const int gm1 = imap[p::gamma1::name()].first;
const int pb_lo = imap[p::bfield::name()].first;
const int pb_hi = imap[p::bfield::name()].second;
const int cb_lo = imap[c::bfield::name()].first;
const int cb_hi = imap[c::bfield::name()].second;
const int pye = imap[p::ye::name()].second; // -1 if not present
const int cye = imap[c::ye::name()].second;
const int sig_lo = imap[impl::cell_signal_speed::name()].first;
const int sig_hi = imap[impl::cell_signal_speed::name()].second;
// We need these to check whether or not these variables are present
// in the pack. They are -1 if not present.
const int contains_b = v.ContainsHost(0, p::bfield());
const int contains_ye = v.ContainsHost(0, p::ye());

auto geom = Geometry::GetCoordinateSystem(rc);
auto coords = pmb->coords;
const int nblocks = v.GetNBlocks();

parthenon::par_for(
DEFAULT_LOOP_PATTERN, "PrimToCons", DevExecSpace(), 0, v.GetDim(5) - 1, kb.s, kb.e,
DEFAULT_LOOP_PATTERN, "PrimToCons", DevExecSpace(), 0, nblocks - 1, kb.s, kb.e,
jb.s, jb.e, ib.s, ib.e,
KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) {
auto &coords = v.GetCoordinates(b);
Real gcov4[4][4];
geom.SpacetimeMetric(CellLocation::Cent, k, j, i, gcov4);
Real gcon[3][3];
Expand All @@ -397,43 +386,43 @@ TaskStatus PrimitiveToConservedRegion(MeshBlockData<Real> *rc, const IndexRange
geom.ContravariantShift(CellLocation::Cent, k, j, i, shift);

Real S[3];
const Real vel[] = {v(b, pvel_lo, k, j, i), v(b, pvel_lo + 1, k, j, i),
v(b, pvel_hi, k, j, i)};
const Real vel[] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i),
v(b, p::velocity(2), k, j, i)};
Real bcons[3];
Real bp[3] = {0.0, 0.0, 0.0};
if (pb_hi > 0) {
bp[0] = v(b, pb_lo, k, j, i);
bp[1] = v(b, pb_lo + 1, k, j, i);
bp[2] = v(b, pb_hi, k, j, i);
if (contains_b) {
for (int d = 0; d < 3; ++d) {
bp[d] = v(b, p::bfield(d), k, j, i);
}
}
Real ye_cons;
Real ye_prim = 0.0;
if (pye > 0) {
ye_prim = v(b, pye, k, j, i);
if (contains_ye) {
ye_prim = v(b, p::ye(), k, j, i);
}

Real sig[3];
prim2con::p2c(v(b, prho, k, j, i), vel, bp, v(b, peng, k, j, i), ye_prim,
v(b, prs, k, j, i), v(b, gm1, k, j, i), gcov4, gcon, shift, lapse,
gdet, v(b, crho, k, j, i), S, bcons, v(b, ceng, k, j, i), ye_cons,
sig);

v(b, cmom_lo, k, j, i) = S[0];
v(b, cmom_lo + 1, k, j, i) = S[1];
v(b, cmom_hi, k, j, i) = S[2];

if (pb_hi > 0) {
v(b, cb_lo, k, j, i) = bcons[0];
v(b, cb_lo + 1, k, j, i) = bcons[1];
v(b, cb_hi, k, j, i) = bcons[2];
prim2con::p2c(v(b, p::density(), k, j, i), vel, bp, v(b, p::energy(), k, j, i),
ye_prim, v(b, p::pressure(), k, j, i), v(b, p::gamma1(), k, j, i),
gcov4, gcon, shift, lapse, gdet, v(b, c::density(), k, j, i), S,
bcons, v(b, c::energy(), k, j, i), ye_cons, sig);

for (int d = 0; d < 3; ++d) {
v(b, c::momentum(d), k, j, i) = S[d];
}

if (pye > 0) {
v(b, cye, k, j, i) = ye_cons;
if (contains_b) {
for (int d = 0; d < 3; ++d) {
v(b, c::bfield(d), k, j, i) = bcons[d];
}
}

for (int m = sig_lo; m <= sig_hi; m++) {
v(b, m, k, j, i) = sig[m - sig_lo];
if (contains_ye) {
v(b, c::ye(), k, j, i) = ye_cons;
}

for (int d = 0; d < ndim; d++) {
v(b, impl::cell_signal_speed(d), k, j, i) = sig[d];
}
});

Expand Down Expand Up @@ -544,10 +533,13 @@ TaskStatus ConservedToPrimitiveClassic(T *rc, const IndexRange &ib, const IndexR
return TaskStatus::complete;
}

TaskStatus CalculateFluidSourceTerms(MeshBlockData<Real> *rc,
MeshBlockData<Real> *rc_src) {
TaskStatus CalculateFluidSourceTerms(MeshData<Real> *rc, MeshData<Real> *rc_src) {
constexpr int ND = Geometry::NDFULL;
constexpr int NS = Geometry::NDSPACE;
using phoebus::MakePackDescriptor;
namespace c = fluid_cons;
namespace diag = diagnostic_variables;

Mesh *pmesh = rc->GetMeshPointer();
auto &fluid = pmesh->packages.Get("fluid");
if (!fluid->Param<bool>("active") || fluid->Param<bool>("zero_sources"))
Expand All @@ -557,48 +549,46 @@ TaskStatus CalculateFluidSourceTerms(MeshBlockData<Real> *rc,
IndexRange jb = rc->GetBoundsJ(IndexDomain::interior);
IndexRange kb = rc->GetBoundsK(IndexDomain::interior);

std::vector<std::string> vars(
{fluid_cons::momentum::name(), fluid_cons::energy::name()});
static auto desc = MakePackDescriptor<c::momentum, c::energy
#if SET_FLUX_SRC_DIAGS
vars.push_back(diagnostic_variables::src_terms::name());
,
diag::src_terms
#endif
PackIndexMap imap;
auto src = rc_src->PackVariables(vars, imap);
const int cmom_lo = imap[fluid_cons::momentum::name()].first;
const int cmom_hi = imap[fluid_cons::momentum::name()].second;
const int ceng = imap[fluid_cons::energy::name()].first;
const int idiag = imap[diagnostic_variables::src_terms::name()].first;

>(rc);
auto src = desc.GetPack(rc_src);
auto tmunu = BuildStressEnergyTensor(rc);
auto geom = Geometry::GetCoordinateSystem(rc);
const int nblocks = src.GetNBlocks();

parthenon::par_for(
DEFAULT_LOOP_PATTERN, "TmunuSourceTerms", DevExecSpace(), kb.s, kb.e, jb.s, jb.e,
ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) {
DEFAULT_LOOP_PATTERN, "TmunuSourceTerms", DevExecSpace(), 0, nblocks - 1, kb.s,
kb.e, jb.s, jb.e, ib.s, ib.e,
KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) {
Real Tmunu[ND][ND], dg[ND][ND][ND], gam[ND][ND][ND];
tmunu(Tmunu, k, j, i);
Real gdet = geom.DetG(CellLocation::Cent, k, j, i);
tmunu(Tmunu, b, k, j, i);
Real gdet = geom.DetG(CellLocation::Cent, b, k, j, i);

geom.MetricDerivative(CellLocation::Cent, k, j, i, dg);
geom.MetricDerivative(CellLocation::Cent, b, k, j, i, dg);
Geometry::Utils::SetConnectionCoeffFromMetricDerivs(dg, gam);

// momentum source terms
SPACELOOP(l) {
Real src_mom = 0.0;
SPACETIMELOOP2(m, n) {
src_mom += Tmunu[m][n] * (dg[n][l + 1][m] - gam[l + 1][n][m]);
}
src(cmom_lo + l, k, j, i) = gdet * src_mom;
src(b, c::momentum(l), k, j, i) = gdet * src_mom;
}

// energy source term
{
Real TGam = 0.0;
#if USE_VALENCIA
// TODO(jcd): maybe use the lapse and shift here instead of gcon
const Real alpha = geom.Lapse(CellLocation::Cent, k, j, i);
const Real alpha = geom.Lapse(CellLocation::Cent, b, k, j, i);
const Real inv_alpha2 = robust::ratio(1, alpha * alpha);
Real shift[NS];
geom.ContravariantShift(CellLocation::Cent, k, j, i, shift);
geom.ContravariantShift(CellLocation::Cent, b, k, j, i, shift);
Real gcon0[4] = {-inv_alpha2, inv_alpha2 * shift[0], inv_alpha2 * shift[1],
inv_alpha2 * shift[2]};
for (int m = 0; m < ND; m++) {
Expand All @@ -613,25 +603,23 @@ TaskStatus CalculateFluidSourceTerms(MeshBlockData<Real> *rc,
Real Ta = 0.0;
Real da[ND];
// Real *da = &gam[1][0][0];
geom.GradLnAlpha(CellLocation::Cent, k, j, i, da);
geom.GradLnAlpha(CellLocation::Cent, b, k, j, i, da);
for (int m = 0; m < ND; m++) {
Ta += Tmunu[m][0] * da[m];
}
src(ceng, k, j, i) = gdet * alpha * (Ta - TGam);
src(b, c::energy(), k, j, i) = gdet * alpha * (Ta - TGam);
#else
SPACETIMELOOP2(mu, nu) {
TGam += Tmunu[mu][nu] * gam[nu][0][mu];
}
src(ceng, k, j, i) = gdet * TGam;
SPACETIMELOOP2(mu, nu) { TGam += Tmunu[mu][nu] * gam[nu][0][mu]; }
src(b, c::energy(), k, j, i) = gdet * TGam;
#endif // USE_VALENCIA
}

#if SET_FLUX_SRC_DIAGS
src(idiag, k, j, i) = 0.0;
src(idiag + 1, k, j, i) = src(cmom_lo, k, j, i);
src(idiag + 2, k, j, i) = src(cmom_lo + 1, k, j, i);
src(idiag + 3, k, j, i) = src(cmom_lo + 2, k, j, i);
src(idiag + 4, k, j, i) = src(ceng, k, j, i);
src(diag::src_terms(0), b, k, j, i) = 0.0;
src(diag::src_terms(1), b, k, j, i) = src(c::momentum(0), k, j, i);
src(diag::src_terms(2), b, k, j, i) = src(c::momentum(1), k, j, i);
src(diag::src_terms(3), b, k, j, i) = src(c::momentum(2), k, j, i);
src(diag::src_terms(4), b, k, j, i) = src(c::energy(), k, j, i);
#endif
});

Expand Down
3 changes: 1 addition & 2 deletions src/fluid/fluid.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,8 +43,7 @@ TaskStatus ConservedToPrimitiveClassic(T *rc, const IndexRange &ib, const IndexR
template <typename T>
TaskStatus ConservedToPrimitiveVanDerHolst(T *rc, const IndexRange &ib,
const IndexRange &jb, const IndexRange &kb);
TaskStatus CalculateFluidSourceTerms(MeshBlockData<Real> *rc,
MeshBlockData<Real> *rc_src);
TaskStatus CalculateFluidSourceTerms(MeshData<Real> *rc, MeshData<Real> *rc_src);
TaskStatus CalculateFluxes(MeshBlockData<Real> *rc);
TaskStatus FluxCT(MeshBlockData<Real> *rc);
TaskStatus CalculateDivB(MeshBlockData<Real> *rc);
Expand Down
40 changes: 22 additions & 18 deletions src/fluid/tmunu.hpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,19 +19,20 @@
#include <utility>
#include <vector>

#include <interface/sparse_pack.hpp>
#include <parthenon/package.hpp>

#include "geometry/geometry.hpp"
#include "phoebus_utils/cell_locations.hpp"
#include "phoebus_utils/relativity_utils.hpp"
#include "phoebus_utils/robust.hpp"
#include "phoebus_utils/variables.hpp"

namespace fluid {
using namespace parthenon::package::prelude;

const std::vector<std::string> TMUNU_VARS = {
fluid_prim::density::name(), fluid_prim::velocity::name(), fluid_prim::energy::name(),
fluid_prim::pressure::name(), fluid_prim::bfield::name()};
// Indices are upstairs
template <typename CoordinateSystem, typename Pack>
template <typename CoordinateSystem>
class StressEnergyTensorCon {
public:
// TODO(JMM): Should these be moved out of Geometry?
Expand All @@ -42,15 +43,18 @@ class StressEnergyTensorCon {

template <typename Data>
StressEnergyTensorCon(Data *rc) {
PackIndexMap imap;
pack_ = rc->PackVariables(TMUNU_VARS, imap);
namespace p = fluid_prim;
using phoebus::MakePackDescriptor;
static auto desc =
MakePackDescriptor<p::density, p::velocity, p::energy, p::pressure, p::bfield>(
rc);
pack_ = desc.GetPack(rc);
system_ = Geometry::GetCoordinateSystem(rc);

ir_ = imap[fluid_prim::density::name()].first;
iv_ = imap[fluid_prim::velocity::name()].first;
iu_ = imap[fluid_prim::energy::name()].first;
ip_ = imap[fluid_prim::pressure::name()].first;
ib_ = imap[fluid_prim::bfield::name()].first;
ir_ = pack_.GetLowerBoundHost(0, p::density());
iv_ = pack_.GetLowerBoundHost(0, p::velocity());
iu_ = pack_.GetLowerBoundHost(0, p::energy());
ip_ = pack_.GetLowerBoundHost(0, p::pressure());
ib_ = pack_.GetLowerBoundHost(0, p::bfield());
}

// TODO(JMM): Assumes cell centers. If that needs to change, this
Expand Down Expand Up @@ -80,7 +84,7 @@ class StressEnergyTensorCon {
}
KOKKOS_FORCEINLINE_FUNCTION
Real GetVar_(int v, const int k, const int j, const int i) const {
return pack_(v, k, j, i);
return pack_(0, v, k, j, i);
}

template <typename... Args>
Expand Down Expand Up @@ -125,15 +129,15 @@ class StressEnergyTensorCon {
bsq = Bsq * iW * iW + Bdotv * Bdotv;
}

Pack pack_;
parthenon::SparsePack<fluid_prim::density, fluid_prim::velocity, fluid_prim::energy,
fluid_prim::pressure, fluid_prim::bfield>
pack_;
CoordinateSystem system_;
int ir_, iv_, iu_, ip_, ib_;
};

using TmunuMesh =
StressEnergyTensorCon<Geometry::CoordSysMesh, MeshBlockPack<VariablePack<Real>>>;
using TmunuMeshBlock =
StressEnergyTensorCon<Geometry::CoordSysMeshBlock, VariablePack<Real>>;
using TmunuMesh = StressEnergyTensorCon<Geometry::CoordSysMesh>;
using TmunuMeshBlock = StressEnergyTensorCon<Geometry::CoordSysMeshBlock>;

TmunuMesh BuildStressEnergyTensor(MeshData<Real> *rc);
TmunuMeshBlock BuildStressEnergyTensor(MeshBlockData<Real> *rc);
Expand Down
Loading

0 comments on commit 7b56701

Please sign in to comment.