From f57529a39390aa13812c2cbe691bda255a12916f Mon Sep 17 00:00:00 2001 From: mari2895 Date: Mon, 8 Jan 2024 14:47:34 -0500 Subject: [PATCH] refactored ApplyRadiationFourForce and CoolingFunctionCalculateFourForce --- src/phoebus_driver.cpp | 10 +-- src/radiation/cooling_function.cpp | 114 +++++++++++++++-------------- src/radiation/radiation.cpp | 41 +++++------ src/radiation/radiation.hpp | 4 +- 4 files changed, 85 insertions(+), 84 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index c888af20b..782908062 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -720,17 +720,17 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer); int reg_dep_id = 0; } + auto calculate_four_force = + tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, + sc0.get(), dt); + auto apply_four_force = tl.AddTask( + calculate_four_force, radiation::ApplyRadiationFourForce, sc0.get(), dt); } TaskRegion &async_region = tc.AddRegion(num_independent_task_lists); for (int ib = 0; ib < num_independent_task_lists; ib++) { auto pmb = blocks[ib].get(); auto &tl = async_region[ib]; auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); - auto calculate_four_force = - tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, - sc0.get(), dt); - auto apply_four_force = tl.AddTask( - calculate_four_force, radiation::ApplyRadiationFourForce, sc0.get(), dt); } } else if (rad_method == "monte_carlo") { return MonteCarloStep(); diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 47746dabf..440dfd247 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -116,38 +116,30 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { return TaskStatus::complete; } -TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const double dt) { +TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt) { namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; auto *pmb = rc->GetParentPointer(); + using parthenon::MakePackDescriptor; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; - std::vector vars({c::density::name(), p::density::name(), - p::velocity::name(), p::temperature::name(), - p::ye::name(), c::energy::name(), iv::Gcov::name(), - iv::GcovHeat::name(), iv::GcovCool::name(), - iv::Gye::name(), iv::tau::name(), p::energy::name()}); + static auto desc = + MakePackDescriptor(resolved_pkgs.get()); + const int nblocks = v.GetNBlocks(); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int crho = imap[c::density::name()].first; - const int prho = imap[p::density::name()].first; - const int pvlo = imap[p::velocity::name()].first; - const int pvhi = imap[p::velocity::name()].second; - const int ptemp = imap[p::temperature::name()].first; - const int pye = imap[p::ye::name()].first; - const int penergy = imap[p::energy::name()].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; - const int ptau = imap[iv::tau::name()].first; - const int GcovHeat = imap[iv::GcovHeat::name()].first; - const int GcovCool = imap[iv::GcovCool::name()].first; + auto v = desc.GetPack(rc); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); + const int nblocks = v.GetNBlocks(); + auto &unit_conv = pmb->packages.Get("phoebus")->Param("unit_conv"); auto rad = pmb->packages.Get("radiation").get(); @@ -176,17 +168,17 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub const Real time_conversion_factor = unit_conv.GetTimeCGSToCode(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s, - kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { // Initialize five-force to zero - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - v(mu, k, j, i) = 0.; + for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + v(b, mu, k, j, i) = 0.; } - v(Gye, k, j, i) = 0.; + v(b, iv::Gye(), k, j, i) = 0.; }); // Light Bulb with Liebendorfer model - auto &coords = pmb->coords; const bool do_liebendorfer = rad->Param("do_liebendorfer"); const bool do_lightbulb = rad->Param("do_lightbulb"); if (do_lightbulb) { @@ -200,16 +192,19 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub const bool do_gain = pdo_gain_reducer->val; parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s, - kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - const Real r = std::abs(coords.Xc<1>(k, j, i)); // TODO(MG) coord transform game - const Real rho = v(prho, k, j, i) * density_conversion_factor; // Density in CGS - const Real cdensity = v(crho, k, j, i); // conserved density + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + auto &coords = v.GetCoordinates(b) const Real r = + std::abs(coords.Xc<1>(k, j, i)); + const Real rho = + v(b, p::density(), k, j, i) * density_conversion_factor; // Density in CGS + const Real cdensity = v(b, c::density(), k, j, i); // conserved density Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(0) + 1, k, j, i), + v(b, p::velocity(0) + 2, k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real Jye = 0.0; @@ -227,7 +222,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub constexpr Real MeVToCGS = 1.16040892301e10; constexpr Real Tnorm = 2.0 * MeVToCGS; - Real Ye = v(pye, k, j, i); + Real Ye = v(b, p::ye(), k, j, i); if (do_liebendorfer) { constexpr Real Ye_beta = 0.27; @@ -257,19 +252,20 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub } Real heat; Real cool; - const Real tau = v(ptau, k, j, i); + const Real tau = v(b, iv::tau(), k, j, i); const Real hfac = LightBulb::HeatAndCool::HFAC * lum; const Real cfac = LightBulb::HeatAndCool::CFAC; Real Xa, Xh, Xn, Xp, Abar, Zbar; Real lambda[2]; lambda[0] = Ye; eos_sc.MassFractionsFromDensityTemperature( - rho, v(ptemp, k, j, i) * temperature_conversion_factor, Xa, Xh, Xn, Xp, - Abar, Zbar, lambda); + rho, v(b, p::temperature(), k, j, i) * temperature_conversion_factor, Xa, + Xh, Xn, Xp, Abar, Zbar, lambda); heat = do_gain * (Xn + Xp) * hfac * std::exp(-tau) * pow((rnorm / (r * length_conversion_factor)), 2); - cool = (Xn + Xp) * cfac * std::exp(-tau) * - pow((v(ptemp, k, j, i) * temperature_conversion_factor / Tnorm), 6); + cool = + (Xn + Xp) * cfac * std::exp(-tau) * + pow((v(b, p::temp(), k, j, i) * temperature_conversion_factor / Tnorm), 6); Real CGSToCodeFact = energy_conversion_factor / mass_conversion_factor / time_conversion_factor; @@ -283,11 +279,13 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { // detg included above - Kokkos::atomic_add(&(v(mu, k, j, i)), -Gcov_coord[mu - Gcov_lo]); + Kokkos::atomic_add(&(v(b, mu, k, j, i)), -Gcov_coord[mu - iv::Gcov(0)]); } - v(GcovHeat, k, j, i) = v(prho, k, j, i) * density_conversion_factor * heat; - v(GcovCool, k, j, i) = v(prho, k, j, i) * density_conversion_factor * cool; - Kokkos::atomic_add(&(v(Gye, k, j, i)), Jye); + v(GcovHeat, k, j, i) = + v(b, p::density(), k, j, i) * density_conversion_factor * heat; + v(GcovCool, k, j, i) = + v(b, p::density(), k, j, i) * density_conversion_factor * cool; + Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), Jye); }); #else PARTHENON_THROW("Lighbulb only supported with HDF5"); @@ -299,32 +297,36 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub auto s = species[sidx]; parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), - kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), - v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), + v(b, p::velocity(0) + 1, k, j, i), + v(b, p::velocity(0) + 2, k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); - const Real Ye = v(pye, k, j, i); + const Real Ye = v(b, p::ye(), k, j, i); - double J = d_opacity.Emissivity(v(prho, k, j, i), v(ptemp, k, j, i), Ye, s); - double Jye = mp_code * d_opacity.NumberEmissivity(v(prho, k, j, i), - v(ptemp, k, j, i), Ye, s); + double J = d_opacity.Emissivity(v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), Ye, s); + double Jye = mp_code * d_opacity.NumberEmissivity( + v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), Ye, s); Real Gcov_tetrad[4] = {-J, 0., 0., 0.}; Real Gcov_coord[4]; Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - Kokkos::atomic_add(&(v(mu, k, j, i)), -detG * Gcov_coord[mu - Gcov_lo]); + for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + Kokkos::atomic_add(&(v(b, mu, k, j, i)), + -detG * Gcov_coord[mu - Gcov_lo]); } - Kokkos::atomic_add(&(v(Gye, k, j, i)), -LeptonSign(s) * detG * Jye); + Kokkos::atomic_add(&(v(b, iv::Gye, k, j, i)), -LeptonSign(s) * detG * Jye); }); } } diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 0303752a6..498c53ac9 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -538,37 +538,36 @@ std::shared_ptr Initialize(ParameterInput *pin) { return physics; } -TaskStatus ApplyRadiationFourForce(MeshBlockData *rc, const double dt) { +TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { PARTHENON_REQUIRE(USE_VALENCIA, "Covariant GRMHD formulation not supported!"); - namespace c = fluid_cons; namespace iv = internal_variables; + using parthenon::MakePackDescriptor; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; + + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); - std::vector vars({c::density::name(), c::energy::name(), - c::momentum::name(), c::ye::name(), iv::Gcov::name(), - iv::Gye::name()}); PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int crho = imap[c::density::name()].first; - const int ceng = imap[c::energy::name()].first; - const int cmom_lo = imap[c::momentum::name()].first; - const int cmom_hi = imap[c::momentum::name()].second; - const int cye = imap[c::ye::name()].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; + auto v = desc.GetPack(rc); + const int nblocks = v.GetNBlocks(); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), kb.s, kb.e, jb.s, - jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { - v(ceng, k, j, i) -= v(Gcov_lo, k, j, i) * dt; - v(cmom_lo, k, j, i) += v(Gcov_lo + 1, k, j, i) * dt; - v(cmom_lo + 1, k, j, i) += v(Gcov_lo + 2, k, j, i) * dt; - v(cmom_lo + 2, k, j, i) += v(Gcov_lo + 3, k, j, i) * dt; - v(cye, k, j, i) += v(Gye, k, j, i) * dt; + DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), 0, nblocks - 1, + kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + v(ceng, k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; + v(cmom_lo, k, j, i) += v(b, iv::Gcov(0) + 1, k, j, i) * dt; + v(cmom_lo + 1, k, j, i) += v(b, iv::Gcov(0) + 2, k, j, i) * dt; + v(cmom_lo + 2, k, j, i) += v(b, iv::Gcov(0) + 3, k, j, i) * dt; + v(cye, k, j, i) += v(b, iv::Gye(), k, j, i) * dt; }); return TaskStatus::complete; } diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index 5daa612f1..b1935d810 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -91,12 +91,12 @@ typedef Kokkos::Random_XorShift64_Pool<> RNGPool; std::shared_ptr Initialize(ParameterInput *pin); -TaskStatus ApplyRadiationFourForce(MeshBlockData *rc, const Real dt); +TaskStatus ApplyRadiationFourForce(MeshData *rc, const Real dt); Real EstimateTimestepBlock(MeshBlockData *rc); // Cooling function tasks -TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const Real dt); +TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const Real dt); TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global); TaskStatus LightBulbCalcTau(MeshData *rc);