Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Inconsistent behavior between CPU and GPU for liebendorf deleptonization and cooling #184

Merged
merged 15 commits into from
Dec 13, 2023
Merged

Inconsistent behavior between CPU and GPU for liebendorf deleptonization and cooling #184

Changes from 10 commits
Commits
Show all changes
15 commits
Select commit Hold shift + click to select a range
b1e83cd
fix bool typo, compile fails
Nov 7, 2023
b7bc221
not a typo, sorry, still debugging density error
Nov 7, 2023
4e5b06b
revert
Nov 7, 2023
4e27bfd
revert
Nov 7, 2023
2d08f52
move host function calls outside of gpu kernel
Nov 21, 2023
afae988
format
Nov 21, 2023
4a34543
Code - CGS not the other way around
Nov 29, 2023
025316e
Merge branch 'main' into fix_typo_cooling
Yurlungur Dec 13, 2023
bbdd56e
formatting
Dec 13, 2023
1a77b2d
typo
Dec 13, 2023
76e58d6
fix params
Dec 13, 2023
f316490
inconsistent references to pmesh and pmb
Dec 13, 2023
7e1b530
remove line that doesn't do anything
mari2895 Dec 13, 2023
ee60114
extraneous const defs
Dec 13, 2023
3be4e88
Merge branch 'fix_typo_cooling' of https://github.com/lanl/phoebus in…
Dec 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 37 additions & 20 deletions src/radiation/cooling_function.cpp
View file
Open in desktop
Yurlungur marked this conversation as resolved.
Show resolved Hide resolved
Original file line number Diff line number Diff line change
Expand Up @@ -39,13 +39,14 @@ TaskStatus LightBulbCalcTau(MeshBlockData<Real> *rc) {
IndexRange jb = rc->GetBoundsJ(IndexDomain::interior);
IndexRange kb = rc->GetBoundsK(IndexDomain::interior);

auto &unit_conv =
pmesh->packages.Get("phoebus")->Param<phoebus::UnitConversions>("unit_conv");
auto &unit_conv = params.Get<phoebus::UnitConversions>("unit_conv");
const Real density_conversion_factor = unit_conv.GetMassDensityCodeToCGS();

pmesh->packages.Get("phoebus")->Param<phoebus::UnitConversions>("unit_conv");
parthenon::par_for(
DEFAULT_LOOP_PATTERN, "CalcTau", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e,
KOKKOS_LAMBDA(const int k, const int j, const int i) {
const Real rho =
v(prho, k, j, i) * unit_conv.GetMassDensityCodeToCGS(); // Density in CGS
const Real rho = v(prho, k, j, i) * density_conversion_factor; // Density in CGS
const Real lRho = std::log10(rho);
// Calculate tau
constexpr Real xl1 = LightBulb::HeatAndCool::XL1;
Expand Down Expand Up @@ -156,6 +157,16 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData<Real> *rc, const doub
rad->Param<bool>("do_nu_electron_anti"),
rad->Param<bool>("do_nu_heavy")};

// Code to CGS
const Real density_conversion_factor = unit_conv.GetMassDensityCodeToCGS();
const Real temperature_conversion_factor = unit_conv.GetTemperatureCodeToCGS();
const Real length_conversion_factor = unit_conv.GetLengthCodeToCGS();

// CGS to code
const Real energy_conversion_factor = unit_conv.GetEnergyCGSToCode();
const Real mass_conversion_factor = unit_conv.GetMassCGSToCode();
const Real time_conversion_factor = unit_conv.GetTimeCGSToCode();

parthenon::par_for(
DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s,
kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) {
Expand All @@ -179,14 +190,24 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData<Real> *rc, const doub
const parthenon::AllReduce<bool> *pdo_gain_reducer =
rad->MutableParam<parthenon::AllReduce<bool>>("do_gain_reducer");
const bool do_gain = pdo_gain_reducer->val;

// Code to CGS
const Real density_conversion_factor = unit_conv.GetMassDensityCodeToCGS();
const Real temperature_conversion_factor = unit_conv.GetTemperatureCodeToCGS();
const Real length_conversion_factor = unit_conv.GetLengthCodeToCGS();

// CGS to code
const Real energy_conversion_factor = unit_conv.GetEnergyCGSToCode();
const Real mass_conversion_factor = unit_conv.GetMassCGSToCode();
const Real time_conversion_factor = unit_conv.GetTimeCGSToCode();
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These lines probably don't need to be here, because the same definitions are above in the same function.


parthenon::par_for(
DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s,
kb.e, jb.s, jb.e, ib.s, ib.e,
KOKKOS_LAMBDA(const int k, const int j, const int i) {
const Real r = std::abs(coords.Xc<1>(k, j, i)); // TODO(MG) coord transform game
const Real rho =
v(prho, k, j, i) * unit_conv.GetMassDensityCodeToCGS(); // Density in CGS
const Real cdensity = v(crho, k, j, i); // conserved density
const Real rho = v(prho, k, j, i) * density_conversion_factor; // Density in CGS
const Real cdensity = v(crho, k, j, i); // conserved density
Real Gcov[4][4];
geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov);
Real Ucon[4];
Expand Down Expand Up @@ -245,17 +266,15 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData<Real> *rc, const doub
Real lambda[2];
lambda[0] = Ye;
eos_sc.MassFractionsFromDensityTemperature(
rho, v(ptemp, k, j, i) * unit_conv.GetTemperatureCodeToCGS(), Xa, Xh, Xn,
Xp, Abar, Zbar, lambda);
rho, v(ptemp, k, j, i) * temperature_conversion_factor, Xa, Xh, Xn, Xp,
Abar, Zbar, lambda);
heat = do_gain * (Xn + Xp) * hfac * std::exp(-tau) *
pow((rnorm / (r * unit_conv.GetLengthCodeToCGS())), 2);
cool =
(Xn + Xp) * cfac * std::exp(-tau) *
pow((v(ptemp, k, j, i) * unit_conv.GetTemperatureCodeToCGS() / Tnorm), 6);
pow((rnorm / (r * length_conversion_factor)), 2);
cool = (Xn + Xp) * cfac * std::exp(-tau) *
pow((v(ptemp, k, j, i) * temperature_conversion_factor / Tnorm), 6);

Real CGSToCodeFact = unit_conv.GetEnergyCGSToCode() /
unit_conv.GetMassCGSToCode() /
unit_conv.GetTimeCGSToCode();
Real CGSToCodeFact =
energy_conversion_factor / mass_conversion_factor / time_conversion_factor;

Real tempr = 1 / 30.76 / 9e20;
Real H = heat * CGSToCodeFact;
Expand All @@ -268,10 +287,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData<Real> *rc, const doub
// detg included above
Kokkos::atomic_add(&(v(mu, k, j, i)), -Gcov_coord[mu - Gcov_lo]);
}
v(GcovHeat, k, j, i) =
v(prho, k, j, i) * unit_conv.GetMassDensityCodeToCGS() * heat;
v(GcovCool, k, j, i) =
v(prho, k, j, i) * unit_conv.GetMassDensityCodeToCGS() * cool;
v(GcovHeat, k, j, i) = v(prho, k, j, i) * density_conversion_factor * heat;
v(GcovCool, k, j, i) = v(prho, k, j, i) * density_conversion_factor * cool;
Kokkos::atomic_add(&(v(Gye, k, j, i)), Jye);
});
#else
Expand Down
Loading