From 1c3a0f8c69c0980ee951419c6410120d8a6b0f7f Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 15:26:36 -0500 Subject: [PATCH 01/26] refactoring LightBulbCalcTau --- src/radiation/cooling_function.cpp | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 2762eaf3..652c09b8 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -16,36 +16,41 @@ #include "phoebus_utils/variables.hpp" #include "radiation.hpp" #include +#include namespace radiation { using Microphysics::Opacities; using Microphysics::RadiationType; -TaskStatus LightBulbCalcTau(MeshBlockData *rc) { +TaskStatus LightBulbCalcTau(MeshData *rc) { namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + using parthenon::MakePackescriptor; Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; + + static auto desc = MakePackDescriptor(resolved_pkgs.get()); - std::vector vars({p::density::name(), iv::tau::name()}); PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int prho = imap[p::density::name()].first; - const int ptau = imap[iv::tau::name()].first; + auto v = rc->desc.GetPack(rc); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); + const int nblocks=v.GetNBlocks(); + auto &unit_conv = pmesh->packages.Get("phoebus")->Param("unit_conv"); const Real density_conversion_factor = unit_conv.GetMassDensityCodeToCGS(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CalcTau", DevExecSpace(), kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - const Real rho = v(prho, k, j, i) * density_conversion_factor; // Density in CGS + DEFAULT_LOOP_PATTERN, "CalcTau", DevExecSpace(), 0, nblocks-1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + const Real rho = v(b, p::density, k, j, i) * density_conversion_factor; // Density in CGS const Real lRho = std::log10(rho); // Calculate tau constexpr Real xl1 = LightBulb::HeatAndCool::XL1; @@ -64,7 +69,7 @@ TaskStatus LightBulbCalcTau(MeshBlockData *rc) { } else { tau = std::pow(10, (yl3 - yl2) / (xl3 - xl2) * (lRho - xl2) + yl2); } - v(ptau, k, j, i) = tau; + v(b, iv::tau, k, j, i) = tau; }); return TaskStatus::complete; } From 882565a8a00d832f47e9d62e64fbdc888646356c Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 15:29:52 -0500 Subject: [PATCH 02/26] formatting --- src/radiation/cooling_function.cpp | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 652c09b8..fcb5d313 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -34,7 +34,6 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { static auto desc = MakePackDescriptor(resolved_pkgs.get()); - PackIndexMap imap; auto v = rc->desc.GetPack(rc); @@ -42,15 +41,17 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); - const int nblocks=v.GetNBlocks(); - + const int nblocks = v.GetNBlocks(); + auto &unit_conv = pmesh->packages.Get("phoebus")->Param("unit_conv"); const Real density_conversion_factor = unit_conv.GetMassDensityCodeToCGS(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CalcTau", DevExecSpace(), 0, nblocks-1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + DEFAULT_LOOP_PATTERN, "CalcTau", DevExecSpace(), 0, nblocks - 1, kb.s, kb.e, jb.s, + jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - const Real rho = v(b, p::density, k, j, i) * density_conversion_factor; // Density in CGS + const Real rho = + v(b, p::density, k, j, i) * density_conversion_factor; // Density in CGS const Real lRho = std::log10(rho); // Calculate tau constexpr Real xl1 = LightBulb::HeatAndCool::XL1; From 8290819c14231bf9b986d67a2ca0f7bb5ed678c7 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 15:52:48 -0500 Subject: [PATCH 03/26] creating new sync region for lightbulb functions --- src/phoebus_driver.cpp | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 38ceb8f3..46c7e7dd 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -698,11 +698,13 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { if (do_lightbulb) { pdo_gain_reducer = rad->MutableParam>("do_gain_reducer"); } - TaskRegion &async_region = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region[ib]; - auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); + //creating a new sync region for light bulb functions + TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); + for (int ib = 0; ib < mun_partiotions; ++ib){ + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage-1], ib); + auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); + auto &tl = sync_region_lb[ib]; auto finish_gain_reducer = none; if (do_lightbulb) { auto calc_tau = tl.AddTask(none, radiation::LightBulbCalcTau, sc0.get()); @@ -717,9 +719,9 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { tl.AddTask(start_gain_reducer, &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); int reg_dep_id = 0; - async_region.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer); - } - + + } + auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, sc0.get(), dt); From 6389fa5305da7c5d293ec5ec2789cadcfe0e0436 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 15:58:15 -0500 Subject: [PATCH 04/26] refactoring CheckDoGian --- src/phoebus_driver.cpp | 11 +++++------ src/radiation/cooling_function.cpp | 21 ++++++++++++--------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 46c7e7dd..9ed2a037 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -698,11 +698,11 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { if (do_lightbulb) { pdo_gain_reducer = rad->MutableParam>("do_gain_reducer"); } - //creating a new sync region for light bulb functions + // creating a new sync region for light bulb functions TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); - for (int ib = 0; ib < mun_partiotions; ++ib){ + for (int ib = 0; ib < mun_partiotions; ++ib) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); - auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage-1], ib); + auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); auto &tl = sync_region_lb[ib]; auto finish_gain_reducer = none; @@ -719,9 +719,8 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { tl.AddTask(start_gain_reducer, &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); int reg_dep_id = 0; - - } - + } + auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, sc0.get(), dt); diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index fcb5d313..3b7e204a 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -75,22 +75,25 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { return TaskStatus::complete; } -TaskStatus CheckDoGain(MeshBlockData *rc, bool *do_gain_global) { +TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { if (*do_gain_global) { return TaskStatus::complete; } namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; + using parthenon::MakePackescriptor; Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; - std::vector vars({iv::tau::name()}); + static auto desc = MakePackDescriptor(resolved_pkgs.get()); PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int ptau = imap[iv::tau::name()].first; + auto v = rc->desc.GetPack(rc); + const int nblocks = v.GetNblocks() - IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); @@ -102,10 +105,10 @@ TaskStatus CheckDoGain(MeshBlockData *rc, bool *do_gain_global) { int do_gain_local = 0; bool do_gain; parthenon::par_reduce( - parthenon::loop_pattern_mdrange_tag, "calc_do_gain", DevExecSpace(), kb.s, kb.e, - jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i, int &do_gain) { - do_gain = do_gain + (v(ptau, k, j, i) > 1.e2); + parthenon::loop_pattern_mdrange_tag, "calc_do_gain", DevExecSpace(), 0, + nblocks - 1 kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, int &do_gain) { + do_gain = do_gain + (v(b, iv::tau, k, j, i) > 1.e2); }, Kokkos::Sum(do_gain_local)); do_gain = do_gain_local; From 73a7883357c7561804f4cc3a03b8c925f73b4a66 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 16:29:54 -0500 Subject: [PATCH 05/26] fixed a typos --- src/phoebus_driver.cpp | 3 ++- src/radiation/cooling_function.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 9ed2a037..ac95a33a 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -699,8 +699,9 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer = rad->MutableParam>("do_gain_reducer"); } // creating a new sync region for light bulb functions + const int num_partitions = pmesh->DefaultNumPartitions(); TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); - for (int ib = 0; ib < mun_partiotions; ++ib) { + for (int ib = 0; ib < num_partiotions; ++ib) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 3b7e204a..52e5768a 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -51,7 +51,7 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { const Real rho = - v(b, p::density, k, j, i) * density_conversion_factor; // Density in CGS + v(b, p::density(), k, j, i) * density_conversion_factor; // Density in CGS const Real lRho = std::log10(rho); // Calculate tau constexpr Real xl1 = LightBulb::HeatAndCool::XL1; @@ -70,7 +70,7 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { } else { tau = std::pow(10, (yl3 - yl2) / (xl3 - xl2) * (lRho - xl2) + yl2); } - v(b, iv::tau, k, j, i) = tau; + v(b, iv::tau(), k, j, i) = tau; }); return TaskStatus::complete; } @@ -87,7 +87,7 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { auto &resolved_pkgs = pmesh->resolved_packages; const int ndim = pmesh->ndim; - static auto desc = MakePackDescriptor(resolved_pkgs.get()); + static auto desc = MakePackDescriptor(resolved_pkgs.get()); PackIndexMap imap; auto v = rc->desc.GetPack(rc); @@ -108,7 +108,7 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { parthenon::loop_pattern_mdrange_tag, "calc_do_gain", DevExecSpace(), 0, nblocks - 1 kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, int &do_gain) { - do_gain = do_gain + (v(b, iv::tau, k, j, i) > 1.e2); + do_gain = do_gain + (v(b, iv::tau(), k, j, i) > 1.e2); }, Kokkos::Sum(do_gain_local)); do_gain = do_gain_local; From 70341076fc13b29ec4cec61da02f580cb7bedb35 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 17:05:16 -0500 Subject: [PATCH 06/26] typo fixed --- src/radiation/cooling_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 52e5768a..2de39d99 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -105,8 +105,8 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { int do_gain_local = 0; bool do_gain; parthenon::par_reduce( - parthenon::loop_pattern_mdrange_tag, "calc_do_gain", DevExecSpace(), 0, - nblocks - 1 kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + parthenon::loop_pattern_mdrange_tag, "calc_do_gain", DevExecSpace(), 0, nblocks - 1, + kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i, int &do_gain) { do_gain = do_gain + (v(b, iv::tau(), k, j, i) > 1.e2); }, From 6ebdc38cc466fa42f157fc3d7c3c13902ad5bdf5 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 17:23:46 -0500 Subject: [PATCH 07/26] typo fixed --- src/radiation/cooling_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 2de39d99..b2ab2939 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -27,7 +27,7 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; - using parthenon::MakePackescriptor; + using parthenon::MakePackDescriptor; Mesh *pmesh = rc->GetMeshPointer(); auto &resolved_pkgs = pmesh->resolved_packages; const int ndim = pmesh->ndim; @@ -82,7 +82,7 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; - using parthenon::MakePackescriptor; + using parthenon::MakePackDescriptor; Mesh *pmesh = rc->GetMeshPointer(); auto &resolved_pkgs = pmesh->resolved_packages; const int ndim = pmesh->ndim; From f8378755e7fbacccc6b135fd15326ee45efd87bf Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 17:33:41 -0500 Subject: [PATCH 08/26] typo fixed --- src/phoebus_driver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index ac95a33a..1e0b122b 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -701,7 +701,7 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { // creating a new sync region for light bulb functions const int num_partitions = pmesh->DefaultNumPartitions(); TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); - for (int ib = 0; ib < num_partiotions; ++ib) { + for (int ib = 0; ib < num_partitions; ++ib) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); From 5418296875a6772815eb3c02655a974bebe6d572 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 17:46:39 -0500 Subject: [PATCH 09/26] typo fixed --- src/radiation/cooling_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index b2ab2939..d387f213 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -35,7 +35,7 @@ TaskStatus LightBulbCalcTau(MeshData *rc) { static auto desc = MakePackDescriptor(resolved_pkgs.get()); PackIndexMap imap; - auto v = rc->desc.GetPack(rc); + auto v = desc.GetPack(rc); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); @@ -90,7 +90,7 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { static auto desc = MakePackDescriptor(resolved_pkgs.get()); PackIndexMap imap; - auto v = rc->desc.GetPack(rc); + auto v = desc.GetPack(rc); const int nblocks = v.GetNblocks() IndexRange ib = rc->GetBoundsI(IndexDomain::interior); From 18790b50ec43da6085e3d412a98fce9a65dc2755 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 17:57:04 -0500 Subject: [PATCH 10/26] typo fixed --- src/radiation/cooling_function.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index d387f213..1e009cc2 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -91,7 +91,7 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { PackIndexMap imap; auto v = desc.GetPack(rc); - const int nblocks = v.GetNblocks() + const int nblocks = v.GetNBlocks() IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); From bbf0cbc3a855ade0d6f4468efcda8318753ebdae Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 4 Jan 2024 18:18:27 -0500 Subject: [PATCH 11/26] typo fixed --- src/radiation/cooling_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 1e009cc2..47746dab 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -91,9 +91,9 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { PackIndexMap imap; auto v = desc.GetPack(rc); - const int nblocks = v.GetNBlocks() + const int nblocks = v.GetNBlocks(); - IndexRange ib = rc->GetBoundsI(IndexDomain::interior); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); From 01ca55f4e6795ae263220bc9d5a6e31896fd4b1e Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 5 Jan 2024 12:09:08 -0500 Subject: [PATCH 12/26] stage_name[stage-1] replaced with base --- src/phoebus_driver.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 1e0b122b..b85348cd 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -703,8 +703,7 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); for (int ib = 0; ib < num_partitions; ++ib) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); - auto &sc0 = pmesh->mesh_data.GetOrAdd(stage_name[stage - 1], ib); - auto &sc1 = pmesh->mesh_data.GetOrAdd(stage_name[stage], ib); + auto &sc0 = pmesh->mesh_data.GetOrAdd("base", ib); auto &tl = sync_region_lb[ib]; auto finish_gain_reducer = none; if (do_lightbulb) { From 5c563ab41051f4e72dd72838d707f65c4e152964 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 5 Jan 2024 12:32:35 -0500 Subject: [PATCH 13/26] changing arguments of light bulb functions in radiation.hpp --- src/radiation/radiation.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index ee2b7365..5daa612f 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -97,8 +97,8 @@ Real EstimateTimestepBlock(MeshBlockData *rc); // Cooling function tasks TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const Real dt); -TaskStatus CheckDoGain(MeshBlockData *rc, bool *do_gain_global); -TaskStatus LightBulbCalcTau(MeshBlockData *rc); +TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global); +TaskStatus LightBulbCalcTau(MeshData *rc); // Monte Carlo tasks TaskStatus MonteCarloSourceParticles(MeshBlock *pmb, MeshBlockData *rc, From 5ec5d555cfb541adefea4de696c67af443e67886 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 5 Jan 2024 13:05:29 -0500 Subject: [PATCH 14/26] fixed incorrectly deleted async region --- src/phoebus_driver.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index b85348cd..c888af20 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -720,7 +720,12 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer); int reg_dep_id = 0; } - + } + TaskRegion &async_region = tc.AddRegion(num_independent_task_lists); + for (int ib = 0; ib < num_independent_task_lists; ib++) { + auto pmb = blocks[ib].get(); + auto &tl = async_region[ib]; + auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, sc0.get(), dt); From f57529a39390aa13812c2cbe691bda255a12916f Mon Sep 17 00:00:00 2001 From: mari2895 Date: Mon, 8 Jan 2024 14:47:34 -0500 Subject: [PATCH 15/26] refactored ApplyRadiationFourForce and CoolingFunctionCalculateFourForce --- src/phoebus_driver.cpp | 10 +-- src/radiation/cooling_function.cpp | 114 +++++++++++++++-------------- src/radiation/radiation.cpp | 41 +++++------ src/radiation/radiation.hpp | 4 +- 4 files changed, 85 insertions(+), 84 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index c888af20..78290806 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -720,17 +720,17 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { pdo_gain_reducer); int reg_dep_id = 0; } + auto calculate_four_force = + tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, + sc0.get(), dt); + auto apply_four_force = tl.AddTask( + calculate_four_force, radiation::ApplyRadiationFourForce, sc0.get(), dt); } TaskRegion &async_region = tc.AddRegion(num_independent_task_lists); for (int ib = 0; ib < num_independent_task_lists; ib++) { auto pmb = blocks[ib].get(); auto &tl = async_region[ib]; auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); - auto calculate_four_force = - tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, - sc0.get(), dt); - auto apply_four_force = tl.AddTask( - calculate_four_force, radiation::ApplyRadiationFourForce, sc0.get(), dt); } } else if (rad_method == "monte_carlo") { return MonteCarloStep(); diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 47746dab..440dfd24 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -116,38 +116,30 @@ TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global) { return TaskStatus::complete; } -TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const double dt) { +TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt) { namespace p = fluid_prim; namespace c = fluid_cons; namespace iv = internal_variables; auto *pmb = rc->GetParentPointer(); + using parthenon::MakePackDescriptor; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; - std::vector vars({c::density::name(), p::density::name(), - p::velocity::name(), p::temperature::name(), - p::ye::name(), c::energy::name(), iv::Gcov::name(), - iv::GcovHeat::name(), iv::GcovCool::name(), - iv::Gye::name(), iv::tau::name(), p::energy::name()}); + static auto desc = + MakePackDescriptor(resolved_pkgs.get()); + const int nblocks = v.GetNBlocks(); - PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int crho = imap[c::density::name()].first; - const int prho = imap[p::density::name()].first; - const int pvlo = imap[p::velocity::name()].first; - const int pvhi = imap[p::velocity::name()].second; - const int ptemp = imap[p::temperature::name()].first; - const int pye = imap[p::ye::name()].first; - const int penergy = imap[p::energy::name()].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; - const int ptau = imap[iv::tau::name()].first; - const int GcovHeat = imap[iv::GcovHeat::name()].first; - const int GcovCool = imap[iv::GcovCool::name()].first; + auto v = desc.GetPack(rc); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); + const int nblocks = v.GetNBlocks(); + auto &unit_conv = pmb->packages.Get("phoebus")->Param("unit_conv"); auto rad = pmb->packages.Get("radiation").get(); @@ -176,17 +168,17 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub const Real time_conversion_factor = unit_conv.GetTimeCGSToCode(); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s, - kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { // Initialize five-force to zero - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - v(mu, k, j, i) = 0.; + for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + v(b, mu, k, j, i) = 0.; } - v(Gye, k, j, i) = 0.; + v(b, iv::Gye(), k, j, i) = 0.; }); // Light Bulb with Liebendorfer model - auto &coords = pmb->coords; const bool do_liebendorfer = rad->Param("do_liebendorfer"); const bool do_lightbulb = rad->Param("do_lightbulb"); if (do_lightbulb) { @@ -200,16 +192,19 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub const bool do_gain = pdo_gain_reducer->val; parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), kb.s, - kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { - const Real r = std::abs(coords.Xc<1>(k, j, i)); // TODO(MG) coord transform game - const Real rho = v(prho, k, j, i) * density_conversion_factor; // Density in CGS - const Real cdensity = v(crho, k, j, i); // conserved density + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + auto &coords = v.GetCoordinates(b) const Real r = + std::abs(coords.Xc<1>(k, j, i)); + const Real rho = + v(b, p::density(), k, j, i) * density_conversion_factor; // Density in CGS + const Real cdensity = v(b, c::density(), k, j, i); // conserved density Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(0) + 1, k, j, i), + v(b, p::velocity(0) + 2, k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real Jye = 0.0; @@ -227,7 +222,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub constexpr Real MeVToCGS = 1.16040892301e10; constexpr Real Tnorm = 2.0 * MeVToCGS; - Real Ye = v(pye, k, j, i); + Real Ye = v(b, p::ye(), k, j, i); if (do_liebendorfer) { constexpr Real Ye_beta = 0.27; @@ -257,19 +252,20 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub } Real heat; Real cool; - const Real tau = v(ptau, k, j, i); + const Real tau = v(b, iv::tau(), k, j, i); const Real hfac = LightBulb::HeatAndCool::HFAC * lum; const Real cfac = LightBulb::HeatAndCool::CFAC; Real Xa, Xh, Xn, Xp, Abar, Zbar; Real lambda[2]; lambda[0] = Ye; eos_sc.MassFractionsFromDensityTemperature( - rho, v(ptemp, k, j, i) * temperature_conversion_factor, Xa, Xh, Xn, Xp, - Abar, Zbar, lambda); + rho, v(b, p::temperature(), k, j, i) * temperature_conversion_factor, Xa, + Xh, Xn, Xp, Abar, Zbar, lambda); heat = do_gain * (Xn + Xp) * hfac * std::exp(-tau) * pow((rnorm / (r * length_conversion_factor)), 2); - cool = (Xn + Xp) * cfac * std::exp(-tau) * - pow((v(ptemp, k, j, i) * temperature_conversion_factor / Tnorm), 6); + cool = + (Xn + Xp) * cfac * std::exp(-tau) * + pow((v(b, p::temp(), k, j, i) * temperature_conversion_factor / Tnorm), 6); Real CGSToCodeFact = energy_conversion_factor / mass_conversion_factor / time_conversion_factor; @@ -283,11 +279,13 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { // detg included above - Kokkos::atomic_add(&(v(mu, k, j, i)), -Gcov_coord[mu - Gcov_lo]); + Kokkos::atomic_add(&(v(b, mu, k, j, i)), -Gcov_coord[mu - iv::Gcov(0)]); } - v(GcovHeat, k, j, i) = v(prho, k, j, i) * density_conversion_factor * heat; - v(GcovCool, k, j, i) = v(prho, k, j, i) * density_conversion_factor * cool; - Kokkos::atomic_add(&(v(Gye, k, j, i)), Jye); + v(GcovHeat, k, j, i) = + v(b, p::density(), k, j, i) * density_conversion_factor * heat; + v(GcovCool, k, j, i) = + v(b, p::density(), k, j, i) * density_conversion_factor * cool; + Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), Jye); }); #else PARTHENON_THROW("Lighbulb only supported with HDF5"); @@ -299,32 +297,36 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const doub auto s = species[sidx]; parthenon::par_for( - DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), - kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, - KOKKOS_LAMBDA(const int k, const int j, const int i) { + DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, + nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(pvlo, k, j, i), v(pvlo + 1, k, j, i), - v(pvlo + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), + v(b, p::velocity(0) + 1, k, j, i), + v(b, p::velocity(0) + 2, k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); - const Real Ye = v(pye, k, j, i); + const Real Ye = v(b, p::ye(), k, j, i); - double J = d_opacity.Emissivity(v(prho, k, j, i), v(ptemp, k, j, i), Ye, s); - double Jye = mp_code * d_opacity.NumberEmissivity(v(prho, k, j, i), - v(ptemp, k, j, i), Ye, s); + double J = d_opacity.Emissivity(v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), Ye, s); + double Jye = mp_code * d_opacity.NumberEmissivity( + v(b, p::density(), k, j, i), + v(b, p::temperature(), k, j, i), Ye, s); Real Gcov_tetrad[4] = {-J, 0., 0., 0.}; Real Gcov_coord[4]; Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { - Kokkos::atomic_add(&(v(mu, k, j, i)), -detG * Gcov_coord[mu - Gcov_lo]); + for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + Kokkos::atomic_add(&(v(b, mu, k, j, i)), + -detG * Gcov_coord[mu - Gcov_lo]); } - Kokkos::atomic_add(&(v(Gye, k, j, i)), -LeptonSign(s) * detG * Jye); + Kokkos::atomic_add(&(v(b, iv::Gye, k, j, i)), -LeptonSign(s) * detG * Jye); }); } } diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 0303752a..498c53ac 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -538,37 +538,36 @@ std::shared_ptr Initialize(ParameterInput *pin) { return physics; } -TaskStatus ApplyRadiationFourForce(MeshBlockData *rc, const double dt) { +TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { PARTHENON_REQUIRE(USE_VALENCIA, "Covariant GRMHD formulation not supported!"); - namespace c = fluid_cons; namespace iv = internal_variables; + using parthenon::MakePackDescriptor; + Mesh *pmesh = rc->GetMeshPointer(); + auto &resolved_pkgs = pmesh->resolved_packages; + const int ndim = pmesh->ndim; + + static auto desc = + MakePackDescriptor( + resolved_pkgs.get()); - std::vector vars({c::density::name(), c::energy::name(), - c::momentum::name(), c::ye::name(), iv::Gcov::name(), - iv::Gye::name()}); PackIndexMap imap; - auto v = rc->PackVariables(vars, imap); - const int crho = imap[c::density::name()].first; - const int ceng = imap[c::energy::name()].first; - const int cmom_lo = imap[c::momentum::name()].first; - const int cmom_hi = imap[c::momentum::name()].second; - const int cye = imap[c::ye::name()].first; - const int Gcov_lo = imap[iv::Gcov::name()].first; - const int Gcov_hi = imap[iv::Gcov::name()].second; - const int Gye = imap[iv::Gye::name()].first; + auto v = desc.GetPack(rc); + const int nblocks = v.GetNBlocks(); + IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); parthenon::par_for( - DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), kb.s, kb.e, jb.s, - jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int k, const int j, const int i) { - v(ceng, k, j, i) -= v(Gcov_lo, k, j, i) * dt; - v(cmom_lo, k, j, i) += v(Gcov_lo + 1, k, j, i) * dt; - v(cmom_lo + 1, k, j, i) += v(Gcov_lo + 2, k, j, i) * dt; - v(cmom_lo + 2, k, j, i) += v(Gcov_lo + 3, k, j, i) * dt; - v(cye, k, j, i) += v(Gye, k, j, i) * dt; + DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), 0, nblocks - 1, + kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, + KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { + v(ceng, k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; + v(cmom_lo, k, j, i) += v(b, iv::Gcov(0) + 1, k, j, i) * dt; + v(cmom_lo + 1, k, j, i) += v(b, iv::Gcov(0) + 2, k, j, i) * dt; + v(cmom_lo + 2, k, j, i) += v(b, iv::Gcov(0) + 3, k, j, i) * dt; + v(cye, k, j, i) += v(b, iv::Gye(), k, j, i) * dt; }); return TaskStatus::complete; } diff --git a/src/radiation/radiation.hpp b/src/radiation/radiation.hpp index 5daa612f..b1935d81 100644 --- a/src/radiation/radiation.hpp +++ b/src/radiation/radiation.hpp @@ -91,12 +91,12 @@ typedef Kokkos::Random_XorShift64_Pool<> RNGPool; std::shared_ptr Initialize(ParameterInput *pin); -TaskStatus ApplyRadiationFourForce(MeshBlockData *rc, const Real dt); +TaskStatus ApplyRadiationFourForce(MeshData *rc, const Real dt); Real EstimateTimestepBlock(MeshBlockData *rc); // Cooling function tasks -TaskStatus CoolingFunctionCalculateFourForce(MeshBlockData *rc, const Real dt); +TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const Real dt); TaskStatus CheckDoGain(MeshData *rc, bool *do_gain_global); TaskStatus LightBulbCalcTau(MeshData *rc); From 1ea215ba0bd1c61d7ba2753c56ff6bbd35212006 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Mon, 8 Jan 2024 15:47:40 -0500 Subject: [PATCH 16/26] more refactoring --- src/radiation/radiation.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 498c53ac..fa365d6f 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -563,11 +563,11 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), 0, nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - v(ceng, k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; - v(cmom_lo, k, j, i) += v(b, iv::Gcov(0) + 1, k, j, i) * dt; - v(cmom_lo + 1, k, j, i) += v(b, iv::Gcov(0) + 2, k, j, i) * dt; - v(cmom_lo + 2, k, j, i) += v(b, iv::Gcov(0) + 3, k, j, i) * dt; - v(cye, k, j, i) += v(b, iv::Gye(), k, j, i) * dt; + v(b, c::engergy(), k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; + v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(0) + 1, k, j, i) * dt; + v(b, c::momentum(0) + 1, k, j, i) += v(b, iv::Gcov(0) + 2, k, j, i) * dt; + v(b, c::momentum(0) + 2, k, j, i) += v(b, iv::Gcov(0) + 3, k, j, i) * dt; + v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; }); return TaskStatus::complete; } From 75194152a8322c2873342f1f5dc68334849f27c3 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 11 Jan 2024 13:06:25 -0500 Subject: [PATCH 17/26] corrected indices --- src/radiation/cooling_function.cpp | 17 +++++++---------- src/radiation/radiation.cpp | 8 ++++---- 2 files changed, 11 insertions(+), 14 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 440dfd24..1123b60a 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -138,8 +138,6 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); IndexRange kb = rc->GetBoundsK(IndexDomain::interior); - const int nblocks = v.GetNBlocks(); - auto &unit_conv = pmb->packages.Get("phoebus")->Param("unit_conv"); auto rad = pmb->packages.Get("radiation").get(); @@ -172,7 +170,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { // Initialize five-force to zero - for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { v(b, mu, k, j, i) = 0.; } v(b, iv::Gye(), k, j, i) = 0.; @@ -203,8 +201,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(0) + 1, k, j, i), - v(b, p::velocity(0) + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), + v(b, p::velocity(2), k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real Jye = 0.0; @@ -303,9 +301,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Real Gcov[4][4]; geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); Real Ucon[4]; - Real vel[3] = {v(b, p::velocity(0), k, j, i), - v(b, p::velocity(0) + 1, k, j, i), - v(b, p::velocity(0) + 2, k, j, i)}; + Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), + v(b, p::velocity(2), k, j, i)}; GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); @@ -322,9 +319,9 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); Real detG = geom.DetG(CellLocation::Cent, k, j, i); - for (int mu = iv::Gcov(0); mu <= iv::Gcov(0) + 3; mu++) { + for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { Kokkos::atomic_add(&(v(b, mu, k, j, i)), - -detG * Gcov_coord[mu - Gcov_lo]); + -detG * Gcov_coord[mu - iv::Gcov(0)]); } Kokkos::atomic_add(&(v(b, iv::Gye, k, j, i)), -LeptonSign(s) * detG * Jye); }); diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index fa365d6f..cfc44dda 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -563,10 +563,10 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { DEFAULT_LOOP_PATTERN, "ApplyRadiationFourForce", DevExecSpace(), 0, nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - v(b, c::engergy(), k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; - v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(0) + 1, k, j, i) * dt; - v(b, c::momentum(0) + 1, k, j, i) += v(b, iv::Gcov(0) + 2, k, j, i) * dt; - v(b, c::momentum(0) + 2, k, j, i) += v(b, iv::Gcov(0) + 3, k, j, i) * dt; + v(b, c::energy(), k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; + v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(1), k, j, i) * dt; + v(b, c::momentum(0) + 1, k, j, i) += v(b, iv::Gcov(2), k, j, i) * dt; + v(b, c::momentum(0) + 2, k, j, i) += v(b, iv::Gcov(3), k, j, i) * dt; v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; }); return TaskStatus::complete; From 60c0a6e932cea50c086a8f00216c8a2723634824 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 11 Jan 2024 16:38:33 -0500 Subject: [PATCH 18/26] more typos --- src/radiation/cooling_function.cpp | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 1123b60a..3e3524d5 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -130,9 +130,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt MakePackDescriptor(resolved_pkgs.get()); - const int nblocks = v.GetNBlocks(); - auto v = desc.GetPack(rc); + const int nblocks = v.GetNBlocks(); IndexRange ib = rc->GetBoundsI(IndexDomain::interior); IndexRange jb = rc->GetBoundsJ(IndexDomain::interior); @@ -193,8 +192,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt DEFAULT_LOOP_PATTERN, "CoolingFunctionCalculateFourForce", DevExecSpace(), 0, nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { - auto &coords = v.GetCoordinates(b) const Real r = - std::abs(coords.Xc<1>(k, j, i)); + auto &coords = v.GetCoordinates(b); + const Real r = std::abs(coords.Xc<1>(k, j, i)); const Real rho = v(b, p::density(), k, j, i) * density_conversion_factor; // Density in CGS const Real cdensity = v(b, c::density(), k, j, i); // conserved density @@ -261,9 +260,10 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Xh, Xn, Xp, Abar, Zbar, lambda); heat = do_gain * (Xn + Xp) * hfac * std::exp(-tau) * pow((rnorm / (r * length_conversion_factor)), 2); - cool = - (Xn + Xp) * cfac * std::exp(-tau) * - pow((v(b, p::temp(), k, j, i) * temperature_conversion_factor / Tnorm), 6); + cool = (Xn + Xp) * cfac * std::exp(-tau) * + pow((v(b, p::temperature(), k, j, i) * temperature_conversion_factor / + Tnorm), + 6); Real CGSToCodeFact = energy_conversion_factor / mass_conversion_factor / time_conversion_factor; @@ -275,13 +275,13 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Real Gcov_tetrad[4] = {J, 0., 0., 0.}; // minus sign included above Real Gcov_coord[4]; Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); - for (int mu = Gcov_lo; mu <= Gcov_lo + 3; mu++) { + for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { // detg included above Kokkos::atomic_add(&(v(b, mu, k, j, i)), -Gcov_coord[mu - iv::Gcov(0)]); } - v(GcovHeat, k, j, i) = + v(b, iv::GcovHeat(), k, j, i) = v(b, p::density(), k, j, i) * density_conversion_factor * heat; - v(GcovCool, k, j, i) = + v(b, iv::GcovCool(), k, j, i) = v(b, p::density(), k, j, i) * density_conversion_factor * cool; Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), Jye); }); @@ -323,7 +323,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Kokkos::atomic_add(&(v(b, mu, k, j, i)), -detG * Gcov_coord[mu - iv::Gcov(0)]); } - Kokkos::atomic_add(&(v(b, iv::Gye, k, j, i)), -LeptonSign(s) * detG * Jye); + Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), + -LeptonSign(s) * detG * Jye); }); } } From 97f35a5677ed3f72296a8381bd6320b77e51c6aa Mon Sep 17 00:00:00 2001 From: mari2895 Date: Thu, 11 Jan 2024 17:16:25 -0500 Subject: [PATCH 19/26] index game agan --- src/phoebus_driver.cpp | 7 +------ src/radiation/cooling_function.cpp | 10 +++++----- src/radiation/radiation.cpp | 4 ++-- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 78290806..9b2984b8 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -719,6 +719,7 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { tl.AddTask(start_gain_reducer, &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); int reg_dep_id = 0; + sync_region.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer); } auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, @@ -726,12 +727,6 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { auto apply_four_force = tl.AddTask( calculate_four_force, radiation::ApplyRadiationFourForce, sc0.get(), dt); } - TaskRegion &async_region = tc.AddRegion(num_independent_task_lists); - for (int ib = 0; ib < num_independent_task_lists; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region[ib]; - auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); - } } else if (rad_method == "monte_carlo") { return MonteCarloStep(); } diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 3e3524d5..633505cb 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -198,11 +198,11 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt v(b, p::density(), k, j, i) * density_conversion_factor; // Density in CGS const Real cdensity = v(b, c::density(), k, j, i); // conserved density Real Gcov[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, Gcov); Real Ucon[4]; Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), v(b, p::velocity(2), k, j, i)}; - GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); + GetFourVelocity(vel, geom, CellLocation::Cent, b, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); Real Jye = 0.0; Real J; @@ -299,11 +299,11 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { Real Gcov[4][4]; - geom.SpacetimeMetric(CellLocation::Cent, k, j, i, Gcov); + geom.SpacetimeMetric(CellLocation::Cent, b, k, j, i, Gcov); Real Ucon[4]; Real vel[3] = {v(b, p::velocity(0), k, j, i), v(b, p::velocity(1), k, j, i), v(b, p::velocity(2), k, j, i)}; - GetFourVelocity(vel, geom, CellLocation::Cent, k, j, i, Ucon); + GetFourVelocity(vel, geom, CellLocation::Cent, b, k, j, i, Ucon); Geometry::Tetrads Tetrads(Ucon, Gcov); const Real Ye = v(b, p::ye(), k, j, i); @@ -317,7 +317,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Real Gcov_tetrad[4] = {-J, 0., 0., 0.}; Real Gcov_coord[4]; Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); - Real detG = geom.DetG(CellLocation::Cent, k, j, i); + Real detG = geom.DetG(CellLocation::Cent, b, k, j, i); for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { Kokkos::atomic_add(&(v(b, mu, k, j, i)), diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index cfc44dda..f187bd40 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -565,8 +565,8 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { v(b, c::energy(), k, j, i) -= v(b, iv::Gcov(0), k, j, i) * dt; v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(1), k, j, i) * dt; - v(b, c::momentum(0) + 1, k, j, i) += v(b, iv::Gcov(2), k, j, i) * dt; - v(b, c::momentum(0) + 2, k, j, i) += v(b, iv::Gcov(3), k, j, i) * dt; + v(b, c::momentum(1), k, j, i) += v(b, iv::Gcov(2), k, j, i) * dt; + v(b, c::momentum(2), k, j, i) += v(b, iv::Gcov(3), k, j, i) * dt; v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; }); return TaskStatus::complete; From 9e25eb4cc24f67cbec944e45587a5c72021a5105 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 12 Jan 2024 14:54:46 -0500 Subject: [PATCH 20/26] fixed loops in mu --- src/radiation/cooling_function.cpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 633505cb..d307e9e0 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -169,8 +169,8 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt nblocks - 1, kb.s, kb.e, jb.s, jb.e, ib.s, ib.e, KOKKOS_LAMBDA(const int b, const int k, const int j, const int i) { // Initialize five-force to zero - for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { - v(b, mu, k, j, i) = 0.; + for (int mu = 0; mu <= 3; mu++) { + v(b, iv::Gcov(mu), k, j, i) = 0.; } v(b, iv::Gye(), k, j, i) = 0.; }); @@ -275,9 +275,10 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Real Gcov_tetrad[4] = {J, 0., 0., 0.}; // minus sign included above Real Gcov_coord[4]; Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); - for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { + for (int mu = 0; mu <= 3; mu++) { // detg included above - Kokkos::atomic_add(&(v(b, mu, k, j, i)), -Gcov_coord[mu - iv::Gcov(0)]); + Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), + -Gcov_coord[iv::Gcov(mu) - iv::Gcov(0)]); } v(b, iv::GcovHeat(), k, j, i) = v(b, p::density(), k, j, i) * density_conversion_factor * heat; @@ -319,9 +320,9 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); Real detG = geom.DetG(CellLocation::Cent, b, k, j, i); - for (int mu = iv::Gcov(0); mu <= iv::Gcov(3); mu++) { - Kokkos::atomic_add(&(v(b, mu, k, j, i)), - -detG * Gcov_coord[mu - iv::Gcov(0)]); + for (int mu = 0; mu <= 3; mu++) { + Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), + -detG * Gcov_coord[iv::Gcov(mu) - iv::Gcov(0)]); } Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), -LeptonSign(s) * detG * Jye); From 2520b982884c1b8614d3fe70f4ac138b968fd9f2 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 12 Jan 2024 16:31:42 -0500 Subject: [PATCH 21/26] GetFourVelocity changed to get Mesh as well as MeshBlock --- src/phoebus_driver.cpp | 14 +++++++------- src/phoebus_utils/relativity_utils.hpp | 22 +++++++++++++++------- src/radiation/cooling_function.cpp | 5 ++--- 3 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 9b2984b8..4765faea 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -719,7 +719,7 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { tl.AddTask(start_gain_reducer, &parthenon::AllReduce::CheckReduce, pdo_gain_reducer); int reg_dep_id = 0; - sync_region.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer); + sync_region_lb.AddRegionalDependencies(reg_dep_id++, ib, finish_gain_reducer); } auto calculate_four_force = tl.AddTask(finish_gain_reducer, radiation::CoolingFunctionCalculateFourForce, @@ -977,12 +977,12 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { // Finalization calls { - TaskCollection tc; - TaskRegion &async_region1 = tc.AddRegion(num_task_lists_executed_independently); - for (int ib = 0; ib < num_task_lists_executed_independently; ib++) { - auto pmb = blocks[ib].get(); - auto &tl = async_region1[ib]; - auto &sc0 = pmb->meshblock_data.Get(stage_name[integrator->nstages]); + const int num_partitions = pmesh->DefaultNumPartitions(); + TaskRegion &sync_region = tc.AddRegion(num_partitions); + for (int ib = 0; ib < num_partitions; ++ib) { + auto &base = pmesh->mesh_data.GetOrAdd("base", ib); + auto &sc0 = pmesh->mesh_data.GetOrAdd("base", ib); + auto &tl = sync_region_lb[ib]; auto apply_four_force = tl.AddTask(none, radiation::ApplyRadiationFourForce, sc0.get(), dt); } diff --git a/src/phoebus_utils/relativity_utils.hpp b/src/phoebus_utils/relativity_utils.hpp index b73db829..ff184f04 100644 --- a/src/phoebus_utils/relativity_utils.hpp +++ b/src/phoebus_utils/relativity_utils.hpp @@ -90,20 +90,28 @@ KOKKOS_INLINE_FUNCTION Real GetLorentzFactor(const Real vcon[Geometry::NDSPACE], * PARAM[IN] - i - X1 index of meshblock cell * PARAM[OUT] - u - Coordinate frame contravariant four-velocity */ -KOKKOS_INLINE_FUNCTION void GetFourVelocity(const Real v[3], - const Geometry::CoordSysMeshBlock &system, - CellLocation loc, const int k, const int j, - const int i, Real u[Geometry::NDFULL]) { +template +KOKKOS_INLINE_FUNCTION void +GetFourVelocity(const Real v[3], const CoordinateSystem_t &system, CellLocation loc, + const int b, const int k, const int j, const int i, + Real u[Geometry::NDFULL]) { Real beta[Geometry::NDSPACE]; - Real W = GetLorentzFactor(v, system, loc, k, j, i); - Real alpha = system.Lapse(loc, k, j, i); - system.ContravariantShift(loc, k, j, i, beta); + Real W = GetLorentzFactor(v, system, loc, b, k, j, i); + Real alpha = system.Lapse(loc, b, k, j, i); + system.ContravariantShift(loc, b, k, j, i, beta); u[0] = robust::ratio(W, std::abs(alpha)); for (int l = 1; l < Geometry::NDFULL; ++l) { u[l] = v[l - 1] - u[0] * beta[l - 1]; } } +template +KOKKOS_INLINE_FUNCTION void +GetFourVelocity(const Real v[3], const CoordinateSystem_t &system, CellLocation loc, + const int k, const int j, const int i, Real u[Geometry::NDFULL]) { + GetFourVelocity(v, system, loc, 0, k, j, i, u); +} + /* * Compute square of magnetic field from primitive velocity and magnetic field * diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index d307e9e0..249c63fe 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -277,8 +277,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); for (int mu = 0; mu <= 3; mu++) { // detg included above - Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), - -Gcov_coord[iv::Gcov(mu) - iv::Gcov(0)]); + Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), -Gcov_coord[iv::Gcov(mu)]); } v(b, iv::GcovHeat(), k, j, i) = v(b, p::density(), k, j, i) * density_conversion_factor * heat; @@ -322,7 +321,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt for (int mu = 0; mu <= 3; mu++) { Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), - -detG * Gcov_coord[iv::Gcov(mu) - iv::Gcov(0)]); + -detG * Gcov_coord[iv::Gcov(mu)]); } Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), -LeptonSign(s) * detG * Jye); From c480616e0ee11bb9c8f9e069d382ed92067ce400 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 12 Jan 2024 16:47:07 -0500 Subject: [PATCH 22/26] GetLorentzFactor modified to take both Mesh and Meshblock --- src/phoebus_driver.cpp | 2 +- src/phoebus_utils/relativity_utils.hpp | 16 ++++++++++++---- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index 4765faea..efb67a3e 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -982,7 +982,7 @@ TaskListStatus PhoebusDriver::MonteCarloStep() { for (int ib = 0; ib < num_partitions; ++ib) { auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc0 = pmesh->mesh_data.GetOrAdd("base", ib); - auto &tl = sync_region_lb[ib]; + auto &tl = sync_region[ib]; auto apply_four_force = tl.AddTask(none, radiation::ApplyRadiationFourForce, sc0.get(), dt); } diff --git a/src/phoebus_utils/relativity_utils.hpp b/src/phoebus_utils/relativity_utils.hpp index ff184f04..a5181081 100644 --- a/src/phoebus_utils/relativity_utils.hpp +++ b/src/phoebus_utils/relativity_utils.hpp @@ -70,14 +70,22 @@ GetLorentzFactor(const Real vcon[Geometry::NDSPACE], * * RETURN - Lorentz factor in normal observer frame */ +template KOKKOS_INLINE_FUNCTION Real GetLorentzFactor(const Real vcon[Geometry::NDSPACE], - const Geometry::CoordSysMeshBlock &system, - CellLocation loc, const int k, const int j, - const int i) { + const CoordinateSystem_t &system, + CellLocation loc, const int b, const int k, + const int j, const int i) { Real gamma[Geometry::NDSPACE][Geometry::NDSPACE]; - system.Metric(loc, k, j, i, gamma); + system.Metric(loc, b, k, j, i, gamma); return GetLorentzFactor(vcon, gamma); } +template +KOKKOS_INLINE_FUNCTION Real GetLorentzFactor(const Real vcon[Geometry::NDSPACE], + const CoordinateSystem_t &system, + CellLocation loc, const int k, const int j, + const int i) { + return GetLorentzFactor(vcon, system, loc, 0, k, j, i); +} /* * Calculate the coordinate frame four-velocity from the normal observer three-velocity From d4e01e344e6effff0b6c5da424131c724e282529 Mon Sep 17 00:00:00 2001 From: mari2895 Date: Fri, 12 Jan 2024 16:55:23 -0500 Subject: [PATCH 23/26] typo --- src/radiation/cooling_function.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/radiation/cooling_function.cpp b/src/radiation/cooling_function.cpp index 249c63fe..b6171ec6 100644 --- a/src/radiation/cooling_function.cpp +++ b/src/radiation/cooling_function.cpp @@ -277,7 +277,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt Tetrads.TetradToCoordCov(Gcov_tetrad, Gcov_coord); for (int mu = 0; mu <= 3; mu++) { // detg included above - Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), -Gcov_coord[iv::Gcov(mu)]); + Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), -Gcov_coord[mu]); } v(b, iv::GcovHeat(), k, j, i) = v(b, p::density(), k, j, i) * density_conversion_factor * heat; @@ -321,7 +321,7 @@ TaskStatus CoolingFunctionCalculateFourForce(MeshData *rc, const double dt for (int mu = 0; mu <= 3; mu++) { Kokkos::atomic_add(&(v(b, iv::Gcov(mu), k, j, i)), - -detG * Gcov_coord[iv::Gcov(mu)]); + -detG * Gcov_coord[mu]); } Kokkos::atomic_add(&(v(b, iv::Gye(), k, j, i)), -LeptonSign(s) * detG * Jye); From ed2f05eb0c62f0e1e3ba8df88a45cd8a89ae2473 Mon Sep 17 00:00:00 2001 From: Mariam Gogilashvili <83436586+mari2895@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:07:26 -0700 Subject: [PATCH 24/26] Update phoebus_driver.cpp --- src/phoebus_driver.cpp | 1 - 1 file changed, 1 deletion(-) diff --git a/src/phoebus_driver.cpp b/src/phoebus_driver.cpp index efb67a3e..9008c61e 100644 --- a/src/phoebus_driver.cpp +++ b/src/phoebus_driver.cpp @@ -702,7 +702,6 @@ TaskListStatus PhoebusDriver::RadiationPostStep() { const int num_partitions = pmesh->DefaultNumPartitions(); TaskRegion &sync_region_lb = tc.AddRegion(num_partitions); for (int ib = 0; ib < num_partitions; ++ib) { - auto &base = pmesh->mesh_data.GetOrAdd("base", ib); auto &sc0 = pmesh->mesh_data.GetOrAdd("base", ib); auto &tl = sync_region_lb[ib]; auto finish_gain_reducer = none; From a628465e918da84c7deb2e74d233a21bd1ae89a3 Mon Sep 17 00:00:00 2001 From: Mariam Gogilashvili <83436586+mari2895@users.noreply.github.com> Date: Wed, 17 Jan 2024 14:07:38 -0700 Subject: [PATCH 25/26] Update src/radiation/radiation.cpp Co-authored-by: Jonah Miller --- src/radiation/radiation.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index f187bd40..51a1036c 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -567,7 +567,9 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(1), k, j, i) * dt; v(b, c::momentum(1), k, j, i) += v(b, iv::Gcov(2), k, j, i) * dt; v(b, c::momentum(2), k, j, i) += v(b, iv::Gcov(3), k, j, i) * dt; - v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; + if (v.Contains(b, c::ye())) { + v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; + } }); return TaskStatus::complete; } From 867a3028393401ade8541c0a2610debf1558466c Mon Sep 17 00:00:00 2001 From: mari2895 Date: Wed, 17 Jan 2024 16:15:56 -0500 Subject: [PATCH 26/26] formating --- src/radiation/radiation.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/radiation/radiation.cpp b/src/radiation/radiation.cpp index 51a1036c..cb0cb633 100644 --- a/src/radiation/radiation.cpp +++ b/src/radiation/radiation.cpp @@ -567,9 +567,9 @@ TaskStatus ApplyRadiationFourForce(MeshData *rc, const double dt) { v(b, c::momentum(0), k, j, i) += v(b, iv::Gcov(1), k, j, i) * dt; v(b, c::momentum(1), k, j, i) += v(b, iv::Gcov(2), k, j, i) * dt; v(b, c::momentum(2), k, j, i) += v(b, iv::Gcov(3), k, j, i) * dt; - if (v.Contains(b, c::ye())) { - v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; - } + if (v.Contains(b, c::ye())) { + v(b, c::ye(), k, j, i) += v(b, iv::Gye(), k, j, i) * dt; + } }); return TaskStatus::complete; }