From ef535be5faf3b9de5fe860f03a68c0d0c6e76aed Mon Sep 17 00:00:00 2001 From: AlexanderSinn Date: Tue, 2 May 2023 16:46:26 +0200 Subject: [PATCH 1/9] More SoA Fixes --- Src/Particle/AMReX_ParIter.H | 6 +-- Src/Particle/AMReX_ParticleContainerI.H | 5 +- Src/Particle/AMReX_ParticleTile.H | 65 +++++++++++++++---------- 3 files changed, 44 insertions(+), 32 deletions(-) diff --git a/Src/Particle/AMReX_ParIter.H b/Src/Particle/AMReX_ParIter.H index 1e8d3782c89..2b932ff01c2 100644 --- a/Src/Particle/AMReX_ParIter.H +++ b/Src/Particle/AMReX_ParIter.H @@ -88,11 +88,11 @@ public: [[nodiscard]] SoARef GetStructOfArrays () const { return GetParticleTile().GetStructOfArrays(); } - [[nodiscard]] int numParticles () const { return GetArrayOfStructs().numParticles(); } + [[nodiscard]] int numParticles () const { return GetParticleTile().numParticles(); } - [[nodiscard]] int numRealParticles () const { return GetArrayOfStructs().numRealParticles(); } + [[nodiscard]] int numRealParticles () const { return GetParticleTile().numRealParticles(); } - [[nodiscard]] int numNeighborParticles () const { return GetArrayOfStructs().numNeighborParticles(); } + [[nodiscard]] int numNeighborParticles () const { return GetParticleTile().numNeighborParticles(); } [[nodiscard]] int GetLevel () const { return m_level; } diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 698cb10aeaa..65fd686d5a1 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1090,9 +1090,8 @@ ParticleContainer_impl ::ReorderParticles (int lev, const MFIter& mfi, const index_type* permutations) { auto& ptile = ParticlesAt(lev, mfi); - auto& aos = ptile.GetArrayOfStructs(); - const size_t np = aos.numParticles(); - const size_t np_total = np + aos.numNeighborParticles(); + const size_t np = ptile.numParticles(); + const size_t np_total = np + ptile.numNeighborParticles(); if (memEfficientSort) { if constexpr(!ParticleType::is_soa_particle) { diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index 0cfedd1c17a..80820ec6fd3 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -109,25 +109,13 @@ struct ParticleTileData } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto* rdata (const int attribute_index) + auto * rdata (const int attribute_index) const { return this->m_rdata[attribute_index]; } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const * rdata (const int attribute_index) const - { - return this->m_rdata[attribute_index]; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto* idata (const int attribute_index) - { - return this->m_idata[attribute_index]; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const * idata (const int attribute_index) const + auto * idata (const int attribute_index) const { return this->m_idata[attribute_index]; } @@ -599,6 +587,12 @@ struct ConstParticleTileData } }; +struct ThisParticleTileHasNoParticleVector {}; + +struct ThisParticleTileHasNoAoS { + using ParticleVector = ThisParticleTileHasNoParticleVector; +}; + template class Allocator=DefaultAllocator> struct ParticleTile @@ -616,7 +610,10 @@ struct ParticleTile using SuperParticleType = Particle; - using AoS = ArrayOfStructs; + using AoS = typename std::conditional< + ParticleType::is_soa_particle, + ThisParticleTileHasNoAoS, + ArrayOfStructs>::type; //using ParticleVector = typename AoS::ParticleVector; using SoA = StructOfArrays; @@ -645,14 +642,14 @@ struct ParticleTile // AoS template ::type = 0> - ParticleCPUWrapper cpu (int index) & { + int& cpu (int index) & { ParticleType p(this->getParticleTileData(), index); return p.cpu(); } // const template ::type = 0> - ConstParticleCPUWrapper cpu (int index) const & { + int cpu (int index) const & { using ConstParticleType = typename ParticleType::ConstType; ConstParticleType p(this->getConstParticleTileData(), index); return p.cpu(); @@ -677,14 +674,14 @@ struct ParticleTile // AoS template ::type = 0> - ParticleIDWrapper id (int index) & { + int& id (int index) & { ParticleType p(this->getParticleTileData(), index); return p.id(); } // const template ::type = 0> - ConstParticleIDWrapper id (int index) const & { + int id (int index) const & { using ConstParticleType = typename ParticleType::ConstType; ConstParticleType p(this->getConstParticleTileData(), index); return p.id(); @@ -746,7 +743,7 @@ struct ParticleTile SoA& GetStructOfArrays () { return m_soa_tile; } const SoA& GetStructOfArrays () const { return m_soa_tile; } - bool empty () const { return m_aos_tile.empty(); } + bool empty () const { return size() == 0; } /** * \brief Returns the total number of particles (real and neighbor) @@ -813,7 +810,9 @@ struct ParticleTile void resize (std::size_t count) { - m_aos_tile.resize(count); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile.resize(count); + } m_soa_tile.resize(count); } @@ -965,7 +964,9 @@ struct ParticleTile void shrink_to_fit () { - m_aos_tile().shrink_to_fit(); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile().shrink_to_fit(); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -982,7 +983,9 @@ struct ParticleTile Long capacity () const { Long nbytes = 0; - nbytes += m_aos_tile().capacity() * sizeof(ParticleType); + if constexpr (!ParticleType::is_soa_particle) { + nbytes += m_aos_tile().capacity() * sizeof(ParticleType); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -999,7 +1002,9 @@ struct ParticleTile void swap (ParticleTile& other) { - m_aos_tile().swap(other.GetArrayOfStructs()()); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile().swap(other.GetArrayOfStructs()()); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -1048,7 +1053,11 @@ struct ParticleTile #endif ParticleTileDataType ptd; - ptd.m_aos = m_aos_tile().dataPtr(); + if constexpr (!ParticleType::is_soa_particle) { + ptd.m_aos = m_aos_tile().dataPtr(); + } else { + ptd.m_aos = nullptr; + } if constexpr(NArrayReal > 0) for (int i = 0; i < NArrayReal; ++i) ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); @@ -1105,7 +1114,11 @@ struct ParticleTile #endif ConstParticleTileDataType ptd; - ptd.m_aos = m_aos_tile().dataPtr(); + if constexpr (!ParticleType::is_soa_particle) { + ptd.m_aos = m_aos_tile().dataPtr(); + } else { + ptd.m_aos = nullptr; + } if constexpr(NArrayReal > 0) for (int i = 0; i < NArrayReal; ++i) ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); From 45e458ee8b8102740e48a611748e24ca96e2a8a8 Mon Sep 17 00:00:00 2001 From: AlexanderSinn Date: Tue, 2 May 2023 16:46:26 +0200 Subject: [PATCH 2/9] More PureSoA FixesMore SoA Fixes --- Src/Particle/AMReX_ParticleContainerI.H | 5 +- Src/Particle/AMReX_ParticleTile.H | 65 +++++++++++++++---------- 2 files changed, 41 insertions(+), 29 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index b40f4a91873..000b8d41fb0 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1090,9 +1090,8 @@ ParticleContainer_impl ::ReorderParticles (int lev, const MFIter& mfi, const index_type* permutations) { auto& ptile = ParticlesAt(lev, mfi); - auto& aos = ptile.GetArrayOfStructs(); - const size_t np = aos.numParticles(); - const size_t np_total = np + aos.numNeighborParticles(); + const size_t np = ptile.numParticles(); + const size_t np_total = np + ptile.numNeighborParticles(); if (memEfficientSort) { if constexpr(!ParticleType::is_soa_particle) { diff --git a/Src/Particle/AMReX_ParticleTile.H b/Src/Particle/AMReX_ParticleTile.H index b11bca9a2bc..441e61c98fa 100644 --- a/Src/Particle/AMReX_ParticleTile.H +++ b/Src/Particle/AMReX_ParticleTile.H @@ -109,25 +109,13 @@ struct ParticleTileData } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto* rdata (const int attribute_index) + auto * rdata (const int attribute_index) const { return this->m_rdata[attribute_index]; } AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const * rdata (const int attribute_index) const - { - return this->m_rdata[attribute_index]; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto* idata (const int attribute_index) - { - return this->m_idata[attribute_index]; - } - - AMREX_GPU_HOST_DEVICE AMREX_FORCE_INLINE - auto const * idata (const int attribute_index) const + auto * idata (const int attribute_index) const { return this->m_idata[attribute_index]; } @@ -599,6 +587,12 @@ struct ConstParticleTileData } }; +struct ThisParticleTileHasNoParticleVector {}; + +struct ThisParticleTileHasNoAoS { + using ParticleVector = ThisParticleTileHasNoParticleVector; +}; + template class Allocator=DefaultAllocator> struct ParticleTile @@ -616,7 +610,10 @@ struct ParticleTile using SuperParticleType = Particle; - using AoS = ArrayOfStructs; + using AoS = typename std::conditional< + ParticleType::is_soa_particle, + ThisParticleTileHasNoAoS, + ArrayOfStructs>::type; //using ParticleVector = typename AoS::ParticleVector; using SoA = StructOfArrays; @@ -645,14 +642,14 @@ struct ParticleTile // AoS template ::type = 0> - ParticleCPUWrapper cpu (int index) & { + int& cpu (int index) & { ParticleType p(this->getParticleTileData(), index); return p.cpu(); } // const template ::type = 0> - ConstParticleCPUWrapper cpu (int index) const & { + int cpu (int index) const & { using ConstParticleType = typename ParticleType::ConstType; ConstParticleType p(this->getConstParticleTileData(), index); return p.cpu(); @@ -677,14 +674,14 @@ struct ParticleTile // AoS template ::type = 0> - ParticleIDWrapper id (int index) & { + int& id (int index) & { ParticleType p(this->getParticleTileData(), index); return p.id(); } // const template ::type = 0> - ConstParticleIDWrapper id (int index) const & { + int id (int index) const & { using ConstParticleType = typename ParticleType::ConstType; ConstParticleType p(this->getConstParticleTileData(), index); return p.id(); @@ -747,7 +744,7 @@ struct ParticleTile const SoA& GetStructOfArrays () const { return m_soa_tile; } template ::type = 0> - bool empty () const { return m_aos_tile.empty(); } + bool empty () const { return size() == 0; } template ::type = 0> bool empty () const { return m_soa_tile.empty(); } @@ -817,7 +814,9 @@ struct ParticleTile void resize (std::size_t count) { - m_aos_tile.resize(count); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile.resize(count); + } m_soa_tile.resize(count); } @@ -969,7 +968,9 @@ struct ParticleTile void shrink_to_fit () { - m_aos_tile().shrink_to_fit(); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile().shrink_to_fit(); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -986,7 +987,9 @@ struct ParticleTile Long capacity () const { Long nbytes = 0; - nbytes += m_aos_tile().capacity() * sizeof(ParticleType); + if constexpr (!ParticleType::is_soa_particle) { + nbytes += m_aos_tile().capacity() * sizeof(ParticleType); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -1003,7 +1006,9 @@ struct ParticleTile void swap (ParticleTile& other) { - m_aos_tile().swap(other.GetArrayOfStructs()()); + if constexpr (!ParticleType::is_soa_particle) { + m_aos_tile().swap(other.GetArrayOfStructs()()); + } for (int j = 0; j < NumRealComps(); ++j) { auto& rdata = GetStructOfArrays().GetRealData(j); @@ -1052,7 +1057,11 @@ struct ParticleTile #endif ParticleTileDataType ptd; - ptd.m_aos = m_aos_tile().dataPtr(); + if constexpr (!ParticleType::is_soa_particle) { + ptd.m_aos = m_aos_tile().dataPtr(); + } else { + ptd.m_aos = nullptr; + } if constexpr(NArrayReal > 0) for (int i = 0; i < NArrayReal; ++i) ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); @@ -1109,7 +1118,11 @@ struct ParticleTile #endif ConstParticleTileDataType ptd; - ptd.m_aos = m_aos_tile().dataPtr(); + if constexpr (!ParticleType::is_soa_particle) { + ptd.m_aos = m_aos_tile().dataPtr(); + } else { + ptd.m_aos = nullptr; + } if constexpr(NArrayReal > 0) for (int i = 0; i < NArrayReal; ++i) ptd.m_rdata[i] = m_soa_tile.GetRealData(i).dataPtr(); From 69215fcac3848c1d364729c4491668712a3e95f5 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 May 2023 17:38:17 -0700 Subject: [PATCH 3/9] [FIXME] Comment Unusued correctCellVectors Hook --- Src/Particle/AMReX_ParticleContainerI.H | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 000b8d41fb0..04e057231bf 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1650,7 +1650,7 @@ ParticleContainer_impl soa.GetRealData(comp)[pindex] = soa.GetRealData(comp)[last]; for (int comp = 0; comp < NumIntComps(); comp++) soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last]; - correctCellVectors(last, pindex, grid, aos[pindex]); + //correctCellVectors(last, pindex, grid, aos[pindex]); // FIXME: TODO --last; continue; } @@ -1664,7 +1664,7 @@ ParticleContainer_impl soa.GetRealData(comp)[pindex] = soa.GetRealData(comp)[last]; for (int comp = 0; comp < NumIntComps(); comp++) soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last]; - correctCellVectors(last, pindex, grid, aos[pindex]); + //correctCellVectors(last, pindex, grid, aos[pindex]); // FIXME: TODO --last; continue; } @@ -1716,7 +1716,7 @@ ParticleContainer_impl soa.GetRealData(comp)[pindex] = soa.GetRealData(comp)[last]; for (int comp = 0; comp < NumIntComps(); comp++) soa.GetIntData(comp)[pindex] = soa.GetIntData(comp)[last]; - correctCellVectors(last, pindex, grid, aos[pindex]); + //correctCellVectors(last, pindex, grid, aos[pindex]); // FIXME: TODO --last; continue; } From d330a95172dba619ef52b1161a21052598fa0373 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Mon, 8 May 2023 22:38:25 -0700 Subject: [PATCH 4/9] test: avoid AoS access --- Tests/Particles/RedistributeSOA/main.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Tests/Particles/RedistributeSOA/main.cpp b/Tests/Particles/RedistributeSOA/main.cpp index 0bf11f6f11c..34d242e6d42 100644 --- a/Tests/Particles/RedistributeSOA/main.cpp +++ b/Tests/Particles/RedistributeSOA/main.cpp @@ -128,7 +128,7 @@ class TestParticleContainer } auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + host_real[0].size(); particle_tile.resize(new_size); From e952a7a86a2cea05e949330bb5620e518b9e0fee Mon Sep 17 00:00:00 2001 From: AlexanderSinn <64009254+AlexanderSinn@users.noreply.github.com> Date: Wed, 20 Sep 2023 17:43:58 +0200 Subject: [PATCH 5/9] fix correctCellVectors for PureSoA --- Src/Particle/AMReX_ParticleContainerI.H | 8 ++++---- Src/Particle/AMReX_ParticleInit.H | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 0b8000f6880..15b836119e8 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1653,8 +1653,8 @@ ParticleContainer_implgetParticleTileData(); while (pindex <= last) { - auto ptd = particle_tile->getParticleTileData(); ParticleType p(ptd,pindex); if ((remove_negative == false) && (p.id() < 0)) { @@ -1669,7 +1669,7 @@ ParticleContainer_impl Date: Wed, 20 Sep 2023 17:50:28 +0200 Subject: [PATCH 6/9] fix issue --- Src/Particle/AMReX_ParticleContainerI.H | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 15b836119e8..72df9356a11 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -1685,7 +1685,7 @@ ParticleContainer_impl Date: Wed, 20 Sep 2023 18:13:31 +0200 Subject: [PATCH 7/9] try fix SENSEI --- Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H | 16 ++++++++-------- Tests/Particles/SENSEI_Insitu_SOA/main.cpp | 2 +- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H index bb5a573c11d..e84844e3411 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H @@ -581,14 +581,14 @@ int ParticleDataAdaptor::GetMeshMetadata( for (const auto& kv : pmap) { // loop over particles in ParticleTile - const auto& aos = kv.second.GetArrayOfStructs(); - const auto &parts = aos(); + auto& particle_tile = kv.second; + auto ptd = particle_tile.getParticleTileData(); // visit only the "real" particles, skip the "neighbor" particles. - long long numReal = aos.numRealParticles(); + long long numReal = particle_tile.numRealParticles(); for (long long i = 0; i < numReal; ++i) { - const auto &part = parts[i]; + const auto &part = ptd[i]; if (part.id() > 0) { #if (AMREX_SPACEDIM == 1) @@ -869,7 +869,7 @@ int ParticleDataAdaptor::AddParticlesSOARea for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order @@ -881,7 +881,7 @@ int ParticleDataAdaptor::AddParticlesSOARea for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = ptd[i]; if (part.id() > 0) { pData[i*nComps + j] = realData[i]; @@ -944,7 +944,7 @@ int ParticleDataAdaptor::AddParticlesSOAInt for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto& aos = pti.GetArrayOfStructs(); + auto ptd = pti.getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order @@ -953,7 +953,7 @@ int ParticleDataAdaptor::AddParticlesSOAInt for (long long i = 0; i < numReal; ++i) { - const auto &part = aos[i]; + const auto &part = ptd[i]; if (part.id() > 0) { pData[i] = intData[i]; diff --git a/Tests/Particles/SENSEI_Insitu_SOA/main.cpp b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp index 8f9e8f7f7e6..f2dd2706df5 100644 --- a/Tests/Particles/SENSEI_Insitu_SOA/main.cpp +++ b/Tests/Particles/SENSEI_Insitu_SOA/main.cpp @@ -115,7 +115,7 @@ class TestParticleContainer } auto& particle_tile = DefineAndReturnParticleTile(lev, mfi.index(), mfi.LocalTileIndex()); - auto old_size = particle_tile.GetArrayOfStructs().size(); + auto old_size = particle_tile.size(); auto new_size = old_size + host_real[0].size(); particle_tile.resize(new_size); From 072cc461325af6ed4d05e35f165c97724511d5b2 Mon Sep 17 00:00:00 2001 From: AlexanderSinn <64009254+AlexanderSinn@users.noreply.github.com> Date: Wed, 20 Sep 2023 18:25:27 +0200 Subject: [PATCH 8/9] fix SENSEI --- Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H index e84844e3411..262089694ab 100644 --- a/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H +++ b/Src/Extern/SENSEI/AMReX_ParticleDataAdaptorI.H @@ -582,7 +582,7 @@ int ParticleDataAdaptor::GetMeshMetadata( { // loop over particles in ParticleTile auto& particle_tile = kv.second; - auto ptd = particle_tile.getParticleTileData(); + auto ptd = particle_tile.getConstParticleTileData(); // visit only the "real" particles, skip the "neighbor" particles. long long numReal = particle_tile.numRealParticles(); @@ -869,7 +869,7 @@ int ParticleDataAdaptor::AddParticlesSOARea for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto ptd = pti.getParticleTileData(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order @@ -944,7 +944,7 @@ int ParticleDataAdaptor::AddParticlesSOAInt for (MyParIter pti(*this->m_particles, level); pti.isValid(); ++pti) { auto& particle_attributes = pti.GetStructOfArrays(); - auto ptd = pti.getParticleTileData(); + auto ptd = pti.GetParticleTile().getParticleTileData(); auto numReal = pti.numParticles(); // shuffle from the AMReX component order From cfdb37f3b63833d1cf45bd6e6c2979164c7d12b0 Mon Sep 17 00:00:00 2001 From: Axel Huebl Date: Fri, 22 Sep 2023 12:32:08 +0200 Subject: [PATCH 9/9] Fix ParticleContainerI --- Src/Particle/AMReX_ParticleContainerI.H | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/Src/Particle/AMReX_ParticleContainerI.H b/Src/Particle/AMReX_ParticleContainerI.H index 72df9356a11..b845d130d5c 100644 --- a/Src/Particle/AMReX_ParticleContainerI.H +++ b/Src/Particle/AMReX_ParticleContainerI.H @@ -2206,13 +2206,15 @@ RedistributeMPI (std::map >& not_ours, const auto& src_tile = kv.second; auto& dst_tile = GetParticles(host_lev)[std::make_pair(grid,tile)]; - auto old_size = dst_tile.GetArrayOfStructs().size(); + auto old_size = dst_tile.size(); auto new_size = old_size + src_tile.size(); dst_tile.resize(new_size); - Gpu::copyAsync(Gpu::hostToDevice, - src_tile.begin(), src_tile.end(), - dst_tile.GetArrayOfStructs().begin() + old_size); + if constexpr(! ParticleType::is_soa_particle) { + Gpu::copyAsync(Gpu::hostToDevice, + src_tile.begin(), src_tile.end(), + dst_tile.GetArrayOfStructs().begin() + old_size); + } for (int i = 0; i < NumRealComps(); ++i) { Gpu::copyAsync(Gpu::hostToDevice,