diff --git a/Src/Extern/openPMD-api/AMReX_ParticlesOPENPMD.H b/Src/Extern/openPMD-api/AMReX_ParticlesOPENPMD.H index ba623a4ebb0..37d28e32ce1 100644 --- a/Src/Extern/openPMD-api/AMReX_ParticlesOPENPMD.H +++ b/Src/Extern/openPMD-api/AMReX_ParticlesOPENPMD.H @@ -2,19 +2,19 @@ #define AMREX_PTL_OPENPMD_API_H //#include - + struct AMReX_PtlCounter { int m_MPIRank = 0; int m_MPISize = 1; - + unsigned long long m_Total = 0; - + std::vector m_ParticleCounterByLevel; unsigned long GetTotalNumParticles () const { return m_Total;} - + std::vector m_ParticleOffsetAtRank; std::vector m_ParticleSizeAtRank; }; @@ -31,8 +31,8 @@ void CountParticles() m_PtlCounter.m_ParticleSizeAtRank.resize(this->finestLevel()+1); auto lf_GetParticleOffsetOfProcessor = [&](const long& numParticles, - unsigned long long& offset, - unsigned long long& sum) -> void + unsigned long long& offset, + unsigned long long& sum) -> void { std::vector result(m_PtlCounter.m_MPISize, 0); amrex::ParallelGather::Gather (numParticles, result.data(), -1, amrex::ParallelDescriptor::Communicator()); @@ -42,7 +42,7 @@ void CountParticles() for (int i=0; i= 2u ) - { - std::string const varname_1st = varname.substr(0u, 1u); // 1st character - std::string const varname_2nd = varname.substr(1u, 1u); // 2nd character - - // Check if this field is a vector. If so, then extract the field name - - std::vector< std::string > const vector_fields = {"E", "B", "j"}; - std::vector< std::string > const field_components = getFieldComponentLabels(); - - for( std::string const& vector_field : vector_fields ) - { - for( std::string const& component : field_components ) - { - if( vector_field.compare( varname_1st ) == 0 && component.compare( varname_2nd ) == 0 ) - { - m_FieldName = varname_1st + varname.substr(2); // Strip component - m_CompName = varname_2nd; - } - } - } - } - - + { + std::string const varname_1st = varname.substr(0u, 1u); // 1st character + std::string const varname_2nd = varname.substr(1u, 1u); // 2nd character + + // Check if this field is a vector. If so, then extract the field name + + std::vector< std::string > const vector_fields = {"E", "B", "j"}; + std::vector< std::string > const field_components = getFieldComponentLabels(); + + for( std::string const& vector_field : vector_fields ) + { + for( std::string const& component : field_components ) + { + if( vector_field.compare( varname_1st ) == 0 && component.compare( varname_2nd ) == 0 ) + { + m_FieldName = varname_1st + varname.substr(2); // Strip component + m_CompName = varname_2nd; + } + } + } + } + + if ( 0 == meshLevel ) - return; - + return; + m_FieldName += std::string("_lvl").append(std::to_string(meshLevel)); } - + // create json options to pass to openpmd inline std::string @@ -116,7 +116,7 @@ namespace amrex { std::map< std::string, std::string > const & engine_parameters) { if (operator_type.empty() && engine_type.empty()) - return "{}"; + return "{}"; std::string options; std::string top_block; @@ -126,74 +126,74 @@ namespace amrex { std::string op_parameters; for (const auto& kv : operator_parameters) { - if (!op_parameters.empty()) op_parameters.append(",\n"); - op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ - .append("\"").append(kv.first).append("\": ") /* key */ - .append("\"").append(kv.second).append("\""); /* value (as string) */ + if (!op_parameters.empty()) op_parameters.append(",\n"); + op_parameters.append(std::string(12, ' ')) /* just pretty alignment */ + .append("\"").append(kv.first).append("\": ") /* key */ + .append("\"").append(kv.second).append("\""); /* value (as string) */ } std::string en_parameters; for (const auto& kv : engine_parameters) { - if (!en_parameters.empty()) en_parameters.append(",\n"); - en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ - .append("\"").append(kv.first).append("\": ") /* key */ - .append("\"").append(kv.second).append("\""); /* value (as string) */ + if (!en_parameters.empty()) en_parameters.append(",\n"); + en_parameters.append(std::string(12, ' ')) /* just pretty alignment */ + .append("\"").append(kv.first).append("\": ") /* key */ + .append("\"").append(kv.second).append("\""); /* value (as string) */ } - top_block = R"END( + top_block = R"END( { "adios2": {)END"; end_block = R"END( } })END"; - + if (!operator_type.empty()) { - op_block = R"END( - "dataset": { + op_block = R"END( + "dataset": { "operators": [ { "type": ")END"; - op_block += operator_type + "\""; - if (!op_parameters.empty()) { - op_block += R"END(, + op_block += operator_type + "\""; + if (!op_parameters.empty()) { + op_block += R"END(, "parameters": { -)END"; - op_block += op_parameters + - "\n }"; - } - op_block += R"END( - } - ] +)END"; + op_block += op_parameters + + "\n }"; + } + op_block += R"END( + } + ] })END"; - - if (!engine_type.empty() || !en_parameters.empty()) - op_block += ","; + + if (!engine_type.empty() || !en_parameters.empty()) + op_block += ","; } - + if (!engine_type.empty() || !en_parameters.empty()) - { - en_block = R"END( - "engine": {)END"; - if (!engine_type.empty()) { - en_block += R"END( + { + en_block = R"END( + "engine": {)END"; + if (!engine_type.empty()) { + en_block += R"END( "type": ")END"; - en_block += engine_type + "\""; - if(!en_parameters.empty()) - en_block += ","; - } - if (!en_parameters.empty()) { - en_block += R"END( - "parameters": { + en_block += engine_type + "\""; + if(!en_parameters.empty()) + en_block += ","; + } + if (!en_parameters.empty()) { + en_block += R"END( + "parameters": { )END"; - en_block += en_parameters + - "\n }"; - } - en_block += R"END( + en_block += en_parameters + + "\n }"; + } + en_block += R"END( })END"; - } - + } + options = top_block + op_block + en_block + end_block; - return options; + return options; } @@ -222,15 +222,15 @@ namespace amrex { void AMReX_openPMDHandler::CreateWriter(const std::string& prefix) { ParmParse pp_prefix(prefix); - - // choose backend (e.g. ADIOS, ADIOS2 or HDF5). Default depends on openPMD-api configuration + + // choose backend (e.g. ADIOS, ADIOS2 or HDF5). Default depends on openPMD-api configuration std::string openpmd_backend {"default"}; pp_prefix.query("openpmd_backend", openpmd_backend); - - std::string openpmd_encoding {"f"}; + + std::string openpmd_encoding {"f"}; pp_prefix.query("openpmd_encoding", openpmd_encoding); openPMD::IterationEncoding encoding = openPMD::IterationEncoding::groupBased; - + if ( 0 == openpmd_encoding.compare("v") ) encoding = openPMD::IterationEncoding::variableBased; else if ( 0 == openpmd_encoding.compare("g") ) @@ -239,23 +239,23 @@ namespace amrex { encoding = openPMD::IterationEncoding::fileBased; auto lf_collect = [&](const char* key, - const std::string& parameter_tag, - std::string& key_type, - std::map< std::string, std::string>& result)->void + const std::string& parameter_tag, + std::string& key_type, + std::map< std::string, std::string>& result)->void { - //std::string key_type; - pp_prefix.query(key, key_type); - std::string const key_prefix = prefix + parameter_tag; - ParmParse pp; - auto entr = pp.getEntries(key_prefix); - - auto const prefix_len = key_prefix.size() + 1; - for (std::string k : entr) { - std::string v; - pp.get(k.c_str(), v); - k.erase(0, prefix_len); - result.insert({k, v}); - } + //std::string key_type; + pp_prefix.query(key, key_type); + std::string const key_prefix = prefix + parameter_tag; + ParmParse pp; + auto entr = pp.getEntries(key_prefix); + + auto const prefix_len = key_prefix.size() + 1; + for (std::string k : entr) { + std::string v; + pp.get(k.c_str(), v); + k.erase(0, prefix_len); + result.insert({k, v}); + } }; std::string operator_type; @@ -263,19 +263,19 @@ namespace amrex { lf_collect("adios2_operator.type", ".adios2_operator.parameters", operator_type, operator_parameters); std::string engine_type; - std::map< std::string, std::string > engine_parameters; - lf_collect("adios2_engine.type", ".adios2_engine.parameters", engine_type, engine_parameters); + std::map< std::string, std::string > engine_parameters; + lf_collect("adios2_engine.type", ".adios2_engine.parameters", engine_type, engine_parameters); std::string options=getSeriesOptions(operator_type, operator_parameters, - engine_type, engine_parameters); - + engine_type, engine_parameters); + m_Writer = std::make_unique(prefix, encoding, openpmd_backend, options); pp_prefix.query("file_min_digits", m_Writer->m_openPMDMinDigits); - + } // CreateWriter() - + //////////////////////////////////////// // // Classs AMReX_openPMDWriter @@ -285,9 +285,9 @@ namespace amrex { {} AMReX_openPMDWriter::AMReX_openPMDWriter (const std::string& prefix, - openPMD::IterationEncoding ie, - std::string filetype, - std::string options) + openPMD::IterationEncoding ie, + std::string filetype, + std::string options) :m_openPMDPrefix(prefix), m_openPMDEncoding(ie), m_openPMDFileType(filetype), @@ -325,7 +325,7 @@ if( m_openPMDFileType == "default" ) void AMReX_openPMDWriter::CloseStep(int ts) { if (m_Series) { - GetIteration(m_CurrentStep).close(); + GetIteration(m_CurrentStep).close(); } } @@ -340,129 +340,129 @@ if( m_openPMDFileType == "default" ) return; if (amrex::ParallelDescriptor::NProcs() > 1) - { + { #if defined(AMREX_USE_MPI) m_Series = std::make_unique( - filepath, access, - amrex::ParallelDescriptor::Communicator(), - m_openPMDSeriesOptions - ); + filepath, access, + amrex::ParallelDescriptor::Communicator(), + m_openPMDSeriesOptions + ); #else amrex::Abort(Utils::TextMsg::Err("AMReX did not build with MPI support!")); #endif - } + } else - { - m_Series = std::make_unique(filepath, access, m_openPMDSeriesOptions); - } - + { + m_Series = std::make_unique(filepath, access, m_openPMDSeriesOptions); + } + m_Series->setIterationEncoding( m_openPMDEncoding ); m_Series->setMeshesPath( "fields" ); // conform to ED-PIC extension of openPMD - + uint32_t const openPMD_ED_PIC = 1u; m_Series->setOpenPMDextension( openPMD_ED_PIC ); // meta info - - m_Series->setSoftware( "AMReX", amrex::Version() ); + + m_Series->setSoftware( "AMReX", amrex::Version() ); } - + void AMReX_openPMDWriter::CompSetup(int lev, - openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom, - const std::vector& varnames, - const amrex::MultiFab* curr_mf) const + openPMD::Container< openPMD::Mesh >& meshes, + amrex::Geometry& full_geom, + const std::vector& varnames, + const amrex::MultiFab* curr_mf) const { int const ncomp = curr_mf->nComp(); for ( int icomp=0; icomp& meshes, - amrex::Geometry& full_geom, - const std::vector& varnames, - const amrex::MultiFab* curr_mf) const + openPMD::Container< openPMD::Mesh >& meshes, + amrex::Geometry& full_geom, + const std::vector& varnames, + const amrex::MultiFab* curr_mf) const { int const ncomp = curr_mf->nComp(); amrex::Box const & global_box = full_geom.Domain(); for ( int icomp=0; icompisManaged() || fab.arena()->isDevice()) - { - amrex::BaseFab foo(local_box, 1, amrex::The_Pinned_Arena()); - std::shared_ptr data_pinned(foo.release()); - amrex::Gpu::dtoh_memcpy_async(data_pinned.get(), fab.dataPtr(icomp), local_box.numPts()*sizeof(amrex::Real)); - // intentionally delayed until before we .flush(): amrex::Gpu::streamSynchronize(); - mesh_comp.storeChunk(data_pinned, chunk_offset, chunk_size); - } - else + if (fab.arena()->isManaged() || fab.arena()->isDevice()) + { + amrex::BaseFab foo(local_box, 1, amrex::The_Pinned_Arena()); + std::shared_ptr data_pinned(foo.release()); + amrex::Gpu::dtoh_memcpy_async(data_pinned.get(), fab.dataPtr(icomp), local_box.numPts()*sizeof(amrex::Real)); + // intentionally delayed until before we .flush(): amrex::Gpu::streamSynchronize(); + mesh_comp.storeChunk(data_pinned, chunk_offset, chunk_size); + } + else #endif - { - amrex::Real const *local_data = fab.dataPtr(icomp); - mesh_comp.storeChunkRaw(local_data, chunk_offset, chunk_size); - } + { + amrex::Real const *local_data = fab.dataPtr(icomp); + mesh_comp.storeChunkRaw(local_data, chunk_offset, chunk_size); + } } - } // icomp store loop + } // icomp store loop } - + void AMReX_openPMDWriter::WriteMesh(const std::vector& varnames, - const amrex::Vector& mf, - const amrex::Vector& geom, - //const int iteration, - const double time ) const + const amrex::Vector& mf, + const amrex::Vector& geom, + //const int iteration, + const double time ) const { BL_PROFILE("AMReX_openPMDWriter::WriteMesh()"); @@ -471,31 +471,31 @@ if( m_openPMDFileType == "default" ) auto meshes = series_iteration.meshes; series_iteration.setTime( time ); - + if ( 0 == varnames.size() ) return; int output_levels = geom.size(); for (int lev=0; lev < output_levels; lev++) - { - amrex::Geometry full_geom = geom[lev]; - - if ( 0 == lev ) - SetupFields(meshes, full_geom); - - CompSetup(lev, meshes, full_geom, varnames, mf[lev]); - CompStorage(lev, meshes, full_geom, varnames, mf[lev]); + { + amrex::Geometry full_geom = geom[lev]; + + if ( 0 == lev ) + SetupFields(meshes, full_geom); + + CompSetup(lev, meshes, full_geom, varnames, mf[lev]); + CompStorage(lev, meshes, full_geom, varnames, mf[lev]); #ifdef AMREX_USE_GPU - amrex::Gpu::streamSynchronize(); + amrex::Gpu::streamSynchronize(); #endif - m_Series->flush(); - } // for lev loop + m_Series->flush(); + } // for lev loop } void AMReX_openPMDWriter::GetFileName(std::string& filepath) { if (filepath.size() == 0) - filepath.append("."); - + filepath.append("."); + filepath.append("/"); // transform paths for Windows #ifdef _WIN32 @@ -505,23 +505,23 @@ if( m_openPMDFileType == "default" ) std::string filename = "openpmd"; // // OpenPMD supports timestepped names - // + // if (m_openPMDEncoding == openPMD::IterationEncoding::fileBased) - { - std::string fileSuffix = std::string("_%0") + std::to_string(m_openPMDMinDigits) + std::string("T"); - filename = filename.append(fileSuffix); - } + { + std::string fileSuffix = std::string("_%0") + std::to_string(m_openPMDMinDigits) + std::string("T"); + filename = filename.append(fileSuffix); + } filename.append(".").append(m_openPMDFileType); filepath.append(filename); } - void AMReX_openPMDWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom) const + void AMReX_openPMDWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes, + amrex::Geometry& full_geom) const { //} // meta data for ED-PIC extension auto const period = full_geom.periodicity(); - + std::vector fieldBoundary(6, "reflecting"); std::vector particleBoundary(6, "absorbing"); fieldBoundary.resize(AMREX_SPACEDIM * 2); @@ -533,23 +533,23 @@ if( m_openPMDFileType == "default" ) #endif for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i) - if (period.isPeriodic(i)) { - fieldBoundary.at(2u * i) = "periodic"; - fieldBoundary.at(2u * i + 1u) = "periodic"; - particleBoundary.at(2u * i) = "periodic"; - particleBoundary.at(2u * i + 1u) = "periodic"; - } - + if (period.isPeriodic(i)) { + fieldBoundary.at(2u * i) = "periodic"; + fieldBoundary.at(2u * i + 1u) = "periodic"; + particleBoundary.at(2u * i) = "periodic"; + particleBoundary.at(2u * i + 1u) = "periodic"; + } + meshes.setAttribute("fieldBoundary", fieldBoundary); meshes.setAttribute("particleBoundary", particleBoundary); } - void AMReX_openPMDWriter::SetupMeshComp (openPMD::Mesh& mesh, - const amrex::Geometry& full_geom, - amrex::MultiFab const& mf, - const AMReX_VarNameParser& varName - ) const + void AMReX_openPMDWriter::SetupMeshComp (openPMD::Mesh& mesh, + const amrex::Geometry& full_geom, + amrex::MultiFab const& mf, + const AMReX_VarNameParser& varName + ) const { BL_PROFILE("SetupMeshComp(default)"); @@ -560,34 +560,34 @@ if( m_openPMDFileType == "default" ) // - Grid spacing std::vector const grid_spacing = helper::getReversedVec(full_geom.CellSize()); mesh.setGridSpacing(grid_spacing); - + // - Global offset std::vector const global_offset = helper::getReversedVec(full_geom.ProbLo()); mesh.setGridGlobalOffset(global_offset); - + // - AxisLabels std::vector axis_labels = varName.getFieldAxisLabels(); mesh.setAxisLabels(axis_labels); - + // Prepare the type of dataset that will be written openPMD::Datatype const datatype = openPMD::determineDatatype(); auto const dataset = openPMD::Dataset(datatype, global_size); mesh.setDataOrder(openPMD::Mesh::DataOrder::C); if (varName.m_ThetaMode) { - mesh.setGeometry("thetaMode"); + mesh.setGeometry("thetaMode"); } mesh.setAttribute("fieldSmoothing", "none"); mesh_comp.resetDataset(dataset); - + helper::setOpenPMDUnit( mesh, varName.m_FieldName ); - + auto relative_cell_pos = helper::getRelativeCellPosition(mf); // AMReX Fortran index order std::reverse( relative_cell_pos.begin(), relative_cell_pos.end() ); // now in C order - mesh_comp.setPosition( relative_cell_pos ); + mesh_comp.setPosition( relative_cell_pos ); } - + } // namespace openpmd_api } // namespace amrex diff --git a/Src/Extern/openPMD-api/AMReX_PlotFileOPENPMD_PTL.cpp b/Src/Extern/openPMD-api/AMReX_PlotFileOPENPMD_PTL.cpp index 3c4ac6ed0e9..e4f0c17ef98 100644 --- a/Src/Extern/openPMD-api/AMReX_PlotFileOPENPMD_PTL.cpp +++ b/Src/Extern/openPMD-api/AMReX_PlotFileOPENPMD_PTL.cpp @@ -19,27 +19,27 @@ namespace amrex { namespace openpmd_api { bool AMReX_openPMDWriter::AllocatePtlProperties(openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const { SetupPos(currSpecies, np); // Allocate _all_ datasets of dtype. // handle scalar and non-scalar records by name auto const lf_compRecordInit = [&currSpecies](const amrex::Vector& write_comp, - const amrex::Vector& comp_names, - openPMD::Dataset& dtype) + const amrex::Vector& comp_names, + openPMD::Dataset& dtype) { - auto const min_counter = std::min(write_comp.size(), comp_names.size()); - for (int i = 0; i < min_counter; ++i) - { - if (write_comp[i]) { - helper::getComponentRecord(currSpecies, comp_names[i]).resetDataset(dtype); - } - } + auto const min_counter = std::min(write_comp.size(), comp_names.size()); + for (int i = 0; i < min_counter; ++i) + { + if (write_comp[i]) { + helper::getComponentRecord(currSpecies, comp_names[i]).resetDataset(dtype); + } + } }; auto dtype_real = openPMD::Dataset(openPMD::determineDatatype(), {np}, m_openPMDDatasetOptions); lf_compRecordInit(write_real_comp, real_comp_names, dtype_real); @@ -51,16 +51,16 @@ namespace amrex { } void AMReX_openPMDWriter::SetupPos(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np) const + const unsigned long long& np) const { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, m_openPMDDatasetOptions); auto idType = openPMD::Dataset(openPMD::determineDatatype< uint64_t >(), {np}, m_openPMDDatasetOptions); - + auto const positionComponents = /*helper::*/getParticlePositionComponentLabels(); for( auto const& comp : positionComponents ) - { - currSpecies["position"][comp].resetDataset( realType ); - } + { + currSpecies["position"][comp].resetDataset( realType ); + } auto const scalar = openPMD::RecordComponent::SCALAR; currSpecies["id"][scalar].resetDataset( idType ); @@ -77,7 +77,7 @@ namespace amrex { // since it is a const, only need to be called before flushing // void AMReX_openPMDWriter::SetupConstant(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np) const + const unsigned long long& np) const { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}, m_openPMDDatasetOptions); //auto const scalar = openPMD::RecordComponent::SCALAR; diff --git a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.H b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.H index 4ad86e90759..9bdbf8d13ad 100644 --- a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.H +++ b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.H @@ -26,30 +26,30 @@ namespace amrex inline void setOpenPMDUnit ( openPMD::Mesh mesh, const std::string field_name ) { - if (field_name[0] == 'E'){ - mesh.setUnitDimension({ - {openPMD::UnitDimension::L, 1}, - {openPMD::UnitDimension::M, 1}, - {openPMD::UnitDimension::T, -3}, - {openPMD::UnitDimension::I, -1}, - }); - } else if (field_name[0] == 'B'){ - mesh.setUnitDimension({ - {openPMD::UnitDimension::M, 1}, - {openPMD::UnitDimension::I, -1}, - {openPMD::UnitDimension::T, -2} - }); - }else if (field_name[0] == 'j'){ // current - mesh.setUnitDimension({ - {openPMD::UnitDimension::L, -2}, - {openPMD::UnitDimension::I, 1}, - }); + if (field_name[0] == 'E'){ + mesh.setUnitDimension({ + {openPMD::UnitDimension::L, 1}, + {openPMD::UnitDimension::M, 1}, + {openPMD::UnitDimension::T, -3}, + {openPMD::UnitDimension::I, -1}, + }); + } else if (field_name[0] == 'B'){ + mesh.setUnitDimension({ + {openPMD::UnitDimension::M, 1}, + {openPMD::UnitDimension::I, -1}, + {openPMD::UnitDimension::T, -2} + }); + }else if (field_name[0] == 'j'){ // current + mesh.setUnitDimension({ + {openPMD::UnitDimension::L, -2}, + {openPMD::UnitDimension::I, 1}, + }); } else if (field_name.substr(0,3) == "rho"){ // charge density - mesh.setUnitDimension({ - {openPMD::UnitDimension::L, -3}, - {openPMD::UnitDimension::I, 1}, - {openPMD::UnitDimension::T, 1}, - }); + mesh.setUnitDimension({ + {openPMD::UnitDimension::L, -3}, + {openPMD::UnitDimension::I, 1}, + {openPMD::UnitDimension::T, 1}, + }); } } @@ -57,57 +57,57 @@ namespace amrex inline std::vector< double > getRelativeCellPosition(amrex::MultiFab const& mf) { - amrex::IndexType const idx_type = mf.ixType(); - - std::vector< double > relative_position(AMREX_SPACEDIM, 0.0); - // amrex::CellIndex::CELL means: 0.5 from lower corner for that index/direction - // amrex::CellIndex::NODE means: at corner for that index/direction - // WarpX::do_nodal means: all indices/directions on CellIndex::NODE - for (int d = 0; d < AMREX_SPACEDIM; d++) - { - if (idx_type.cellCentered(d)) - relative_position.at(d) = 0.5; - } - return relative_position; + amrex::IndexType const idx_type = mf.ixType(); + + std::vector< double > relative_position(AMREX_SPACEDIM, 0.0); + // amrex::CellIndex::CELL means: 0.5 from lower corner for that index/direction + // amrex::CellIndex::NODE means: at corner for that index/direction + // WarpX::do_nodal means: all indices/directions on CellIndex::NODE + for (int d = 0; d < AMREX_SPACEDIM; d++) + { + if (idx_type.cellCentered(d)) + relative_position.at(d) = 0.5; + } + return relative_position; } inline std::vector getReversedVec( const IntVect& v ) { - // Convert the IntVect v to and std::vector u - std::vector u = { - AMREX_D_DECL( - static_cast(v[0]), - static_cast(v[1]), - static_cast(v[2]) - ) - }; - // Reverse the order of elements, if v corresponds to the indices of a - // Fortran-order array (like an AMReX FArrayBox) - // but u is intended to be used with a C-order API (like openPMD) - std::reverse( u.begin(), u.end() ); - - return u; + // Convert the IntVect v to and std::vector u + std::vector u = { + AMREX_D_DECL( + static_cast(v[0]), + static_cast(v[1]), + static_cast(v[2]) + ) + }; + // Reverse the order of elements, if v corresponds to the indices of a + // Fortran-order array (like an AMReX FArrayBox) + // but u is intended to be used with a C-order API (like openPMD) + std::reverse( u.begin(), u.end() ); + + return u; } inline std::vector getReversedVec( const Real* v ) { - // Convert Real* v to and std::vector u - std::vector u = { - AMREX_D_DECL( - static_cast(v[0]), - static_cast(v[1]), - static_cast(v[2]) - ) - }; - // Reverse the order of elements, if v corresponds to the indices of a - // Fortran-order array (like an AMReX FArrayBox) - // but u is intended to be used with a C-order API (like openPMD) - - std::reverse( u.begin(), u.end() ); - - return u; + // Convert Real* v to and std::vector u + std::vector u = { + AMREX_D_DECL( + static_cast(v[0]), + static_cast(v[1]), + static_cast(v[2]) + ) + }; + // Reverse the order of elements, if v corresponds to the indices of a + // Fortran-order array (like an AMReX FArrayBox) + // but u is intended to be used with a C-order API (like openPMD) + + std::reverse( u.begin(), u.end() ); + + return u; } } @@ -127,35 +127,35 @@ namespace amrex //void SetCustomizer(AMReX_Optional* f); void SetStep(int ts); - void CloseStep(int ts); + void CloseStep(int ts); void CloseHandler(); - + void WriteSingleLevel (//const std::string &plotfilename, - const MultiFab &mf, - const Vector &varnames, - const Geometry &geom, - Real t, - //int level_step, - const std::string &ignored_versionName = "HyperCLaw-V1.1", - const std::string &ignored_levelPrefix = "Level_", - const std::string &ignored_mfPrefix = "Cell", - const Vector& ignored_extra_dirs = Vector()); + const MultiFab &mf, + const Vector &varnames, + const Geometry &geom, + Real t, + //int level_step, + const std::string &ignored_versionName = "HyperCLaw-V1.1", + const std::string &ignored_levelPrefix = "Level_", + const std::string &ignored_mfPrefix = "Cell", + const Vector& ignored_extra_dirs = Vector()); void WriteMultiLevel ( - //int nlevels, will all levels in mf & geom - const Vector &mf, - const Vector &varnames, - const Vector &geom, - Real time, - //const Vector &level_steps, - const Vector &ref_ratio, - const std::string &ignored_versionName = "HyperCLaw-V1.1", - const std::string &ignored_levelPrefix = "Level_", - const std::string &ignored_mfPrefix = "Cell", - const Vector& ignored_extra_dirs = Vector()); - - - //////////////////////////// + //int nlevels, will all levels in mf & geom + const Vector &mf, + const Vector &varnames, + const Vector &geom, + Real time, + //const Vector &level_steps, + const Vector &ref_ratio, + const std::string &ignored_versionName = "HyperCLaw-V1.1", + const std::string &ignored_levelPrefix = "Level_", + const std::string &ignored_mfPrefix = "Cell", + const Vector& ignored_extra_dirs = Vector()); + + + //////////////////////////// struct AMReX_VarNameParser { AMReX_VarNameParser(std::string varname); @@ -167,18 +167,18 @@ namespace amrex { using vs = std::vector< std::string >; if (m_ThetaMode) - { - // if we write individual modes - vs const fieldComponents{"r", "t", "z"}; - return fieldComponents; - } - else - { - // if we just write reconstructed fields at theta=0 or are Cartesian - // note: 1D3V and 2D3V simulations still have 3 components for the fields - vs const fieldComponents{"x", "y", "z"}; - return fieldComponents; - } + { + // if we write individual modes + vs const fieldComponents{"r", "t", "z"}; + return fieldComponents; + } + else + { + // if we just write reconstructed fields at theta=0 or are Cartesian + // note: 1D3V and 2D3V simulations still have 3 components for the fields + vs const fieldComponents{"x", "y", "z"}; + return fieldComponents; + } } virtual std::vector< std::string > @@ -186,48 +186,48 @@ namespace amrex { using vs = std::vector< std::string >; - // temporary resolution + // temporary resolution #if AMREX_SPACEDIM == 1 - vs const axisLabels{"z"}; + vs const axisLabels{"z"}; #elif AMREX_SPACEDIM == 2 - vs const axisLabels{"x", "z"}; + vs const axisLabels{"x", "z"}; #elif AMREX_SPACEDIM == 3 - vs const axisLabels{"x", "y", "z"}; // x varies fastest in memory + vs const axisLabels{"x", "y", "z"}; // x varies fastest in memory #else # error Unable to label more than 3d - // no labels to be addressed - vs const axisLabels{} + // no labels to be addressed + vs const axisLabels{} #endif - // revert to C order (fastest varying index last) - return {axisLabels.rbegin(), axisLabels.rend()}; + // revert to C order (fastest varying index last) + return {axisLabels.rbegin(), axisLabels.rend()}; } // getFieldAxisLabels - ///// members + ///// members std::string m_FieldName; std::string m_CompName; bool m_ThetaMode = false; int m_ModeIndex=-1; - + }; //////////////////////////// - //////////////////////////// + //////////////////////////// class AMReX_openPMDWriter { - public: + public: AMReX_openPMDWriter(const std::string& prefix, - openPMD::IterationEncoding ie, - std::string filetype, - std::string openpmdOptions); + openPMD::IterationEncoding ie, + std::string filetype, + std::string openpmdOptions); AMReX_openPMDWriter(); - + virtual ~AMReX_openPMDWriter(); virtual openPMD::Iteration GetIteration (int const iteration) const { - return m_Series->writeIterations()[iteration]; + return m_Series->writeIterations()[iteration]; } virtual void SetStep(int ts); @@ -235,40 +235,40 @@ namespace amrex virtual void Init(openPMD::Access access); virtual void WriteMesh(const std::vector& varnames, - const amrex::Vector& mf, - const amrex::Vector& geom, - //const Vector &iteration, - //const int iteration, /* note: all levels are outputing the same step */ - const double time ) const; + const amrex::Vector& mf, + const amrex::Vector& geom, + //const Vector &iteration, + //const int iteration, /* note: all levels are outputing the same step */ + const double time ) const; template void DumpParticles(PC const& pc, - const std::string& name, - //const int iteration, - const amrex::Vector& write_real_comp, // size = NStructReal + NArrayReal - const amrex::Vector& write_int_comp, // size = NStructInt + NArrayInt - const amrex::Vector& real_comp_names, - const amrex::Vector& int_comp_names - ) const; + const std::string& name, + //const int iteration, + const amrex::Vector& write_real_comp, // size = NStructReal + NArrayReal + const amrex::Vector& write_int_comp, // size = NStructInt + NArrayInt + const amrex::Vector& real_comp_names, + const amrex::Vector& int_comp_names + ) const; template void DumpParticles(PC const& pc, - const std::string& name, - //const int iteration, - const amrex::Vector& write_real_comp, // size = NStructReal + NArrayReal - const amrex::Vector& write_int_comp, // size = NStructInt + NArrayInt - const amrex::Vector& real_comp_names, - const amrex::Vector& int_comp_names, - FUNC_pc&& pcFunc, - FUNC_pit&& pitFunc - ) const; + const std::string& name, + //const int iteration, + const amrex::Vector& write_real_comp, // size = NStructReal + NArrayReal + const amrex::Vector& write_int_comp, // size = NStructInt + NArrayInt + const amrex::Vector& real_comp_names, + const amrex::Vector& int_comp_names, + FUNC_pc&& pcFunc, + FUNC_pit&& pitFunc + ) const; virtual bool AllocatePtlProperties(openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const; + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const; virtual void SetParticleSpecieAttributes(openPMD::ParticleSpecies& currSpecies) const {} @@ -278,50 +278,50 @@ namespace amrex // called once because it is constant valued record virtual void SetupConstant(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np) const; + const unsigned long long& np) const; virtual std::vector< std::string > getParticlePositionComponentLabels() const { #if (AMREX_SPACEDIM == 3) - std::vector< std::string > const positionComponents{"x", "y", "z"}; + std::vector< std::string > const positionComponents{"x", "y", "z"}; #elif (AMREX_SPACEDIM == 2) - std::vector< std::string > const positionComponents{"x", "y"}; + std::vector< std::string > const positionComponents{"x", "y"}; #elif (AMREX_SPACEDIM == 1) - std::vector< std::string > const positionComponents{"x"}; + std::vector< std::string > const positionComponents{"x"}; #else # error Unknown dimensionality. #endif - return positionComponents; + return positionComponents; } template void SetupRealProperties(PC const& pc, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const; + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const; /* */ template void SaveRealProperty (PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names) const; - + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names) const; + std::unique_ptr m_Series = nullptr; std::string m_openPMDPrefix = std::string(); int m_openPMDMinDigits = 6; - std::string m_openPMDDatasetOptions = "{}"; + std::string m_openPMDDatasetOptions = "{}"; openPMD::IterationEncoding m_openPMDEncoding = openPMD::IterationEncoding::fileBased; @@ -330,65 +330,65 @@ namespace amrex protected: void CompSetup(int lev, - openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom, - const std::vector& varnames, - const amrex::MultiFab* mf) const; + openPMD::Container< openPMD::Mesh >& meshes, + amrex::Geometry& full_geom, + const std::vector& varnames, + const amrex::MultiFab* mf) const; void CompStorage(int lev, - openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom, - const std::vector& varnames, - const amrex::MultiFab* mf) const; + openPMD::Container< openPMD::Mesh >& meshes, + amrex::Geometry& full_geom, + const std::vector& varnames, + const amrex::MultiFab* mf) const; virtual void SetupFields (openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom) const; + amrex::Geometry& full_geom) const; virtual void SetupMeshComp (openPMD::Mesh& mesh, - const amrex::Geometry& full_geom, - amrex::MultiFab const& mf, - const AMReX_VarNameParser& varName) const; + const amrex::Geometry& full_geom, + amrex::MultiFab const& mf, + const AMReX_VarNameParser& varName) const; void GetFileName(std::string& filepath); template void StoreAoS_Real(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - unsigned long long offset) const; + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + unsigned long long offset) const; template void SavePosId(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset) const; + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset) const; template void StoreAoS_Int(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - unsigned long long offset) const; - + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + unsigned long long offset) const; + template void StoreSoAReal(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - unsigned long long offset) const; + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + unsigned long long offset) const; template void StoreSoAInt(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - unsigned long long offset) const; - + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + unsigned long long offset) const; + int m_CurrentStep = -1; }; @@ -400,16 +400,16 @@ namespace amrex public: AMReX_openPMDHandler(const std::string& prefix = std::string()); - ~AMReX_openPMDHandler(); - + ~AMReX_openPMDHandler(); + std::unique_ptr m_Writer = nullptr; - + private: void CreateWriter(const std::string& prefix = std::string()); - + }; // class AMReX_openPMDHandler - - + + }// name space openpmd_api } // namespace amrex diff --git a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.cpp b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.cpp index 1122a8e2331..9642e8e2001 100644 --- a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.cpp +++ b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD.cpp @@ -16,7 +16,7 @@ namespace amrex -{ +{ namespace openpmd_api { /* global handler, activate with InitHandler() & deactivate with CloseHandler() */ @@ -26,12 +26,12 @@ namespace amrex { std::string filePath {""}; if (prefix.size() == 0) - { - ParmParse pp; - pp.query("openpmd_directory", filePath); - } + { + ParmParse pp; + pp.query("openpmd_directory", filePath); + } else { - filePath = prefix; + filePath = prefix; } std::unique_ptr< AMReX_openPMDHandler > userHandler; @@ -42,7 +42,7 @@ namespace amrex void CloseUserHandler(std::unique_ptr& userHandler) { if (userHandler == nullptr) - return; + return; userHandler.reset(nullptr); } @@ -51,19 +51,19 @@ namespace amrex { std::string filePath {""}; if (prefix.size() == 0) - { - ParmParse pp; - pp.query("openpmd_directory", filePath); - } + { + ParmParse pp; + pp.query("openpmd_directory", filePath); + } else { - filePath = prefix; + filePath = prefix; } if (m_OpenPMDHandler == nullptr) - m_OpenPMDHandler.reset(new AMReX_openPMDHandler(filePath)); + m_OpenPMDHandler.reset(new AMReX_openPMDHandler(filePath)); else if (m_OpenPMDHandler->m_Writer != nullptr) - if (m_OpenPMDHandler->m_Writer->m_openPMDPrefix != filePath) - m_OpenPMDHandler.reset(new AMReX_openPMDHandler(filePath)); + if (m_OpenPMDHandler->m_Writer->m_openPMDPrefix != filePath) + m_OpenPMDHandler.reset(new AMReX_openPMDHandler(filePath)); // already using the directory, no action needed } @@ -80,15 +80,15 @@ namespace amrex void CloseHandler() { if (m_OpenPMDHandler == nullptr) - return; + return; m_OpenPMDHandler.reset(nullptr); } - + void SetStep(int ts) { if ((m_OpenPMDHandler == nullptr) || (m_OpenPMDHandler->m_Writer == nullptr)) - return; + return; m_OpenPMDHandler->m_Writer->SetStep(ts); } @@ -96,21 +96,21 @@ namespace amrex void CloseStep(int ts) { if ((m_OpenPMDHandler == nullptr) || (m_OpenPMDHandler->m_Writer == nullptr)) - return; + return; - m_OpenPMDHandler->m_Writer->CloseStep(ts); + m_OpenPMDHandler->m_Writer->CloseStep(ts); } - + void WriteSingleLevel (//const std::string &plotfilename, - const MultiFab &mf, - const Vector &varnames, - const Geometry &geom, - Real t, - //int level_step, - const std::string &versionName, - const std::string &levelPrefix, - const std::string &mfPrefix, - const Vector& extra_dirs) + const MultiFab &mf, + const Vector &varnames, + const Geometry &geom, + Real t, + //int level_step, + const std::string &versionName, + const std::string &levelPrefix, + const std::string &mfPrefix, + const Vector& extra_dirs) { Vector v_mf(1,&mf); Vector v_geom(1,geom); @@ -119,31 +119,31 @@ namespace amrex WriteMultiLevel(v_mf, varnames, v_geom, t, /*v_level_steps,*/ ref_ratio, versionName, levelPrefix, mfPrefix, extra_dirs); } - + void WriteMultiLevel ( - //int nlevels, // will write all levels in mf & geom - const Vector &mf, - const Vector &varnames, - const Vector &geom, - Real time, - //const Vector &level_steps, - const Vector &ref_ratio, - const std::string &versionName, - const std::string &levelPrefix, - const std::string &mfPrefix, - const Vector& extra_dirs) + //int nlevels, // will write all levels in mf & geom + const Vector &mf, + const Vector &varnames, + const Vector &geom, + Real time, + //const Vector &level_steps, + const Vector &ref_ratio, + const std::string &versionName, + const std::string &levelPrefix, + const std::string &mfPrefix, + const Vector& extra_dirs) { if ((m_OpenPMDHandler == nullptr) || (m_OpenPMDHandler->m_Writer == nullptr)) - return; + return; BL_ASSERT ( geom.size() == mf.size() ); BL_ASSERT ( mf[0]->nComp() <= varnames.size() ); m_OpenPMDHandler->m_Writer->WriteMesh(varnames, - mf, //amrex::GetVecOfConstPtrs(mf), - geom, - //level_steps[0], - time); + mf, //amrex::GetVecOfConstPtrs(mf), + geom, + //level_steps[0], + time); } } // namespace openpmd_api } // namespace amrex diff --git a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD_PTLImpl.H b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD_PTLImpl.H index f2456698ffc..f9827c238b1 100644 --- a/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD_PTLImpl.H +++ b/Src/Extern/openPMD-api/AMReX_PlotFileUtilOPENPMD_PTLImpl.H @@ -17,14 +17,14 @@ namespace amrex template void WriteParticles(PC const& pc, - const std::string& specieName, - //int iteration, - const Vector& real_comp_names, - const Vector& int_comp_names, - const Vector& do_write_real_comp, - const Vector& do_write_int_comp, - FUNC_pc&& pcFunc, - FUNC_pit&& posIdFunc) + const std::string& specieName, + //int iteration, + const Vector& real_comp_names, + const Vector& int_comp_names, + const Vector& do_write_real_comp, + const Vector& do_write_int_comp, + FUNC_pc&& pcFunc, + FUNC_pit&& posIdFunc) { BL_PROFILE("amrex::openpmd_api::WriteParticles()"); @@ -32,26 +32,26 @@ namespace amrex amrex::Print()<<" Int: "<m_Writer == nullptr)) - return; + return; AMREX_ASSERT( do_write_real_comp.size() <= real_comp_names.size() ); AMREX_ASSERT( do_write_int_comp.size() <= int_comp_names.size() ); AMREX_ASSERT( real_comp_names.size() <= pc.NStructReal + pc.NumRealComps() ); m_OpenPMDHandler->m_Writer->DumpParticles(pc, specieName, //iteration, - do_write_real_comp, do_write_int_comp, real_comp_names, int_comp_names, - pcFunc, posIdFunc); + do_write_real_comp, do_write_int_comp, real_comp_names, int_comp_names, + pcFunc, posIdFunc); } // use default pos_id func template void WriteParticles(PC const& pc, - const std::string& specieName, - //int iteration, - const Vector& real_comp_names, - const Vector& int_comp_names, - const Vector& do_write_real_comp, - const Vector& do_write_int_comp) + const std::string& specieName, + //int iteration, + const Vector& real_comp_names, + const Vector& int_comp_names, + const Vector& do_write_real_comp, + const Vector& do_write_int_comp) { BL_PROFILE("amrex::openpmd_api::WriteParticles(..)"); @@ -59,125 +59,125 @@ namespace amrex amrex::Print()<<" Int: "<m_Writer == nullptr)) - return; + return; AMREX_ASSERT( do_write_real_comp.size() <= real_comp_names.size() ); AMREX_ASSERT( do_write_int_comp.size() <= int_comp_names.size() ); AMREX_ASSERT( real_comp_names.size() <= pc.NStructReal + pc.NumRealComps() ); m_OpenPMDHandler->m_Writer->DumpParticles(pc, specieName, //iteration, - do_write_real_comp, do_write_int_comp, real_comp_names, int_comp_names); + do_write_real_comp, do_write_int_comp, real_comp_names, int_comp_names); } template void WriteParticles(PC const& pc, - const std::string& specieName, - //int iteration, - const Vector& real_comp_names, - const Vector& int_comp_names) + const std::string& specieName, + //int iteration, + const Vector& real_comp_names, + const Vector& int_comp_names) { Vector write_real_comp; for (int i = 0; i < real_comp_names.size(); ++i ) - { - write_real_comp.push_back(1); - } + { + write_real_comp.push_back(1); + } Vector write_int_comp; for (int i = 0; i < int_comp_names.size(); ++i ) - { - write_int_comp.push_back(1); - } + { + write_int_comp.push_back(1); + } WriteParticles(pc, specieName, //iteration, - real_comp_names, int_comp_names, - write_real_comp, write_int_comp); + real_comp_names, int_comp_names, + write_real_comp, write_int_comp); } template void WriteParticles(PC const& pc, - const std::string& specieName) + const std::string& specieName) { Vector write_real_comp; Vector real_comp_names; for (int i = 0; i < pc.NStructReal + pc.NumRealComps(); ++i ) - { - write_real_comp.push_back(1); + { + write_real_comp.push_back(1); - std::stringstream ss; - if (i < pc.NStructReal) - ss << "aos-r_" << i; - else - ss << "SoA-r" << i; + std::stringstream ss; + if (i < pc.NStructReal) + ss << "aos-r_" << i; + else + ss << "SoA-r" << i; - real_comp_names.push_back(ss.str()); - } + real_comp_names.push_back(ss.str()); + } Vector write_int_comp; Vector int_comp_names; for (int i = 0; i < pc.NStructInt + pc.NumIntComps(); ++i ) - { - write_int_comp.push_back(1); - std::stringstream ss; + { + write_int_comp.push_back(1); + std::stringstream ss; - if (i < pc.NStructInt) - ss << "aos-i_" << i; - else - ss << "SoA-i" << i; + if (i < pc.NStructInt) + ss << "aos-i_" << i; + else + ss << "SoA-i" << i; - int_comp_names.push_back(ss.str()); - } + int_comp_names.push_back(ss.str()); + } WriteParticles(pc, specieName, //iteration, - real_comp_names, int_comp_names, - write_real_comp, write_int_comp); + real_comp_names, int_comp_names, + write_real_comp, write_int_comp); } template void WriteParticles(PC const& pc, - const std::string& specieName, - FUNC_pc&& pcFunc, - FUNC_pit&& pitFunc - ) + const std::string& specieName, + FUNC_pc&& pcFunc, + FUNC_pit&& pitFunc + ) { Vector write_real_comp; Vector real_comp_names; for (int i = 0; i < pc.NStructReal + pc.NumRealComps(); ++i ) - { - write_real_comp.push_back(1); - - std::stringstream ss; - if (i < pc.NStructReal) - ss << "aos-r_" << i; - else - ss << "SoA-r" << i; - - real_comp_names.push_back(ss.str()); - } - + { + write_real_comp.push_back(1); + + std::stringstream ss; + if (i < pc.NStructReal) + ss << "aos-r_" << i; + else + ss << "SoA-r" << i; + + real_comp_names.push_back(ss.str()); + } + Vector write_int_comp; Vector int_comp_names; for (int i = 0; i < pc.NStructInt + pc.NumIntComps(); ++i ) - { - write_int_comp.push_back(1); - std::stringstream ss; + { + write_int_comp.push_back(1); + std::stringstream ss; - if (i < pc.NStructInt) - ss << "aos-i_" << i; - else - ss << "SoA-i" << i; - int_comp_names.push_back(ss.str()); - } + if (i < pc.NStructInt) + ss << "aos-i_" << i; + else + ss << "SoA-i" << i; + int_comp_names.push_back(ss.str()); + } WriteParticles(pc, specieName, //iteration, - real_comp_names, int_comp_names, - write_real_comp, write_int_comp, pcFunc, pitFunc); + real_comp_names, int_comp_names, + write_real_comp, write_int_comp, pcFunc, pitFunc); } - + namespace helper { //////////////////////////////////// // @@ -187,66 +187,66 @@ namespace amrex inline std::map< openPMD::UnitDimension, double > getUnitDimension ( std::string const & record_name ) { - - if( record_name == "position" ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "positionOffset" ) return { - {openPMD::UnitDimension::L, 1.} - }; - else if( record_name == "momentum" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -1.} - }; - else if( record_name == "charge" ) return { - {openPMD::UnitDimension::T, 1.}, - {openPMD::UnitDimension::I, 1.} - }; - else if( record_name == "mass" ) return { - {openPMD::UnitDimension::M, 1.} - }; - else if( record_name == "E" ) return { - {openPMD::UnitDimension::L, 1.}, - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::T, -3.}, - {openPMD::UnitDimension::I, -1.}, - }; - else if( record_name == "B" ) return { - {openPMD::UnitDimension::M, 1.}, - {openPMD::UnitDimension::I, -1.}, - {openPMD::UnitDimension::T, -2.} - }; - else return {}; + + if( record_name == "position" ) return { + {openPMD::UnitDimension::L, 1.} + }; + else if( record_name == "positionOffset" ) return { + {openPMD::UnitDimension::L, 1.} + }; + else if( record_name == "momentum" ) return { + {openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -1.} + }; + else if( record_name == "charge" ) return { + {openPMD::UnitDimension::T, 1.}, + {openPMD::UnitDimension::I, 1.} + }; + else if( record_name == "mass" ) return { + {openPMD::UnitDimension::M, 1.} + }; + else if( record_name == "E" ) return { + {openPMD::UnitDimension::L, 1.}, + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::T, -3.}, + {openPMD::UnitDimension::I, -1.}, + }; + else if( record_name == "B" ) return { + {openPMD::UnitDimension::M, 1.}, + {openPMD::UnitDimension::I, -1.}, + {openPMD::UnitDimension::T, -2.} + }; + else return {}; } // // - // + // constexpr uint64_t localIDtoGlobal(int const id, int const cpu) { - // from WarpXUtilIO::localIDtoGlobal - static_assert(sizeof(int) * 2u <= sizeof(uint64_t), - "int size might cause collisions in global IDs"); - - return uint64_t(id) | uint64_t(cpu) << 32u; + // from WarpXUtilIO::localIDtoGlobal + static_assert(sizeof(int) * 2u <= sizeof(uint64_t), + "int size might cause collisions in global IDs"); + + return uint64_t(id) | uint64_t(cpu) << 32u; } - + inline std::pair< std::string, std::string > name2openPMD ( std::string const& fullName ) { - std::string record_name = fullName; - std::string component_name = openPMD::RecordComponent::SCALAR; - - // we use "_" as separator in names to group vector records - std::size_t startComp = fullName.find_last_of("_"); - if( startComp != std::string::npos ) { // non-scalar - record_name = fullName.substr(0, startComp); - component_name = fullName.substr(startComp + 1u); - } - return make_pair(record_name, component_name); + std::string record_name = fullName; + std::string component_name = openPMD::RecordComponent::SCALAR; + + // we use "_" as separator in names to group vector records + std::size_t startComp = fullName.find_last_of("_"); + if( startComp != std::string::npos ) { // non-scalar + record_name = fullName.substr(0, startComp); + component_name = fullName.substr(startComp + 1u); + } + return make_pair(record_name, component_name); } // @@ -254,11 +254,11 @@ namespace amrex // inline auto getComponentRecord (openPMD::ParticleSpecies& currSpecies, - std::string const comp_name) + std::string const comp_name) { - // handle scalar and non-scalar records by name - const auto [record_name, component_name] = name2openPMD(comp_name); - return currSpecies[record_name][component_name]; + // handle scalar and non-scalar records by name + const auto [record_name, component_name] = name2openPMD(comp_name); + return currSpecies[record_name][component_name]; }; } // namespace helper @@ -271,35 +271,35 @@ namespace amrex template void AMReX_openPMDWriter::StoreAoS_Real(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - unsigned long long offset) const + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + unsigned long long offset) const { auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); - + // first we concatinate the AoS into contiguous arrays { - for( auto idx=0; idx d( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); - - for( auto kk=0; kk d( + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); + + for( auto kk=0; kk void AMReX_openPMDWriter::StoreAoS_Int(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - unsigned long long offset) const + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + unsigned long long offset) const { auto const& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); - + // first we concatinate the AoS into contiguous arrays { - for( auto idx=0; idx d( - new int[numParticleOnTile], - [](int const *p){ delete[] p; } - ); - - for( auto kk=0; kk d( + new int[numParticleOnTile], + [](int const *p){ delete[] p; } + ); + + for( auto kk=0; kk void AMReX_openPMDWriter::StoreSoAReal(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - unsigned long long offset) const - { + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + unsigned long long offset) const + { auto const& soa = pti.GetStructOfArrays(); auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); @@ -355,50 +355,50 @@ namespace amrex auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()) - PIt::ContainerType::NStructReal; for (auto idx=0; idx void AMReX_openPMDWriter::StoreSoAInt(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - unsigned long long offset) const + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + unsigned long long offset) const { auto const& soa = pti.GetStructOfArrays(); auto const numParticleOnTile = pti.numParticles(); uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); - auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size()) - PIt::ContainerType::NStructInt; + auto const int_counter = std::min(write_int_comp.size(), int_comp_names.size()) - PIt::ContainerType::NStructInt; for (auto idx=0; idx void AMReX_openPMDWriter::SavePosId(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset) const + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset) const { BL_PROFILE("amrex::openpmd_api::SavePosId(default..)"); auto const numParticleOnTile = pti.numParticles(); @@ -409,50 +409,50 @@ namespace amrex // if we flush late as we do now, we can also copy out the data in one go const auto& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile { - // Save positions - auto const positionComponents = /*helper::*/getParticlePositionComponentLabels(); - - for (auto currDim = 0; currDim < AMREX_SPACEDIM; currDim++) - { - //amrex::AllPrint()< curr(new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); - for (auto i = 0; i < numParticleOnTile; i++) { - //amrex::AllPrint()<<" i = "< ids(new uint64_t[numParticleOnTile], - [](uint64_t const *p){ delete[] p; } - ); - for (auto i=0; i curr(new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); + for (auto i = 0; i < numParticleOnTile; i++) { + //amrex::AllPrint()<<" i = "< ids(new uint64_t[numParticleOnTile], + [](uint64_t const *p){ delete[] p; } + ); + for (auto i=0; i void AMReX_openPMDWriter::SaveRealProperty (PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names) const + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names) const { // note: WarpX does not yet use extra AoS Real attributes StoreAoS_Real(pti, currSpecies, write_real_comp, real_comp_names, offset); - StoreAoS_Int(pti, currSpecies, write_int_comp, int_comp_names, offset); + StoreAoS_Int(pti, currSpecies, write_int_comp, int_comp_names, offset); //auto const& soa = pti.GetStructOfArrays(); StoreSoAReal(pti, currSpecies, write_real_comp, real_comp_names, offset); @@ -470,42 +470,42 @@ namespace amrex template void AMReX_openPMDWriter::DumpParticles(PC const& pc, - const std::string& name, - //int iteration, - const amrex::Vector& write_real_comp, - const amrex::Vector& write_int_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& int_comp_names - ) const + const std::string& name, + //int iteration, + const amrex::Vector& write_real_comp, + const amrex::Vector& write_int_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& int_comp_names + ) const { DumpParticles(pc, name, write_real_comp, write_int_comp, real_comp_names, int_comp_names, - [=] (auto& pc, openPMD::ParticleSpecies& currSpecies, unsigned long long size) - { - }, // no extra pc level info to save - [=] (auto& pti, openPMD::ParticleSpecies& currSpecies, unsigned long long offset) - { - SavePosId(pti, currSpecies, offset); // default way of save pos & id - }); + [=] (auto& pc, openPMD::ParticleSpecies& currSpecies, unsigned long long size) + { + }, // no extra pc level info to save + [=] (auto& pti, openPMD::ParticleSpecies& currSpecies, unsigned long long offset) + { + SavePosId(pti, currSpecies, offset); // default way of save pos & id + }); } template void AMReX_openPMDWriter::DumpParticles(PC const& pc, - const std::string& name, - //int iteration, - const amrex::Vector& write_real_comp, - const amrex::Vector& write_int_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& int_comp_names, - FUNC_pc&& pc_func, - FUNC_pit&& posId_func - ) const + const std::string& name, + //int iteration, + const amrex::Vector& write_real_comp, + const amrex::Vector& write_int_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& int_comp_names, + FUNC_pc&& pc_func, + FUNC_pit&& posId_func + ) const { AMREX_ALWAYS_ASSERT_WITH_MESSAGE(m_Series != nullptr, "openPMD series must be initialized"); - + // TODO have to Count before this function due to const restriction // Count is not a const function //pc.CountParticles(); - + openPMD::Iteration currIteration = GetIteration( m_CurrentStep ); openPMD::ParticleSpecies currSpecies = currIteration.particles[name]; @@ -513,15 +513,15 @@ namespace amrex // define positions & offsets // /*AllocatePtlProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, - pc.m_PtlCounter.GetTotalNumParticles()); */ + pc.m_PtlCounter.GetTotalNumParticles()); */ if ( AllocatePtlProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, - pc.m_PtlCounter.GetTotalNumParticles()) ) - { - SetupRealProperties(pc, currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, - pc.m_PtlCounter.GetTotalNumParticles()); - SetupConstant(currSpecies, pc.m_PtlCounter.GetTotalNumParticles()); - } + pc.m_PtlCounter.GetTotalNumParticles()) ) + { + SetupRealProperties(pc, currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, + pc.m_PtlCounter.GetTotalNumParticles()); + SetupConstant(currSpecies, pc.m_PtlCounter.GetTotalNumParticles()); + } pc_func(pc, currSpecies, pc.m_PtlCounter.GetTotalNumParticles()); // open files from all processors, in case some will not contribute below @@ -529,38 +529,38 @@ namespace amrex bool emptyPC = true; for (auto currentLevel = 0; currentLevel <= pc.finestLevel(); currentLevel++) - { - uint64_t offset = static_cast( pc.m_PtlCounter.m_ParticleOffsetAtRank[currentLevel] ); - using ParIter = typename PC::ParConstIterType; - - //for (WarpXParIter pti(*pc, currentLevel); pti.isValid(); ++pti) { - for (ParIter pti(pc, currentLevel); pti.isValid(); ++pti) - { - auto const numParticleOnTile = pti.numParticles(); - uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); - - // Do not call storeChunk() with zero-sized particle tiles: - // https://github.com/openPMD/openPMD-api/issues/1147 - // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 - if (numParticleOnTile == 0) continue; - - emptyPC = false; - - //SavePosId(pti, currSpecies, offset); - posId_func(pti, currSpecies, offset); - // save "extra" particle properties in AoS and SoA - SaveRealProperty(pti, - currSpecies, - offset + GetGrandOffset(), - write_real_comp,real_comp_names, - write_int_comp, int_comp_names); - - offset += numParticleOnTile64; - } - } + { + uint64_t offset = static_cast( pc.m_PtlCounter.m_ParticleOffsetAtRank[currentLevel] ); + using ParIter = typename PC::ParConstIterType; + + //for (WarpXParIter pti(*pc, currentLevel); pti.isValid(); ++pti) { + for (ParIter pti(pc, currentLevel); pti.isValid(); ++pti) + { + auto const numParticleOnTile = pti.numParticles(); + uint64_t const numParticleOnTile64 = static_cast( numParticleOnTile ); + + // Do not call storeChunk() with zero-sized particle tiles: + // https://github.com/openPMD/openPMD-api/issues/1147 + // https://github.com/ECP-WarpX/WarpX/pull/1898#discussion_r745008290 + if (numParticleOnTile == 0) continue; + + emptyPC = false; + + //SavePosId(pti, currSpecies, offset); + posId_func(pti, currSpecies, offset); + // save "extra" particle properties in AoS and SoA + SaveRealProperty(pti, + currSpecies, + offset + GetGrandOffset(), + write_real_comp,real_comp_names, + write_int_comp, int_comp_names); + + offset += numParticleOnTile64; + } + } if (emptyPC && m_Series->backend() == "ADIOS2") { - // TODO adiosWorkaround with empty rank reading error() for __BTD__ + // TODO adiosWorkaround with empty rank reading error() for __BTD__ } m_Series->flush(); } @@ -568,12 +568,12 @@ namespace amrex template void AMReX_openPMDWriter::SetupRealProperties (PC const& pc, - openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const + openPMD::ParticleSpecies& currSpecies, + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const { //AllocateRealProperties(currSpecies, write_real_comp, real_comp_names, write_int_comp, int_comp_names, np); @@ -584,54 +584,54 @@ namespace amrex auto const real_counter = std::min(write_real_comp.size(), real_comp_names.size()) - PC::NStructReal; //for (auto idx=0; idx& domains, + Vector& domains, Vector& grids, Vector& ref_ratio); @@ -49,13 +49,13 @@ int main(int argc, char* argv[]) } struct TestField -{ +{ TestField(const InputParams& inputs) { m_Time = 0.0; const int nghost = 0; - + Vector domains; Vector ba; set_grids_nested(inputs, domains, ba, m_Ref_ratio); @@ -98,7 +98,7 @@ struct TestField } Vector m_Level_steps; - + Vector m_Varnames; Real m_Time; Vector m_Ref_ratio; @@ -106,7 +106,7 @@ struct TestField Vector > m_mf; }; - + void saveFile(char* fname, const InputParams& inputs, const TestField& testField) { #ifdef AMREX_USE_OPENPMD_API @@ -116,49 +116,49 @@ void saveFile(char* fname, const InputParams& inputs, const TestField& testFiel { openpmd_api::SetStep(ts); openpmd_api::WriteMultiLevel(//fname, - amrex::GetVecOfConstPtrs(testField.m_mf), testField.m_Varnames, - testField.m_Geom, testField.m_Time, /*testField.m_Level_steps,*/ testField.m_Ref_ratio); + amrex::GetVecOfConstPtrs(testField.m_mf), testField.m_Varnames, + testField.m_Geom, testField.m_Time, /*testField.m_Level_steps,*/ testField.m_Ref_ratio); openpmd_api::CloseStep(ts); - - //saveFile(fname, ts, inputs, testField); + + //saveFile(fname, ts, inputs, testField); amrex::Print()<<"Timestep: "<& domains, + Vector& domains, Vector& grids, Vector& ref_ratio) { //int ncells, max_grid_size, nlevs; AMREX_ALWAYS_ASSERT(input.nlevs < 2); // relax this later - + IntVect domain_lo(AMREX_D_DECL(0, 0, 0)); IntVect domain_hi(AMREX_D_DECL(input.ncells-1, input.ncells-1, input.ncells-1)); diff --git a/Tests/openPMDTests/ptls/main.cpp b/Tests/openPMDTests/ptls/main.cpp index 732105ec408..c606f133a02 100644 --- a/Tests/openPMDTests/ptls/main.cpp +++ b/Tests/openPMDTests/ptls/main.cpp @@ -24,31 +24,31 @@ struct TestParams { int max_grid_size; int nppc; bool verbose; - + int nplotfile=1; }; void checkMFBox(const TestParams& parms, - Vector outputMF) + Vector outputMF) { for (int lev=0; lev < parms.nlevs; lev++) - { - auto curr_mf = outputMF[lev]; - int const ncomp = curr_mf->nComp(); - amrex::Print()<<" checking boxes, lev="<nComp(); + amrex::Print()<<" checking boxes, lev="< outputRR(output_levs); for (int lev = 0; lev < output_levs; ++lev) { outputMF1[lev] = &density1[lev]; - outputMF2[lev] = &density2[lev]; + outputMF2[lev] = &density2[lev]; outputRR[lev] = IntVect(AMREX_D_DECL(2, 2, 2)); } @@ -192,46 +192,46 @@ void testParticleMesh (TestParams& parms, int nghost) int nsteps = 3; for (int ts = 0; ts < nsteps; ts++) { - //Vector level_steps; - //level_steps.push_back(ts); - //level_steps.push_back(ts); - - openpmd_api::SetStep(ts); - openpmd_api::WriteParticles(myPC, specieName /*ts*/); // with default component names - - char name[512]; - std::snprintf(name, sizeof name, "plotfile_%05d", ts); - - if ( 1 == parms.nlevs ) - { - // example to store coarse level - openpmd_api::WriteSingleLevel(*(outputMF1[0]), varnames1, geom[0], 0.0, 0); - openpmd_api::WriteSingleLevel(*(outputMF2[0]), varnames2, geom[0], 0.0, 0); - } - else - { - // store multi mesh levels - openpmd_api::WriteMultiLevel(//parms.nlevs, - outputMF1, - varnames1, - geom, - 0.0, - //level_steps, - outputRR - ); - // store multi mesh levels - openpmd_api::WriteMultiLevel(//parms.nlevs, - outputMF2, - varnames2, - geom, - 0.0, - //level_steps, - outputRR - ); - } - openpmd_api::CloseStep(ts); - - amrex::Print()<<"Timestep: "< level_steps; + //level_steps.push_back(ts); + //level_steps.push_back(ts); + + openpmd_api::SetStep(ts); + openpmd_api::WriteParticles(myPC, specieName /*ts*/); // with default component names + + char name[512]; + std::snprintf(name, sizeof name, "plotfile_%05d", ts); + + if ( 1 == parms.nlevs ) + { + // example to store coarse level + openpmd_api::WriteSingleLevel(*(outputMF1[0]), varnames1, geom[0], 0.0, 0); + openpmd_api::WriteSingleLevel(*(outputMF2[0]), varnames2, geom[0], 0.0, 0); + } + else + { + // store multi mesh levels + openpmd_api::WriteMultiLevel(//parms.nlevs, + outputMF1, + varnames1, + geom, + 0.0, + //level_steps, + outputRR + ); + // store multi mesh levels + openpmd_api::WriteMultiLevel(//parms.nlevs, + outputMF2, + varnames2, + geom, + 0.0, + //level_steps, + outputRR + ); + } + openpmd_api::CloseStep(ts); + + amrex::Print()<<"Timestep: "<AssignPtlOffset(num_particles); - testWriter->SetLastFlush(); - } - if (0) - { // test with default component names - openpmd_api::WriteParticles(myPC, specieName); - } - else - { // test with RZ style pos id - openpmd_api::WriteParticles(myPC, - specieName, - [=] (auto& pc, openPMD::ParticleSpecies& currSpecies, unsigned long long localTotal) - { - amrex::ParticleReal charge = 0.01; // warpx: pc->getCharge() - amrex::ParticleReal mass = 0.5; // warpx: pc->getMass(); - - testWriter->SetConstantMassCharge(currSpecies, localTotal, charge, mass); - }, - [=] (auto& pti, openPMD::ParticleSpecies& currSpecies, unsigned long long offset) - { - testWriter->SavePosId_RZ(pti, currSpecies, offset); // also supports RZ - }); - } - - { - // store multi mesh levels - openpmd_api::WriteMultiLevel(//parms.nlevs, - outputMF1, - varnames1, - geom, - 0.0, - outputRR - ); - } - - openpmd_api::CloseStep(ts); - - amrex::Print()<<"Timestep: "<AssignPtlOffset(num_particles); + testWriter->SetLastFlush(); + } + if (0) + { // test with default component names + openpmd_api::WriteParticles(myPC, specieName); + } + else + { // test with RZ style pos id + openpmd_api::WriteParticles(myPC, + specieName, + [=] (auto& pc, openPMD::ParticleSpecies& currSpecies, unsigned long long localTotal) + { + amrex::ParticleReal charge = 0.01; // warpx: pc->getCharge() + amrex::ParticleReal mass = 0.5; // warpx: pc->getMass(); + + testWriter->SetConstantMassCharge(currSpecies, localTotal, charge, mass); + }, + [=] (auto& pti, openPMD::ParticleSpecies& currSpecies, unsigned long long offset) + { + testWriter->SavePosId_RZ(pti, currSpecies, offset); // also supports RZ + }); + } + + { + // store multi mesh levels + openpmd_api::WriteMultiLevel(//parms.nlevs, + outputMF1, + varnames1, + geom, + 0.0, + outputRR + ); + } + + openpmd_api::CloseStep(ts); + + amrex::Print()<<"Timestep: "< fieldPMLdirections, - openPMD::IterationEncoding ie = openPMD::IterationEncoding::groupBased, - std::string filetype = "default", - std::string openpmdOptions = "{}"); - + openPMD::IterationEncoding ie = openPMD::IterationEncoding::groupBased, + std::string filetype = "default", + std::string openpmdOptions = "{}"); + //~AMReX_warpxBTDWriter() override = default; ~AMReX_warpxBTDWriter() override; @@ -38,47 +38,47 @@ class AMReX_warpxBTDWriter final : public AMReX_warpxWriter /* */ bool AllocatePtlProperties(openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const override; + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const override; void SetupConstant(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np) const override; + const unsigned long long& np) const override; void CloseStep(int ts) override; - + void Init(openPMD::Access access) override; // // in warpx, btd ignores "geom" and takes in a single geometry "full_BTD_snapshot" // and loops over all levels in the vector "geom" // might as well supply equivalent vector of full_BTD_snapshot instead of both "geom" and "full_BTD_snapshot" - // + // // TODO -- need to find out whether BTD is only for levels if all levels are using full_BTD_snapshot in warpX // void WriteMesh(const std::vector& varnames, - const amrex::Vector& mf, - const amrex::Vector& geom, - //const int iteration, - const double time ) const override; + const amrex::Vector& mf, + const amrex::Vector& geom, + //const int iteration, + const double time ) const override; // BTD specific: void SetLastFlush() {m_LastFlush = true;}; bool IsLastFlush() {return m_LastFlush;} void AssignPtlOffset(unsigned long long m) { m_AssignedPtlOffset = m; }; - + template void SavePosId_RZ(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset) const; + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset) const; void SetConstantMassCharge(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np, - amrex::ParticleReal const charge, - amrex::ParticleReal const mass) const override; - + const unsigned long long& np, + amrex::ParticleReal const charge, + amrex::ParticleReal const mass) const override; + private: bool m_LastFlush = false; unsigned long long m_AssignedPtlOffset = 0; @@ -88,14 +88,14 @@ class AMReX_warpxBTDWriter final : public AMReX_warpxWriter -AMReX_warpxBTDWriter::AMReX_warpxBTDWriter( - std::vector fieldPMLdirections, - openPMD::IterationEncoding ie, - std::string filetype, - std::string openpmdOptions) +AMReX_warpxBTDWriter::AMReX_warpxBTDWriter( + std::vector fieldPMLdirections, + openPMD::IterationEncoding ie, + std::string filetype, + std::string openpmdOptions) // :AMReX_openPMDWriter("{}", ie, filetype, openpmdOptions) :AMReX_warpxWriter(fieldPMLdirections, ie, filetype, openpmdOptions) -{ +{ m_openPMDDatasetOptions="{ \"resizable\": true }"; if (ie == openPMD::IterationEncoding::variableBased) @@ -107,11 +107,11 @@ AMReX_warpxBTDWriter::AMReX_warpxBTDWriter( AMReX_warpxBTDWriter::~AMReX_warpxBTDWriter() { - + } -void AMReX_warpxBTDWriter::CloseStep(int ts) +void AMReX_warpxBTDWriter::CloseStep(int ts) { if (!m_LastFlush) return; @@ -119,32 +119,32 @@ void AMReX_warpxBTDWriter::CloseStep(int ts) AMReX_openPMDWriter::CloseStep(ts); m_LastFlush = false; // this step is over } - -void AMReX_warpxBTDWriter::Init(openPMD::Access access) + +void AMReX_warpxBTDWriter::Init(openPMD::Access access) { if (m_Series != nullptr) { m_FirstSightOfStep = ! m_Series->iterations.contains( m_CurrentStep ); return; } - m_FirstSightOfStep = true; + m_FirstSightOfStep = true; AMReX_openPMDWriter::Init(access); } bool AMReX_warpxBTDWriter::AllocatePtlProperties(openPMD::ParticleSpecies& currSpecies, - const amrex::Vector& write_real_comp, - const amrex::Vector& real_comp_names, - const amrex::Vector& write_int_comp, - const amrex::Vector& int_comp_names, - const unsigned long long np) const + const amrex::Vector& write_real_comp, + const amrex::Vector& real_comp_names, + const amrex::Vector& write_int_comp, + const amrex::Vector& int_comp_names, + const unsigned long long np) const { //amrex::Print() <<"allocate np="< 0) { //AMReX_openPMDWriter::SetupPos(currSpecies, np + m_AssignedPtlOffset); return AMReX_openPMDWriter::AllocatePtlProperties(currSpecies, write_real_comp, real_comp_names, - write_int_comp, int_comp_names, np + GetGrandOffset()); + write_int_comp, int_comp_names, np + GetGrandOffset()); } - // np == 0 + // np == 0 if ( m_LastFlush && ( m_AssignedPtlOffset == 0 ) ) { // properties were never allocated @@ -157,34 +157,34 @@ bool AMReX_warpxBTDWriter::AllocatePtlProperties(openPMD::ParticleSpecies& currS /* */ void AMReX_warpxBTDWriter::SetupConstant(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np) const + const unsigned long long& np) const { if (m_LastFlush) { - AMReX_openPMDWriter::SetupConstant(currSpecies, np + GetGrandOffset()); - } + AMReX_openPMDWriter::SetupConstant(currSpecies, np + GetGrandOffset()); + } } void AMReX_warpxBTDWriter::SetConstantMassCharge(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np, - amrex::ParticleReal const charge, - amrex::ParticleReal const mass) const + const unsigned long long& np, + amrex::ParticleReal const charge, + amrex::ParticleReal const mass) const { if (m_LastFlush) { BL_PROFILE("SetChargeMass(BTD)"); AMReX_warpxWriter::SetConstantMassCharge(currSpecies, np + GetGrandOffset(), - charge, mass); + charge, mass); } } void AMReX_warpxBTDWriter::WriteMesh(const std::vector& varnames, - const amrex::Vector& mf, - const amrex::Vector& geom, - //const int iteration, - const double time ) const + const amrex::Vector& mf, + const amrex::Vector& geom, + //const int iteration, + const double time ) const { BL_PROFILE("WriteMesh(BTD)"); if (m_FirstSightOfStep) { @@ -215,51 +215,51 @@ void AMReX_warpxBTDWriter::WriteMesh(const std::vector& varnames, template void AMReX_warpxBTDWriter::SavePosId_RZ(PIt& pti, - openPMD::ParticleSpecies& currSpecies, - unsigned long long offset) const -{ + openPMD::ParticleSpecies& currSpecies, + unsigned long long offset) const +{ BL_PROFILE("amrex::openpmd_api::SavePosId( RZ ..)"); - + #if defined(WARPX_DIM_RZ) - const auto& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile - { + const auto& aos = pti.GetArrayOfStructs(); // size = numParticlesOnTile + { // Save positions auto const positionComponents = /*helper::*/getParticlePositionComponentLabels(); { std::shared_ptr z( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p) { delete[] p; } - ); + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p) { delete[] p; } + ); for (auto i = 0; i < numParticleOnTile; i++) - z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} - + z.get()[i] = aos[i].pos(1); // {0: "r", 1: "z"} + std::string const positionComponent = "z"; currSpecies["position"]["z"].storeChunk(z, {offset}, {numParticleOnTile64}); } - + // reconstruct x and y from polar coordinates r, theta auto const& soa = pti.GetStructOfArrays(); amrex::ParticleReal const* theta = soa.GetRealData(PIdx::theta).dataPtr(); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(theta != nullptr, "openPMD: invalid theta pointer."); WARPX_ALWAYS_ASSERT_WITH_MESSAGE(int(soa.GetRealData(PIdx::theta).size()) == numParticleOnTile, - "openPMD: theta and tile size do not match"); + "openPMD: theta and tile size do not match"); { std::shared_ptr< amrex::ParticleReal > x( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); std::shared_ptr< amrex::ParticleReal > y( - new amrex::ParticleReal[numParticleOnTile], - [](amrex::ParticleReal const *p){ delete[] p; } - ); + new amrex::ParticleReal[numParticleOnTile], + [](amrex::ParticleReal const *p){ delete[] p; } + ); for (auto i=0; i fieldPMLdirections, - openPMD::IterationEncoding ie = openPMD::IterationEncoding::variableBased, - std::string filetype = "default", - std::string openpmdOptions = "{}") + openPMD::IterationEncoding ie = openPMD::IterationEncoding::variableBased, + std::string filetype = "default", + std::string openpmdOptions = "{}") :AMReX_openPMDWriter("{}", ie, filetype, openpmdOptions), m_fieldPMLdirections(std::move(fieldPMLdirections)) {}; - // + // // fields related funcitons // - void SetupMeshComp (openPMD::Mesh& mesh, - const amrex::Geometry& full_geom, - amrex::MultiFab const& mf, - const amrex::openpmd_api::AMReX_VarNameParser& varName) const override; + void SetupMeshComp (openPMD::Mesh& mesh, + const amrex::Geometry& full_geom, + amrex::MultiFab const& mf, + const amrex::openpmd_api::AMReX_VarNameParser& varName) const override; void SetupFields (openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom) const override; + amrex::Geometry& full_geom) const override; inline std::vector< std::string > getParticlePositionComponentLabels() const override; virtual void SetConstantMassCharge(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np, - amrex::ParticleReal const charge, - amrex::ParticleReal const mass) const; + const unsigned long long& np, + amrex::ParticleReal const charge, + amrex::ParticleReal const mass) const; void SetParticleSpecieAttributes(openPMD::ParticleSpecies& currSpecies) const override; - + private: std::vector m_fieldPMLdirections; @@ -53,67 +53,67 @@ private: }; -void AMReX_warpxWriter::SetupMeshComp (openPMD::Mesh& mesh, - const amrex::Geometry& full_geom, - amrex::MultiFab const& mf, - const amrex::openpmd_api::AMReX_VarNameParser& varName) const +void AMReX_warpxWriter::SetupMeshComp (openPMD::Mesh& mesh, + const amrex::Geometry& full_geom, + amrex::MultiFab const& mf, + const amrex::openpmd_api::AMReX_VarNameParser& varName) const { BL_PROFILE("SetupMeshComp(WarpX)"); - + auto mesh_comp = mesh[varName.m_CompName]; amrex::Box const & global_box = full_geom.Domain(); auto global_size = amrex::openpmd_api::helper::getReversedVec(global_box.size() ); - + // - Grid spacing std::vector const grid_spacing = amrex::openpmd_api::helper::getReversedVec(full_geom.CellSize()); mesh.setGridSpacing(grid_spacing); - + // - Global offset std::vector const global_offset = amrex::openpmd_api::helper::getReversedVec(full_geom.ProbLo()); mesh.setGridGlobalOffset(global_offset); - + // - AxisLabels std::vector axis_labels = getFieldAxisLabels(varName); mesh.setAxisLabels(axis_labels); - + #if defined(WARPX_DIM_RZ) auto & warpx = WarpX::GetInstance(); if (curr.m_ThetaMode) { global_size.emplace(global_size.begin(), warpx.ncomps); } #endif - + // Prepare the type of dataset that will be written openPMD::Datatype const datatype = openPMD::determineDatatype(); auto const dataset = openPMD::Dataset(datatype, global_size); mesh.setDataOrder(openPMD::Mesh::DataOrder::C); - + if (varName.m_ThetaMode) { - mesh.setGeometry("thetaMode"); - mesh.setGeometryParameters("m=" + std::to_string(WarpX::n_rz_azimuthal_modes) + ";imag=+"); + mesh.setGeometry("thetaMode"); + mesh.setGeometryParameters("m=" + std::to_string(WarpX::n_rz_azimuthal_modes) + ";imag=+"); } - + mesh.setAttribute("fieldSmoothing", "none"); mesh_comp.resetDataset(dataset); - + amrex::openpmd_api::helper::setOpenPMDUnit( mesh, varName.m_FieldName ); - - auto relative_cell_pos = amrex::openpmd_api::helper::getRelativeCellPosition(mf); // AMReX Fortran index order + + auto relative_cell_pos = amrex::openpmd_api::helper::getRelativeCellPosition(mf); // AMReX Fortran index order std::reverse( relative_cell_pos.begin(), relative_cell_pos.end() ); // now in C order - mesh_comp.setPosition( relative_cell_pos ); + mesh_comp.setPosition( relative_cell_pos ); } void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes, - amrex::Geometry& full_geom) const -{ + amrex::Geometry& full_geom) const +{ // meta data for ED-PIC extension - auto const period = full_geom.periodicity(); - + auto const period = full_geom.periodicity(); + std::vector fieldBoundary(6, "reflecting"); std::vector particleBoundary(6, "absorbing"); fieldBoundary.resize(AMREX_SPACEDIM * 2); particleBoundary.resize(AMREX_SPACEDIM * 2); - + #if AMREX_SPACEDIM != 3 fieldBoundary.resize(4); particleBoundary.resize(4); @@ -121,10 +121,10 @@ void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes if (m_fieldPMLdirections.size() >= fieldBoundary.size()) { for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i) - if (m_fieldPMLdirections.at(i)) - fieldBoundary.at(i) = "open"; - } - + if (m_fieldPMLdirections.at(i)) + fieldBoundary.at(i) = "open"; + } + for (auto i = 0u; i < fieldBoundary.size() / 2u; ++i) if (period.isPeriodic(i)) { fieldBoundary.at(2u * i) = "periodic"; @@ -132,8 +132,8 @@ void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes particleBoundary.at(2u * i) = "periodic"; particleBoundary.at(2u * i + 1u) = "periodic"; } - - + + meshes.setAttribute("fieldSolver", []() { switch (WarpX::electromagnetic_solver_id) { case ElectromagneticSolverAlgo::Yee : @@ -147,15 +147,15 @@ void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes } }()) ; - + meshes.setAttribute("fieldBoundary", fieldBoundary); meshes.setAttribute("particleBoundary", particleBoundary); - + meshes.setAttribute("currentSmoothing", []() { if (WarpX::use_filter) return "Binomial"; else return "none"; }()); - + if (WarpX::use_filter) meshes.setAttribute("currentSmoothingParameters", []() { std::stringstream ss; @@ -166,7 +166,7 @@ void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes #if (AMREX_SPACEDIM == 3) ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1]; ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2]; -#endif +#endif #if defined(WARPX_DIM_3D) ss << ";numPasses_y=" << WarpX::filter_npass_each_dir[1]; ss << ";numPasses_z=" << WarpX::filter_npass_each_dir[2]; @@ -178,14 +178,14 @@ void AMReX_warpxWriter::SetupFields (openPMD::Container< openPMD::Mesh >& meshes std::string currentSmoothingParameters = ss.str(); return currentSmoothingParameters; }()); - + meshes.setAttribute("chargeCorrection", []() { - if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check + if (WarpX::do_dive_cleaning) return "hyperbolic"; // TODO or "spectral" or something? double-check else return "none"; }()); - + if (WarpX::do_dive_cleaning) - meshes.setAttribute("chargeCorrectionParameters", "period=1"); + meshes.setAttribute("chargeCorrectionParameters", "period=1"); } @@ -205,13 +205,13 @@ std::vector< std::string > AMReX_warpxWriter::getFieldAxisLabels ( const amrex:: vs const cartAxisLabels{"x", "z"}; // x varies fastest in memory vs const axisLabels = varName.m_ThetaMode ? circAxisLabels : cartAxisLabels; #elif defined(WARPX_DIM_3D) - vs const axisLabels{"x", "y", "z"}; // x varies fastest in memory + vs const axisLabels{"x", "y", "z"}; // x varies fastest in memory #else //error Unknown WarpX dimensionality. vs const axisLabels{"w", "k", "i"}; // default as this is just a test #endif // revert to C order (fastest varying index last) - return {axisLabels.rbegin(), axisLabels.rend()}; + return {axisLabels.rbegin(), axisLabels.rend()}; } // @@ -222,7 +222,7 @@ AMReX_warpxWriter::getParticlePositionComponentLabels() const { using vs = std::vector< std::string >; #if defined(WARPX_DIM_1D_Z) - vs const positionComponents{"z"}; + vs const positionComponents{"z"}; #elif defined(WARPX_DIM_XZ) vs const positionComponents{"x", "z"}; #elif defined(WARPX_DIM_RZ) @@ -245,13 +245,13 @@ AMReX_warpxWriter::getParticlePositionComponentLabels() const // TODO should it be here? void AMReX_warpxWriter::SetConstantMassCharge(openPMD::ParticleSpecies& currSpecies, - const unsigned long long& np, - amrex::ParticleReal const charge, - amrex::ParticleReal const mass) const + const unsigned long long& np, + amrex::ParticleReal const charge, + amrex::ParticleReal const mass) const { auto realType = openPMD::Dataset(openPMD::determineDatatype(), {np}); auto const scalar = openPMD::RecordComponent::SCALAR; - + currSpecies["charge"][scalar].resetDataset( realType ); currSpecies["mass"][scalar].resetDataset( realType ); @@ -266,60 +266,60 @@ void AMReX_warpxWriter::SetConstantMassCharge(openPMD::ParticleSpecies& currSpec currSpecies["mass"].setAttribute( "macroWeighted", 0u ); currSpecies["mass"].setAttribute( "weightingPower", 1.0 ); } - + // should be called after // - setupPos so records "position" and "id" are defined // - setConstants so record "positonOffset" is defined void AMReX_warpxWriter::SetParticleSpecieAttributes(openPMD::ParticleSpecies& currSpecies) const - { - + { + // ED-PIC extention currSpecies["id"].setAttribute( "macroWeighted", 0u ); currSpecies["id"].setAttribute( "weightingPower", 0.0 ); currSpecies["position"].setAttribute( "macroWeighted", 0u ); currSpecies["position"].setAttribute( "weightingPower", 0.0 ); - + currSpecies["positionOffset"].setAttribute( "macroWeighted", 0u ); - currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); - + currSpecies["positionOffset"].setAttribute( "weightingPower", 0.0 ); + // meta data for ED-PIC extension currSpecies.setAttribute( "particleShape", double( WarpX::noz ) ); - + currSpecies.setAttribute( "particleShapes", [](){ - return std::vector< double >{ + return std::vector< double >{ double(WarpX::nox), #if AMREX_SPACEDIM==3 double(WarpX::noy), #endif double(WarpX::noz) - }; + }; }() ); - + currSpecies.setAttribute( "particlePush", [](){ - switch( WarpX::particle_pusher_algo ) { - case ParticlePusherAlgo::Boris : return "Boris"; - case ParticlePusherAlgo::Vay : return "Vay"; - case ParticlePusherAlgo::HigueraCary : return "HigueraCary"; - default: return "other"; - } + switch( WarpX::particle_pusher_algo ) { + case ParticlePusherAlgo::Boris : return "Boris"; + case ParticlePusherAlgo::Vay : return "Vay"; + case ParticlePusherAlgo::HigueraCary : return "HigueraCary"; + default: return "other"; + } }() ); currSpecies.setAttribute( "particleInterpolation", [](){ - switch( WarpX::field_gathering_algo ) { - case GatheringAlgo::EnergyConserving : return "energyConserving"; - case GatheringAlgo::MomentumConserving : return "momentumConserving"; - default: return "other"; - } + switch( WarpX::field_gathering_algo ) { + case GatheringAlgo::EnergyConserving : return "energyConserving"; + case GatheringAlgo::MomentumConserving : return "momentumConserving"; + default: return "other"; + } }() ); currSpecies.setAttribute( "particleSmoothing", "none" ); currSpecies.setAttribute( "currentDeposition", [](){ - switch( WarpX::current_deposition_algo ) { - case CurrentDepositionAlgo::Esirkepov : return "Esirkepov"; - default: return "directMorseNielson"; - } + switch( WarpX::current_deposition_algo ) { + case CurrentDepositionAlgo::Esirkepov : return "Esirkepov"; + default: return "directMorseNielson"; + } }() ); } - + #endif