From d35f7534b7f85e8a45ee7d2c01f08c9f49bdb468 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Franz=20P=C3=B6schel?= Date: Mon, 13 Jan 2025 16:13:28 +0100 Subject: [PATCH] variableBased as default in ADIOS2 --- examples/5_write_parallel.py | 5 +- include/openPMD/Series.hpp | 10 ++ src/IO/ADIOS/ADIOS2IOHandler.cpp | 1 + src/Series.cpp | 119 +++++++++++------- test/Files_SerialIO/close_and_reopen_test.cpp | 5 +- test/ParallelIOTest.cpp | 5 +- test/SerialIOTest.cpp | 2 +- 7 files changed, 98 insertions(+), 49 deletions(-) diff --git a/examples/5_write_parallel.py b/examples/5_write_parallel.py index 8574c1d66e..64ae652348 100644 --- a/examples/5_write_parallel.py +++ b/examples/5_write_parallel.py @@ -56,8 +56,7 @@ # in streaming setups, e.g. an iteration cannot be opened again once # it has been closed. # `Series.iterations` can be directly accessed in random-access workflows. - series.iterations[1].open() - mymesh = series.iterations[1]. \ + mymesh = series.write_iterations()[1]. \ meshes["mymesh"] # example 1D domain decomposition in first index @@ -92,7 +91,7 @@ # The iteration can be closed in order to help free up resources. # The iteration's content will be flushed automatically. # An iteration once closed cannot (yet) be reopened. - series.iterations[1].close() + series.write_iterations()[1].close() if 0 == comm.rank: print("Dataset content has been fully written to disk") diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 7cbac21f2d..f0487fe238 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -68,6 +68,11 @@ class Series; namespace internal { + enum class default_or_explicit : bool + { + default_, + explicit_ + }; /** * @brief Data members for Series. Pinned at one memory location. * @@ -183,6 +188,8 @@ namespace internal * The iteration encoding used in this series. */ IterationEncoding m_iterationEncoding{}; + default_or_explicit m_iterationEncodingSetExplicitly = + default_or_explicit::default_; /** * Detected IO format (backend). */ @@ -969,6 +976,9 @@ OPENPMD_private */ void flushStep(bool doFlush); + Series &setIterationEncoding_internal( + IterationEncoding iterationEncoding, internal::default_or_explicit); + /* * Returns the current content of the /data/snapshot attribute. * (We could also add this to the public API some time) diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 76ac1b47b7..b209a18f9f 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -1581,6 +1581,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( auto IO = adios.DeclareIO("PreparseSnapshots"); // @todo check engine type IO.SetEngine(realEngineType()); + IO.SetParameter("StreamReader", "ON"); // this be for BP4 auto engine = IO.Open(fullPath(*file), adios2::Mode::Read); auto status = engine.BeginStep(); auto type = detail::attributeInfo(IO, name, /* verbose = */ true); diff --git a/src/Series.cpp b/src/Series.cpp index 2d631e3221..2dd4efc888 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -581,45 +581,7 @@ IterationEncoding Series::iterationEncoding() const Series &Series::setIterationEncoding(IterationEncoding ie) { - auto &series = get(); - if (series.m_deferred_initialization) - { - runDeferredInitialization(); - } - if (written()) - throw std::runtime_error( - "A files iterationEncoding can not (yet) be changed after it has " - "been written."); - - series.m_iterationEncoding = ie; - switch (ie) - { - case IterationEncoding::fileBased: - setIterationFormat(series.m_name); - setAttribute("iterationEncoding", std::string("fileBased")); - // This checks that the name contains the expansion pattern - // (e.g. %T) and parses it - if (series.m_filenamePadding < 0) - { - if (!reparseExpansionPattern(series.m_name)) - { - throw error::WrongAPIUsage( - "For fileBased formats the iteration expansion pattern " - "%T must " - "be included in the file name"); - } - } - break; - case IterationEncoding::groupBased: - setIterationFormat(BASEPATH); - setAttribute("iterationEncoding", std::string("groupBased")); - break; - case IterationEncoding::variableBased: - setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); - setAttribute("iterationEncoding", std::string("variableBased")); - break; - } - IOHandler()->setIterationEncoding(ie); + setIterationEncoding_internal(ie, internal::default_or_explicit::default_); return *this; } @@ -1170,7 +1132,9 @@ Given file pattern: ')END" setWritten(false, Attributable::EnqueueAsynchronously::No); initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, + series.m_iterationEncodingSetExplicitly); setWritten(true, Attributable::EnqueueAsynchronously::No); } @@ -1186,12 +1150,14 @@ Given file pattern: ')END" } case Access::CREATE: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); break; } case Access::APPEND: { initDefaults(input->iterationEncoding); - setIterationEncoding(input->iterationEncoding); + setIterationEncoding_internal( + input->iterationEncoding, series.m_iterationEncodingSetExplicitly); if (input->iterationEncoding != IterationEncoding::fileBased) { break; @@ -1857,7 +1823,8 @@ void Series::readOneIterationFileBased(std::string const &filePath) "Unknown iterationEncoding: " + encoding); auto old_written = written(); setWritten(false, Attributable::EnqueueAsynchronously::No); - setIterationEncoding(encoding_out); + setIterationEncoding_internal( + encoding_out, internal::default_or_explicit::explicit_); setWritten(old_written, Attributable::EnqueueAsynchronously::Yes); } else @@ -2645,6 +2612,59 @@ void Series::flushStep(bool doFlush) series.m_wroteAtLeastOneIOStep = true; } +Series &Series::setIterationEncoding_internal( + IterationEncoding ie, internal::default_or_explicit doe) +{ + auto &series = get(); + switch (doe) + { + case internal::default_or_explicit::default_: + case internal::default_or_explicit::explicit_: + // mark this option as set explicitly by the user + series.m_iterationEncodingSetExplicitly = doe; + break; + } + if (series.m_deferred_initialization) + { + runDeferredInitialization(); + } + if (written()) + throw std::runtime_error( + "A files iterationEncoding can not (yet) be changed after it has " + "been written."); + + series.m_iterationEncoding = ie; + switch (ie) + { + case IterationEncoding::fileBased: + setIterationFormat(series.m_name); + setAttribute("iterationEncoding", std::string("fileBased")); + // This checks that the name contains the expansion pattern + // (e.g. %T) and parses it + if (series.m_filenamePadding < 0) + { + if (!reparseExpansionPattern(series.m_name)) + { + throw error::WrongAPIUsage( + "For fileBased formats the iteration expansion pattern " + "%T must " + "be included in the file name"); + } + } + break; + case IterationEncoding::groupBased: + setIterationFormat(BASEPATH); + setAttribute("iterationEncoding", std::string("groupBased")); + break; + case IterationEncoding::variableBased: + setIterationFormat(auxiliary::replace_first(basePath(), "/%T/", "")); + setAttribute("iterationEncoding", std::string("variableBased")); + break; + } + IOHandler()->setIterationEncoding(ie); + return *this; +} + auto Series::openIterationIfDirty(IterationIndex_t index, Iteration &iteration) -> IterationOpened { @@ -2939,6 +2959,8 @@ void Series::parseJsonOptions(TracingJSON &options, ParsedInput &input) options, "iteration_encoding", iterationEncoding); if (!iterationEncoding.empty()) { + series.m_iterationEncodingSetExplicitly = + internal::default_or_explicit::explicit_; auto it = ieDescriptors.find(iterationEncoding); if (it != ieDescriptors.end()) { @@ -3190,6 +3212,17 @@ Series::snapshots(std::optional const snapshot_workflow) } } + if (series.m_iterationEncodingSetExplicitly == + internal::default_or_explicit::default_ && + series.m_iterationEncoding == IterationEncoding ::groupBased && + usedSnapshotWorkflow == SnapshotWorkflow::Synchronous && + access::writeOnly(access) && IOHandler()->backendName() == "ADIOS2") + { + setIterationEncoding_internal( + IterationEncoding::variableBased, + internal::default_or_explicit::default_); + } + switch (usedSnapshotWorkflow) { case SnapshotWorkflow::RandomAccess: { diff --git a/test/Files_SerialIO/close_and_reopen_test.cpp b/test/Files_SerialIO/close_and_reopen_test.cpp index dcc1969ded..b4cb52be5e 100644 --- a/test/Files_SerialIO/close_and_reopen_test.cpp +++ b/test/Files_SerialIO/close_and_reopen_test.cpp @@ -181,7 +181,10 @@ auto run_test_groupbased( { std::string filename = "../samples/close_iteration_reopen/groupbased." + ext; - Series series(filename, Access::CREATE, write_cfg); + Series series( + filename, + Access::CREATE, + json::merge(write_cfg, R"({"iteration_encoding": "group_based"})")); { auto it = writeIterations(series)[0]; auto E_x = it.meshes["E"]["x"]; diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 9552c3911e..0a82f14662 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -1213,7 +1213,10 @@ void adios2_streaming(bool variableBasedLayout) Series writeSeries( "../samples/adios2_stream.sst", Access::CREATE, - "adios2.engine.type = \"sst\""); + variableBasedLayout + ? "adios2.engine.type = \"sst\"" + : "adios2.engine.type = \"sst\"\niteration_encoding = " + "\"group_based\""); if (variableBasedLayout) { writeSeries.setIterationEncoding(IterationEncoding::variableBased); diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index b5eace69c1..512c91f45d 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -7027,7 +7027,7 @@ TEST_CASE("unfinished_iteration_test", "[serial]") unfinished_iteration_test( "bp", IterationEncoding::groupBased, - R"({"backend": "adios2"})", + R"({"backend": "adios2", "iteration_encoding": "group_based"})", /* test_linear_access = */ false); #if openPMD_HAS_ADIOS_2_9 unfinished_iteration_test(