[WIP] Remove adios2.usesteps, set it always to true

This uncovered loads of bugs
franzpoeschel · Oct 19, 2023 · 3f629cc · 3f629cc
1 parent f2db458
commit 3f629cc
Show file tree

Hide file tree

Showing 9 changed files with 170 additions and 144 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1363,16 +1363,18 @@ if(openPMD_BUILD_TESTING)
                         ${MPI_TEST_EXE} ${Python_EXECUTABLE}                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/data00000100.h5         \
-                            --outfile ../samples/git-sample/single_iteration.bp && \
+                            --outfile                                              \
+                                ../samples/git-sample/single_iteration_%T.bp &&    \
                                                                                    \
                         ${MPI_TEST_EXE} ${Python_EXECUTABLE}                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.h5     \
-                            --outfile ../samples/git-sample/thetaMode/data.bp &&   \
+                            --outfile                                              \
+                                ../samples/git-sample/thetaMode/data_%T.bp &&      \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
                             ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
-                            --infile ../samples/git-sample/thetaMode/data.bp       \
+                            --infile ../samples/git-sample/thetaMode/data_%T.bp    \
                             --outfile ../samples/git-sample/thetaMode/data%T.json  \
                         "
                     WORKING_DIRECTORY ${openPMD_RUNTIME_OUTPUT_DIRECTORY}
@@ -1381,17 +1383,17 @@ if(openPMD_BUILD_TESTING)
                 add_test(NAME CLI.pipe.py
                     COMMAND sh -c
                         "${Python_EXECUTABLE}                                      \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/data%T.h5               \
                             --outfile ../samples/git-sample/data%T.bp &&           \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.h5     \
                             --outfile ../samples/git-sample/thetaMode/data%T.bp && \
                                                                                    \
                         ${Python_EXECUTABLE}                                       \
-                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe         \
+                            ${openPMD_RUNTIME_OUTPUT_DIRECTORY}/openpmd-pipe       \
                             --infile ../samples/git-sample/thetaMode/data%T.bp     \
                             --outfile ../samples/git-sample/thetaMode/data%T.json  \
                         "

diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp
@@ -229,6 +229,10 @@ class ADIOS2IOHandlerImpl
      * If the iteration encoding is variableBased, we default to using a group
      * table, since it is the only reliable way to recover currently active
      * groups.
+     * If group-based encoding is used without group table, then
+     * READ_RANDOM_ACCESS is forbidden as it will be unreliable in reporting
+     * currently available data.
+     * Use AbstractIOHandler::m_encoding for this though
      */
     IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
     /**
@@ -434,6 +438,8 @@ namespace ADIOS2Defaults
         "__openPMD_internal/openPMD2_adios2_schema";
     constexpr const_str str_isBoolean = "__is_boolean__";
     constexpr const_str str_activeTablePrefix = "__openPMD_groups";
+    constexpr const_str str_groupBasedWarning =
+        "__openPMD_internal/warning_bugprone_groupbased_encoding";
 } // namespace ADIOS2Defaults
 
 namespace detail
@@ -1064,7 +1070,7 @@ namespace detail
              *    without steps. This is not a workaround since not using steps,
              *    while inefficient in ADIOS2, is something that we support.
              */
-            NoStream,
+            ReadWithoutStream,
             /**
              * Rationale behind this state:
              * When user code opens a Series, series.iterations should contain
@@ -1152,8 +1158,8 @@ namespace detail
         void create_IO();
 
         void configure_IO(ADIOS2IOHandlerImpl &impl);
-        void configure_IO_Read(std::optional<bool> userSpecifiedUsesteps);
-        void configure_IO_Write(std::optional<bool> userSpecifiedUsesteps);
+        void configure_IO_Read();
+        void configure_IO_Write();
     };
 
 } // namespace detail

diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp
@@ -23,6 +23,7 @@
 #include "openPMD/IO/Access.hpp"
 #include "openPMD/IO/Format.hpp"
 #include "openPMD/IO/IOTask.hpp"
+#include "openPMD/IterationEncoding.hpp"
 #include "openPMD/config.hpp"
 
 #if openPMD_HAVE_MPI
@@ -180,6 +181,12 @@ class AbstractIOHandler
 {
     friend class Series;
 
+public:
+    /* @todo access control, remove this from open_file, create_file tasks,
+     *       remove the same field from ADIOS2IOHandlerImpl
+     */
+    IterationEncoding m_encoding = IterationEncoding::groupBased;
+
 private:
     void setIterationEncoding(IterationEncoding encoding)
     {
@@ -193,6 +200,8 @@ class AbstractIOHandler
             // do we really want to have those as const members..?
             *const_cast<Access *>(&m_backendAccess) = Access::CREATE;
         }
+
+        m_encoding = encoding;
     }
 
 public:

diff --git a/include/openPMD/cli/ls.hpp b/include/openPMD/cli/ls.hpp
@@ -110,7 +110,7 @@ namespace cli
             {
                 auto s = Series(
                     argv[1],
-                    Access::READ_ONLY,
+                    Access::READ_LINEAR,
                     R"({"defer_iteration_parsing": true})");
 
                 helper::listSeries(s, true, std::cout);