RecordComponent: Properly handle uninitialized datasets (#1316)

* Use a std::optional for BaseRecordComponentData::m_dataset

* Fix the bugs found by this check

Fix the examples, the unfinished_iteratoin_test and the Coretests

* Don't flush in JSONIOHandlerImpl destructor

It's not needed, and it avoids weird situations while recovering from an
error.

* Accept undefined dataset as long as no chunks are written yet

Just ignore and skip the component in that case

examples/7_extended_write_serial.cpp

@@ -83,8 +83,9 @@ int main()
                 {{io::UnitDimension::M, 1}});
             electrons["displacement"]["x"].setUnitSI(1e-6);
             electrons.erase("displacement");
-            electrons["weighting"][io::RecordComponent::SCALAR].makeConstant(
-                1.e-5);
+            electrons["weighting"][io::RecordComponent::SCALAR]
+                .resetDataset({io::Datatype::FLOAT, {1}})
+                .makeConstant(1.e-5);
         }
 
         io::Mesh mesh = cur_it.meshes["lowRez_2D_field"];

examples/7_extended_write_serial.py

@@ -90,7 +90,9 @@
     electrons["displacement"].unit_dimension = {Unit_Dimension.M: 1}
     electrons["displacement"]["x"].unit_SI = 1.e-6
     del electrons["displacement"]
-    electrons["weighting"][SCALAR].make_constant(1.e-5)
+    electrons["weighting"][SCALAR] \
+        .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \
+        .make_constant(1.e-5)
 
     mesh = cur_it.meshes["lowRez_2D_field"]
     mesh.axis_labels = ["x", "y"]

examples/9_particle_write_serial.py

@@ -44,7 +44,9 @@
     # don't like it anymore? remove it with:
     # del electrons["displacement"]
 
-    electrons["weighting"][SCALAR].make_constant(1.e-5)
+    electrons["weighting"][SCALAR] \
+        .reset_dataset(Dataset(np.dtype("float32"), extent=[1])) \
+        .make_constant(1.e-5)
 
     particlePos_x = np.random.rand(234).astype(np.float32)
     particlePos_y = np.random.rand(234).astype(np.float32)

include/openPMD/RecordComponent.tpp

@@ -332,7 +332,13 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
         dCreate.name = rc.m_name;
         dCreate.extent = getExtent();
         dCreate.dtype = getDatatype();
-        dCreate.options = rc.m_dataset.options;
+        if (!rc.m_dataset.has_value())
+        {
+            throw error::WrongAPIUsage(
+                "[RecordComponent] Must specify dataset type and extent before "
+                "using storeChunk() (see RecordComponent::resetDataset()).");
+        }
+        dCreate.options = rc.m_dataset.value().options;
         IOHandler()->enqueue(IOTask(this, dCreate));
     }
     Parameter<Operation::GET_BUFFER_VIEW> getBufferView;

include/openPMD/backend/BaseRecordComponent.hpp

@@ -24,6 +24,8 @@
 #include "openPMD/Error.hpp"
 #include "openPMD/backend/Attributable.hpp"
 
+#include <optional>
+
 // expose private and protected members for invasive testing
 #ifndef OPENPMD_protected
 #define OPENPMD_protected protected:
@@ -39,7 +41,7 @@ namespace internal
         /**
          * The type and extent of the dataset defined by this component.
          */
-        Dataset m_dataset{Datatype::UNDEFINED, {}};
+        std::optional<Dataset> m_dataset;
         /**
          * True if this is defined as a constant record component as specified
          * in the openPMD standard.

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -58,23 +58,7 @@ JSONIOHandlerImpl::JSONIOHandlerImpl(AbstractIOHandler *handler)
     : AbstractIOHandlerImpl(handler)
 {}
 
-JSONIOHandlerImpl::~JSONIOHandlerImpl()
-{
-    // we must not throw in a destructor
-    try
-    {
-        flush();
-    }
-    catch (std::exception const &ex)
-    {
-        std::cerr << "[~JSONIOHandlerImpl] An error occurred: " << ex.what()
-                  << std::endl;
-    }
-    catch (...)
-    {
-        std::cerr << "[~JSONIOHandlerImpl] An error occurred." << std::endl;
-    }
-}
+JSONIOHandlerImpl::~JSONIOHandlerImpl() = default;
 
 std::future<void> JSONIOHandlerImpl::flush()
 {

src/RecordComponent.cpp

@@ -42,7 +42,6 @@ namespace internal
         RecordComponent impl{
             std::shared_ptr<RecordComponentData>{this, [](auto const *) {}}};
         impl.setUnitSI(1);
-        impl.resetDataset(Dataset(Datatype::CHAR, {1}));
     }
 } // namespace internal
 
@@ -71,11 +70,17 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
     auto &rc = get();
     if (written())
     {
+        if (!rc.m_dataset.has_value())
+        {
+            throw error::Internal(
+                "Internal control flow error: Written record component must "
+                "have defined datatype and extent.");
+        }
         if (d.dtype == Datatype::UNDEFINED)
         {
-            d.dtype = rc.m_dataset.dtype;
+            d.dtype = rc.m_dataset.value().dtype;
         }
-        else if (d.dtype != rc.m_dataset.dtype)
+        else if (d.dtype != rc.m_dataset.value().dtype)
         {
             throw std::runtime_error(
                 "Cannot change the datatype of a dataset.");
@@ -99,7 +104,7 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
     rc.m_isEmpty = false;
     if (written())
     {
-        rc.m_dataset.extend(std::move(d.extent));
+        rc.m_dataset.value().extend(std::move(d.extent));
     }
     else
     {
@@ -112,12 +117,28 @@ RecordComponent &RecordComponent::resetDataset(Dataset d)
 
 uint8_t RecordComponent::getDimensionality() const
 {
-    return get().m_dataset.rank;
+    auto &rc = get();
+    if (rc.m_dataset.has_value())
+    {
+        return rc.m_dataset.value().rank;
+    }
+    else
+    {
+        return 1;
+    }
 }
 
 Extent RecordComponent::getExtent() const
 {
-    return get().m_dataset.extent;
+    auto &rc = get();
+    if (rc.m_dataset.has_value())
+    {
+        return rc.m_dataset.value().extent;
+    }
+    else
+    {
+        return {1};
+    }
 }
 
 namespace detail
@@ -149,6 +170,12 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d)
     auto &rc = get();
     if (written())
     {
+        if (!rc.m_dataset.has_value())
+        {
+            throw error::Internal(
+                "Internal control flow error: Written record component must "
+                "have defined datatype and extent.");
+        }
         if (!constant())
         {
             throw std::runtime_error(
@@ -158,30 +185,30 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d)
         }
         if (d.dtype == Datatype::UNDEFINED)
         {
-            d.dtype = rc.m_dataset.dtype;
+            d.dtype = rc.m_dataset.value().dtype;
         }
-        else if (d.dtype != rc.m_dataset.dtype)
+        else if (d.dtype != rc.m_dataset.value().dtype)
         {
             throw std::runtime_error(
                 "Cannot change the datatype of a dataset.");
         }
-        rc.m_dataset.extend(std::move(d.extent));
+        rc.m_dataset.value().extend(std::move(d.extent));
         rc.m_hasBeenExtended = true;
     }
     else
     {
         rc.m_dataset = std::move(d);
     }
 
-    if (rc.m_dataset.extent.size() == 0)
+    if (rc.m_dataset.value().extent.size() == 0)
         throw std::runtime_error("Dataset extent must be at least 1D.");
 
     rc.m_isEmpty = true;
     dirty() = true;
     if (!written())
     {
         switchType<detail::DefaultValue<RecordComponent> >(
-            rc.m_dataset.dtype, *this);
+            rc.m_dataset.value().dtype, *this);
     }
     return *this;
 }
@@ -213,11 +240,23 @@ void RecordComponent::flush(
         /*
          * This catches when a user forgets to use resetDataset.
          */
-        if (rc.m_dataset.dtype == Datatype::UNDEFINED)
+        if (!rc.m_dataset.has_value())
         {
-            throw error::WrongAPIUsage(
-                "[RecordComponent] Must set specific datatype (Use "
-                "resetDataset call).");
+            // The check for !written() is technically not needed, just
+            // defensive programming against internal bugs that go on us.
+            if (!written() && rc.m_chunks.empty())
+            {
+                // No data written yet, just accessed the object so far without
+                // doing anything
+                // Just do nothing and skip this record component.
+                return;
+            }
+            else
+            {
+                throw error::WrongAPIUsage(
+                    "[RecordComponent] Must specify dataset type and extent "
+                    "before flushing (see RecordComponent::resetDataset()).");
+            }
         }
         if (!written())
         {
@@ -243,7 +282,7 @@ void RecordComponent::flush(
                 dCreate.name = name;
                 dCreate.extent = getExtent();
                 dCreate.dtype = getDatatype();
-                dCreate.options = rc.m_dataset.options;
+                dCreate.options = rc.m_dataset.value().options;
                 IOHandler()->enqueue(IOTask(this, dCreate));
             }
         }
@@ -262,7 +301,7 @@ void RecordComponent::flush(
             else
             {
                 Parameter<Operation::EXTEND_DATASET> pExtend;
-                pExtend.extent = rc.m_dataset.extent;
+                pExtend.extent = rc.m_dataset.value().extent;
                 IOHandler()->enqueue(IOTask(this, std::move(pExtend)));
                 rc.m_hasBeenExtended = false;
             }

src/backend/BaseRecordComponent.cpp

@@ -35,13 +35,29 @@ BaseRecordComponent &BaseRecordComponent::resetDatatype(Datatype d)
             "A Records Datatype can not (yet) be changed after it has been "
             "written.");
 
-    get().m_dataset.dtype = d;
+    auto &rc = get();
+    if (rc.m_dataset.has_value())
+    {
+        rc.m_dataset.value().dtype = d;
+    }
+    else
+    {
+        rc.m_dataset = Dataset{d, {1}};
+    }
     return *this;
 }
 
 Datatype BaseRecordComponent::getDatatype() const
 {
-    return get().m_dataset.dtype;
+    auto &rc = get();
+    if (rc.m_dataset.has_value())
+    {
+        return rc.m_dataset.value().dtype;
+    }
+    else
+    {
+        return Datatype::UNDEFINED;
+    }
 }
 
 bool BaseRecordComponent::constant() const
@@ -54,8 +70,12 @@ ChunkTable BaseRecordComponent::availableChunks()
     auto &rc = get();
     if (rc.m_isConstant)
     {
-        Offset offset(rc.m_dataset.extent.size(), 0);
-        return ChunkTable{{std::move(offset), rc.m_dataset.extent}};
+        if (!rc.m_dataset.has_value())
+        {
+            return ChunkTable{};
+        }
+        Offset offset(rc.m_dataset.value().extent.size(), 0);
+        return ChunkTable{{std::move(offset), rc.m_dataset.value().extent}};
     }
     containingIteration().open();
     Parameter<Operation::AVAILABLE_CHUNKS> param;

src/backend/PatchRecordComponent.cpp

@@ -67,7 +67,15 @@ uint8_t PatchRecordComponent::getDimensionality() const
 
 Extent PatchRecordComponent::getExtent() const
 {
-    return get().m_dataset.extent;
+    auto &rc = get();
+    if (rc.m_dataset.has_value())
+    {
+        return rc.m_dataset.value().extent;
+    }
+    else
+    {
+        return {1};
+    }
 }
 
 PatchRecordComponent::PatchRecordComponent() : BaseRecordComponent{nullptr}
@@ -94,13 +102,32 @@ void PatchRecordComponent::flush(
     }
     else
     {
+        if (!rc.m_dataset.has_value())
+        {
+            // The check for !written() is technically not needed, just
+            // defensive programming against internal bugs that go on us.
+            if (!written() && rc.m_chunks.empty())
+            {
+                // No data written yet, just accessed the object so far without
+                // doing anything
+                // Just do nothing and skip this record component.
+                return;
+            }
+            else
+            {
+                throw error::WrongAPIUsage(
+                    "[PatchRecordComponent] Must specify dataset type and "
+                    "extent before flushing (see "
+                    "RecordComponent::resetDataset()).");
+            }
+        }
         if (!written())
         {
             Parameter<Operation::CREATE_DATASET> dCreate;
             dCreate.name = name;
             dCreate.extent = getExtent();
             dCreate.dtype = getDatatype();
-            dCreate.options = rc.m_dataset.options;
+            dCreate.options = rc.m_dataset.value().options;
             IOHandler()->enqueue(IOTask(this, dCreate));
         }