Add further safety guards to createDataset and extendDataset tasks

Author: franzpoeschel

include/openPMD/IO/IOTask.hpp

@@ -116,6 +116,13 @@ struct OPENPMDAPI_EXPORT AbstractParameter
         std::string const &currentBackendName,
         std::string const &warningMessage);
 
+    // Used as a tag in some constructors to make callsites explicitly
+    // acknowledge that joined dimensions will not be automatically resolved
+    struct I_dont_want_to_use_joined_dimensions_t
+    {};
+    constexpr static I_dont_want_to_use_joined_dimensions_t
+        I_dont_want_to_use_joined_dimensions{};
+
 protected:
     // avoid object slicing
     // by allow only child classes to use these things for defining their own
@@ -359,7 +366,17 @@ template <>
 struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
     : public AbstractParameter
 {
-    Parameter() = default;
+    Parameter(Dataset const &ds)
+        : extent(ds.extent)
+        , dtype(ds.dtype)
+        , options(ds.options)
+        , joinedDimension(ds.joinedDimension())
+    {}
+
+    // default constructor, but callsites need to explicitly acknowledge that
+    // joined dimensions will not be automatically configured when using it
+    Parameter(I_dont_want_to_use_joined_dimensions_t)
+    {}
     Parameter(Parameter &&) = default;
     Parameter(Parameter const &) = default;
     Parameter &operator=(Parameter &&) = default;

include/openPMD/RecordComponent.tpp

@@ -356,18 +356,14 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer)
     if (!written())
     {
         auto &rc = get();
-        Parameter<Operation::CREATE_DATASET> dCreate;
-        dCreate.name = rc.m_name;
-        dCreate.extent = getExtent();
-        dCreate.dtype = getDatatype();
-        dCreate.joinedDimension = joinedDimension();
         if (!rc.m_dataset.has_value())
         {
             throw error::WrongAPIUsage(
                 "[RecordComponent] Must specify dataset type and extent before "
                 "using storeChunk() (see RecordComponent::resetDataset()).");
         }
-        dCreate.options = rc.m_dataset.value().options;
+        Parameter<Operation::CREATE_DATASET> dCreate(rc.m_dataset.value());
+        dCreate.name = rc.m_name;
         IOHandler()->enqueue(IOTask(this, dCreate));
     }
     Parameter<Operation::GET_BUFFER_VIEW> getBufferView;

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -888,6 +888,57 @@ namespace detail
             {
                 dims.push_back(ext);
             }
+            auto joinedDim = joinedDimension(var.Shape());
+            auto make_runtime_error = [&](char const *message) {
+                std::stringstream s;
+                s << "[ADIOS2IOHandlerImpl::extendDataset()] " << message
+                  << "\nNote: Variable '" << variable << "' has old shape ";
+                auxiliary::write_vec_to_stream(s, var.Shape());
+                if (joinedDim.has_value())
+                {
+                    s << " (joined dimension on index " << *joinedDim << ").";
+                }
+                else
+                {
+                    s << " (no joined dimension)";
+                }
+                s << " and is extended to new shape ";
+                auxiliary::write_vec_to_stream(s, newShape) << ".";
+                return std::runtime_error(s.str());
+            };
+            if (joinedDim.has_value() ||
+                var.ShapeID() == adios2::ShapeID::JoinedArray)
+            {
+                if (!joinedDim.has_value())
+                {
+                    throw make_runtime_error(
+                        "Inconsistent state of variable: Has shape ID "
+                        "JoinedArray, but its shape contains no value "
+                        "adios2::JoinedDim.");
+                }
+                if (newShape.at(*joinedDim) != Dataset::JOINED_DIMENSION)
+                {
+                    throw make_runtime_error(
+                        "Variable was previously configured with a joined "
+                        "dimension, so the new dataset extent must keep the "
+                        "joined dimension on that index.");
+                }
+                dims[*joinedDim] = adios2::JoinedDim;
+            }
+            else
+            {
+                for (auto s : newShape)
+                {
+                    if (s == Dataset::JOINED_DIMENSION)
+                    {
+                        throw make_runtime_error(
+                            "Variable was not previously configured with a "
+                            "joined dimension, but is now requested to change "
+                            "extent to a joined array.");
+                    }
+                }
+            }
+
             var.SetShape(dims);
         }
 

src/RecordComponent.cpp

@@ -312,12 +312,9 @@ void RecordComponent::flush(
             }
             else
             {
-                Parameter<Operation::CREATE_DATASET> dCreate;
+                Parameter<Operation::CREATE_DATASET> dCreate(
+                    rc.m_dataset.value());
                 dCreate.name = name;
-                dCreate.extent = getExtent();
-                dCreate.dtype = getDatatype();
-                dCreate.options = rc.m_dataset.value().options;
-                dCreate.joinedDimension = joinedDimension();
                 IOHandler()->enqueue(IOTask(this, dCreate));
             }
         }

src/Series.cpp

@@ -415,7 +415,8 @@ void Series::flushRankTable()
         {
             return;
         }
-        Parameter<Operation::CREATE_DATASET> param;
+        Parameter<Operation::CREATE_DATASET> param(
+            AbstractParameter::I_dont_want_to_use_joined_dimensions);
         param.name = "rankTable";
         param.dtype = Datatype::CHAR;
         param.extent = {uint64_t(size), uint64_t(maxSize)};

src/backend/BaseRecordComponent.cpp

@@ -21,6 +21,7 @@
 #include "openPMD/backend/BaseRecordComponent.hpp"
 #include "openPMD/Error.hpp"
 #include "openPMD/Iteration.hpp"
+#include <optional>
 
 namespace openPMD
 {
@@ -75,7 +76,7 @@ std::optional<size_t> BaseRecordComponent::joinedDimension() const
     }
     else
     {
-        return false;
+        return std::nullopt;
     }
 }