Backend additions

1) New streaming status: RANDOM_ACCESS, for non-streaming situations
2) Variable attributes, to be written only if the backend has support
   for steps

Author: franzpoeschel

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -1156,13 +1156,6 @@ namespace detail
          */
         void invalidateVariablesMap();
 
-    private:
-        ADIOS2IOHandlerImpl *m_impl;
-        std::optional<adios2::Engine> m_engine; //! ADIOS engine
-        /**
-         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
-         */
-        std::string m_engineType;
         /*
          * streamStatus is NoStream for file-based ADIOS engines.
          * This is relevant for the method BufferedActions::requireActiveStep,
@@ -1254,6 +1247,14 @@ namespace detail
         };
         StreamStatus streamStatus = StreamStatus::OutsideOfStep;
 
+    private:
+        ADIOS2IOHandlerImpl *m_impl;
+        std::optional<adios2::Engine> m_engine; //! ADIOS engine
+        /**
+         * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
+         */
+        std::string m_engineType;
+
         /**
          * See documentation for StreamStatus::Parsing.
          * Will be set true under the circumstance described there in order to

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -252,8 +252,10 @@ class AbstractIOHandlerImpl
      * The advance mode is determined by parameters.mode.
      * The return status code shall be stored as parameters.status.
      */
-    virtual void advance(Writable *, Parameter<Operation::ADVANCE> &)
-    {}
+    virtual void advance(Writable *, Parameter<Operation::ADVANCE> &parameters)
+    {
+        *parameters.status = AdvanceStatus::RANDOMACCESS;
+    }
 
     /** Close an openPMD group.
      *
@@ -488,8 +490,13 @@ class AbstractIOHandlerImpl
      * datatype parameters.dtype. Any existing attribute with the same name
      * should be overwritten. If possible, only the value should be changed if
      * the datatype stays the same. The attribute should be written to physical
-     * storage after the operation completes successfully. All datatypes of
-     * Datatype should be supported in a type-safe way.
+     * storage after the operation completes successfully. If the parameter
+     * changesOverSteps is true, then the attribute must be able to hold
+     * different values across IO steps. If the backend does not support IO
+     * steps in such a way, the attribute should not be written. (IO steps are
+     * an optional backend feature and the frontend must implement fallback
+     * measures in such a case) All datatypes of Datatype should be supported in
+     * a type-safe way.
      */
     virtual void
     writeAttribute(Writable *, Parameter<Operation::WRITE_ATT> const &) = 0;

include/openPMD/IO/IOTask.hpp

@@ -537,6 +537,7 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_ATT>
         : AbstractParameter()
         , name(p.name)
         , dtype(p.dtype)
+        , changesOverSteps(p.changesOverSteps)
         , resource(p.resource)
     {}
 
@@ -548,6 +549,13 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::WRITE_ATT>
 
     std::string name = "";
     Datatype dtype = Datatype::UNDEFINED;
+    /*
+     * If true, this attribute changes across IO steps.
+     * It should only be written in backends that support IO steps,
+     * otherwise writing should be skipped.
+     * The frontend is responsible for handling both situations.
+     */
+    bool changesOverSteps = false;
     Attribute::resource resource;
 };
 

include/openPMD/Streaming.hpp

@@ -20,7 +20,8 @@ namespace openPMD
 enum class AdvanceStatus : unsigned char
 {
     OK, /* stream goes on */
-    OVER /* stream is over */
+    OVER, /* stream is over */
+    RANDOMACCESS /* there is no stream, it will never be over */
 };
 
 /**

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -815,6 +815,11 @@ void ADIOS2IOHandlerImpl::writeAttribute(
     switch (attributeLayout())
     {
     case AttributeLayout::ByAdiosAttributes:
+        if (parameters.changesOverSteps)
+        {
+            // cannot do this
+            return;
+        }
         switchType<detail::OldAttributeWriter>(
             parameters.dtype, this, writable, parameters);
         break;
@@ -829,6 +834,13 @@ void ADIOS2IOHandlerImpl::writeAttribute(
         auto prefix = filePositionToString(pos);
 
         auto &filedata = getFileData(file, IfFileNotOpen::ThrowError);
+        if (parameters.changesOverSteps &&
+            filedata.streamStatus ==
+                detail::BufferedActions::StreamStatus::NoStream)
+        {
+            // cannot do this
+            return;
+        }
         filedata.requireActiveStep();
         filedata.invalidateAttributesMap();
         m_dirty.emplace(std::move(file));
@@ -2800,6 +2812,7 @@ namespace detail
                     "[ADIOS2] Operation requires active step but no step is "
                     "left.");
             case AdvanceStatus::OK:
+            case AdvanceStatus::RANDOMACCESS:
                 // pass
                 break;
             }
@@ -3003,7 +3016,7 @@ namespace detail
             m_IO.DefineAttribute<bool_representation>(
                 ADIOS2Defaults::str_usesstepsAttribute, 0);
             flush({FlushLevel::UserFlush}, /* writeAttributes = */ false);
-            return AdvanceStatus::OK;
+            return AdvanceStatus::RANDOMACCESS;
         }
 
         /*

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -1129,6 +1129,11 @@ template <typename ChildClass>
 void CommonADIOS1IOHandlerImpl<ChildClass>::writeAttribute(
     Writable *writable, Parameter<Operation::WRITE_ATT> const &parameters)
 {
+    if (parameters.changesOverSteps)
+    {
+        // cannot do this
+        return;
+    }
     if (m_handler->m_backendAccess == Access::READ_ONLY)
         throw std::runtime_error(
             "[ADIOS1] Writing an attribute in a file opened as read only is "

src/IO/HDF5/HDF5IOHandler.cpp

@@ -1328,6 +1328,11 @@ void HDF5IOHandlerImpl::writeDataset(
 void HDF5IOHandlerImpl::writeAttribute(
     Writable *writable, Parameter<Operation::WRITE_ATT> const &parameters)
 {
+    if (parameters.changesOverSteps)
+    {
+        // cannot do this
+        return;
+    }
     if (m_handler->m_backendAccess == Access::READ_ONLY)
         throw std::runtime_error(
             "[HDF5] Writing an attribute in a file opened as read only is not "

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -797,6 +797,11 @@ void JSONIOHandlerImpl::writeDataset(
 void JSONIOHandlerImpl::writeAttribute(
     Writable *writable, Parameter<Operation::WRITE_ATT> const &parameter)
 {
+    if (parameter.changesOverSteps)
+    {
+        // cannot do this
+        return;
+    }
     if (m_handler->m_backendAccess == Access::READ_ONLY)
     {
         throw std::runtime_error(