Reading changes: Use snapshot attribute

This means that the snapshot attribute, if present, is used for
accessing iterations inside `series.readIterations()`. Fallback to the
old behavior (linear progression through iterations) if the attribute is
not found.

Variable-b. encoding: Allow several (equivalent) iterations per step

This means that a single step can be marked by /data/snapshot to
represent iterations 0,10,20,30 at the same time.
The underlying data is the same, but the API will treat it as 4 times a
different iteration with equivalent content.

Avoid const_cast by introducing a parsing state and use that when
re-parsing.

Skip repeated iterations that occur in Append mode
Before the explicit iteration-step mapping, these were not seen by
reading procedures at all. Now they are, so we skip the second instance.

Better error message when calling readIterations() too late

This commit includes some refactoring

1. Remove recursion of operator++(), this leads to constant memory usage
   rather than filling the stack at some point
2. Extract subroutines from operator++()
3. Steal some refactoring that solved some bugs on topic-read-leniently, so it stands
to reason that we should apply it here already

Author: franzpoeschel

include/openPMD/IO/AbstractIOHandler.hpp

@@ -121,6 +121,28 @@ namespace internal
     FlushParams const defaultFlushParams{};
 
     struct ParsedFlushParams;
+
+    /**
+     * Some parts of the openPMD object model are read-only when accessing
+     * a Series in Access::READ_ONLY mode, notably Containers and Attributes.
+     * They are filled at parse time and not modified afterwards.
+     * Such state-changing operations are hence allowed under either of two
+     * conditions:
+     * 1) The Series is opened in an open mode that allows writing in any way.
+     *    (Currently any but Access::READ_ONLY).
+     * 2) The Series is in Parsing state. This way, modifying the open mode
+     *    during parsing can be avoided.
+     */
+    enum class SeriesStatus : unsigned char
+    {
+        Default, ///< Mutability of objects in the openPMD object model is
+                 ///< determined by the open mode (Access enum), normal state in
+                 ///< which the user interacts with the Series.
+        Parsing ///< All objects in the openPMD object model are temporarily
+                ///< mutable to allow inserting newly-parsed data.
+                ///< Special state only active while internal routines are
+                ///< running.
+    };
 } // namespace internal
 
 /** Interface for communicating between logical and physically persistent data.
@@ -192,6 +214,7 @@ class AbstractIOHandler
     // why do these need to be separate?
     Access const m_backendAccess;
     Access const m_frontendAccess;
+    internal::SeriesStatus m_seriesStatus = internal::SeriesStatus::Default;
     std::queue<IOTask> m_work;
 }; // AbstractIOHandler
 

include/openPMD/Iteration.hpp

@@ -28,7 +28,10 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
 
+#include <cstdint>
+#include <deque>
 #include <optional>
+#include <tuple>
 
 namespace openPMD
 {
@@ -282,14 +285,57 @@ class Iteration : public Attributable
     void readGorVBased(std::string const &groupPath, bool beginStep);
     void read_impl(std::string const &groupPath);
 
+    /**
+     * Status after beginning an IO step. Currently includes:
+     * * The advance status (OK, OVER, RANDOMACCESS)
+     * * The opened iterations, in case the snapshot attribute is found
+     */
+    struct BeginStepStatus
+    {
+        using AvailableIterations_t = std::optional<std::deque<uint64_t> >;
+
+        AdvanceStatus stepStatus{};
+        /*
+         * If the iteration attribute `snapshot` is present, the value of that
+         * attribute. Otherwise empty.
+         */
+        AvailableIterations_t iterationsInOpenedStep;
+
+        /*
+         * Most of the time, the AdvanceStatus part of this struct is what we
+         * need, so let's make it easy to access.
+         */
+        inline operator AdvanceStatus() const
+        {
+            return stepStatus;
+        }
+
+        /*
+         * Support for std::tie()
+         */
+        inline operator std::tuple<AdvanceStatus &, AvailableIterations_t &>()
+        {
+            return std::tuple<AdvanceStatus &, AvailableIterations_t &>{
+                stepStatus, iterationsInOpenedStep};
+        }
+    };
+
     /**
      * @brief Begin an IO step on the IO file (or file-like object)
      *        containing this iteration. In case of group-based iteration
      *        layout, this will be the complete Series.
      *
-     * @return AdvanceStatus
+     * @return BeginStepStatus
+     */
+    BeginStepStatus beginStep(bool reread);
+
+    /*
+     * Iteration-independent variant for beginStep().
+     * Useful in group-based iteration encoding where the Iteration will only
+     * be known after opening the step.
      */
-    AdvanceStatus beginStep(bool reread);
+    static BeginStepStatus
+    beginStep(std::optional<Iteration> thisObject, Series &series, bool reread);
 
     /**
      * @brief End an IO step on the IO file (or file-like object)

include/openPMD/ReadIterations.hpp

@@ -23,6 +23,8 @@
 #include "openPMD/Iteration.hpp"
 #include "openPMD/Series.hpp"
 
+#include <deque>
+#include <iostream>
 #include <optional>
 
 namespace openPMD
@@ -54,7 +56,8 @@ class SeriesIterator
     using maybe_series_t = std::optional<Series>;
 
     maybe_series_t m_series;
-    iteration_index_t m_currentIteration = 0;
+    std::deque<iteration_index_t> m_iterationsInCurrentStep;
+    uint64_t m_currentIteration{};
 
 public:
     //! construct the end() iterator
@@ -71,6 +74,39 @@ class SeriesIterator
     bool operator!=(SeriesIterator const &other) const;
 
     static SeriesIterator end();
+
+private:
+    inline bool setCurrentIteration()
+    {
+        if (m_iterationsInCurrentStep.empty())
+        {
+            std::cerr << "[ReadIterations] Encountered a step without "
+                         "iterations. Closing the Series."
+                      << std::endl;
+            *this = end();
+            return false;
+        }
+        m_currentIteration = *m_iterationsInCurrentStep.begin();
+        return true;
+    }
+
+    inline std::optional<uint64_t> peekCurrentIteration()
+    {
+        if (m_iterationsInCurrentStep.empty())
+        {
+            return std::nullopt;
+        }
+        else
+        {
+            return {*m_iterationsInCurrentStep.begin()};
+        }
+    }
+
+    std::optional<SeriesIterator *> nextIterationInStep();
+
+    std::optional<SeriesIterator *> nextStep();
+
+    std::optional<SeriesIterator *> loopBody();
 };
 
 /**

include/openPMD/Series.hpp

@@ -38,6 +38,7 @@
 #endif
 
 #include <cstdint>
+#include <deque>
 #include <map>
 #include <optional>
 #include <set>
@@ -584,8 +585,10 @@ OPENPMD_private
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
      * Iterations/Records/Record Components etc.
+     * If series.iterations contains the attribute `snapshot`, returns its
+     * value.
      */
-    void readGorVBased(bool init = true);
+    std::optional<std::deque<uint64_t> > readGorVBased(bool init = true);
     void readBase();
     std::string iterationFilename(uint64_t i);
 
@@ -636,13 +639,21 @@ OPENPMD_private
         iterations_iterator it,
         Iteration &iteration);
 
+    AdvanceStatus advance(AdvanceMode mode);
+
     /**
      * @brief Called at the end of an IO step to store the iterations defined
      *        in the IO step to the snapshot attribute.
      *
      * @param doFlush If true, flush the IO handler.
      */
     void flushStep(bool doFlush);
+
+    /*
+     * Returns the current content of the /data/snapshot attribute.
+     * (We could also add this to the public API some time)
+     */
+    std::optional<std::vector<uint64_t> > currentSnapshot() const;
 }; // Series
 } // namespace openPMD
 

include/openPMD/backend/Attributable.hpp

@@ -463,7 +463,9 @@ inline bool Attributable::setAttributeImpl(
     internal::attr_value_check(key, value, setAttributeMode);
 
     auto &attri = get();
-    if (IOHandler() && Access::READ_ONLY == IOHandler()->m_frontendAccess)
+    if (IOHandler() &&
+        IOHandler()->m_seriesStatus == internal::SeriesStatus::Default &&
+        Access::READ_ONLY == IOHandler()->m_frontendAccess)
     {
         auxiliary::OutOfRangeMsg const out_of_range_msg(
             "Attribute", "can not be set (read-only).");

include/openPMD/backend/Container.hpp

@@ -288,7 +288,9 @@ class Container : public Attributable
             return it->second;
         else
         {
-            if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
+            if (IOHandler()->m_seriesStatus !=
+                    internal::SeriesStatus::Parsing &&
+                Access::READ_ONLY == IOHandler()->m_frontendAccess)
             {
                 auxiliary::OutOfRangeMsg const out_of_range_msg;
                 throw std::out_of_range(out_of_range_msg(key));
@@ -321,7 +323,9 @@ class Container : public Attributable
             return it->second;
         else
         {
-            if (Access::READ_ONLY == IOHandler()->m_frontendAccess)
+            if (IOHandler()->m_seriesStatus !=
+                    internal::SeriesStatus::Parsing &&
+                Access::READ_ONLY == IOHandler()->m_frontendAccess)
             {
                 auxiliary::OutOfRangeMsg out_of_range_msg;
                 throw std::out_of_range(out_of_range_msg(key));

src/Iteration.cpp

@@ -573,49 +573,92 @@ void Iteration::read_impl(std::string const &groupPath)
     readAttributes(ReadMode::FullyReread);
 }
 
-AdvanceStatus Iteration::beginStep(bool reread)
+auto Iteration::beginStep(bool reread) -> BeginStepStatus
 {
-    using IE = IterationEncoding;
+    BeginStepStatus res;
     auto series = retrieveSeries();
+    return beginStep({*this}, series, reread);
+}
+
+auto Iteration::beginStep(
+    std::optional<Iteration> thisObject, Series &series, bool reread)
+    -> BeginStepStatus
+{
+    BeginStepStatus res;
+    using IE = IterationEncoding;
     // Initialize file with this to quiet warnings
     // The following switch is comprehensive
     internal::AttributableData *file = nullptr;
     switch (series.iterationEncoding())
     {
     case IE::fileBased:
-        file = &Attributable::get();
+        if (thisObject.has_value())
+        {
+            file = &static_cast<Attributable &>(*thisObject).get();
+        }
+        else
+        {
+            throw error::Internal(
+                "Advancing a step in file-based iteration encoding is "
+                "iteration-specific.");
+        }
         break;
     case IE::groupBased:
     case IE::variableBased:
         file = &series.get();
         break;
     }
-    AdvanceStatus status = series.advance(
-        AdvanceMode::BEGINSTEP, *file, series.indexOf(*this), *this);
-    if (status != AdvanceStatus::OK)
+
+    AdvanceStatus status;
+    if (thisObject.has_value())
     {
-        return status;
+        status = series.advance(
+            AdvanceMode::BEGINSTEP,
+            *file,
+            series.indexOf(*thisObject),
+            *thisObject);
+    }
+    else
+    {
+        status = series.advance(AdvanceMode::BEGINSTEP);
+    }
+
+    switch (status)
+    {
+    case AdvanceStatus::OVER:
+        res.stepStatus = status;
+        return res;
+    case AdvanceStatus::OK:
+    case AdvanceStatus::RANDOMACCESS:
+        break;
     }
 
     // re-read -> new datasets might be available
-    if (reread &&
+    auto IOHandl = series.IOHandler();
+    if (reread && status != AdvanceStatus::RANDOMACCESS &&
         (series.iterationEncoding() == IE::groupBased ||
          series.iterationEncoding() == IE::variableBased) &&
-        (this->IOHandler()->m_frontendAccess == Access::READ_ONLY ||
-         this->IOHandler()->m_frontendAccess == Access::READ_WRITE))
+        (IOHandl->m_frontendAccess == Access::READ_ONLY ||
+         IOHandl->m_frontendAccess == Access::READ_WRITE))
     {
-        switch (IOHandler()->m_frontendAccess)
+        switch (IOHandl->m_frontendAccess)
         {
         case Access::READ_ONLY:
         case Access::READ_WRITE: {
             bool previous = series.iterations.written();
             series.iterations.written() = false;
-            auto oldType = this->IOHandler()->m_frontendAccess;
-            auto newType =
-                const_cast<Access *>(&this->IOHandler()->m_frontendAccess);
-            *newType = Access::READ_WRITE;
-            series.readGorVBased(false);
-            *newType = oldType;
+            auto oldStatus = IOHandl->m_seriesStatus;
+            IOHandl->m_seriesStatus = internal::SeriesStatus::Parsing;
+            try
+            {
+                res.iterationsInOpenedStep = series.readGorVBased(false);
+            }
+            catch (...)
+            {
+                IOHandl->m_seriesStatus = oldStatus;
+                throw;
+            }
+            IOHandl->m_seriesStatus = oldStatus;
             series.iterations.written() = previous;
             break;
         }
@@ -626,7 +669,8 @@ AdvanceStatus Iteration::beginStep(bool reread)
         }
     }
 
-    return status;
+    res.stepStatus = status;
+    return res;
 }
 
 void Iteration::endStep()
@@ -648,6 +692,7 @@ void Iteration::endStep()
     }
     // @todo filebased check
     series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this), *this);
+    series.get().m_currentlyActiveIterations.clear();
 }
 
 StepStatus Iteration::getStepStatus()
@@ -731,9 +776,8 @@ void Iteration::runDeferredParseAccess()
         }
         auto const &deferred = it.m_deferredParseAccess.value();
 
-        auto oldAccess = IOHandler()->m_frontendAccess;
-        auto newAccess = const_cast<Access *>(&IOHandler()->m_frontendAccess);
-        *newAccess = Access::READ_WRITE;
+        auto oldStatus = IOHandler()->m_seriesStatus;
+        IOHandler()->m_seriesStatus = internal::SeriesStatus::Parsing;
         try
         {
             if (deferred.fileBased)
@@ -750,12 +794,12 @@ void Iteration::runDeferredParseAccess()
         {
             // reset this thing
             it.m_deferredParseAccess = std::optional<DeferredParseAccess>();
-            *newAccess = oldAccess;
+            IOHandler()->m_seriesStatus = oldStatus;
             throw;
         }
         // reset this thing
         it.m_deferredParseAccess = std::optional<DeferredParseAccess>();
-        *newAccess = oldAccess;
+        IOHandler()->m_seriesStatus = oldStatus;
         break;
     }
     case Access::CREATE: