Reading changes: Use snapshot attribute

This means that the snapshot attribute, if present, is used for
accessing iterations inside `series.readIterations()`. Fallback to the
old behavior (linear progression through iterations) if the attribute is
not found.

Variable-b. encoding: Allow several (equivalent) iterations per step

This means that a single step can be marked by /data/snapshot to
represent iterations 0,10,20,30 at the same time.
The underlying data is the same, but the API will treat it as 4 times a
different iteration with equivalent content.

Use RW-mode in some places when re-parsing

Skip repeated iterations that occur in Append mode
Before the explicit iteration-step mapping, these were not seen by
reading procedures at all. Now they are, so we skip the second instance.

Better error message when calling readIterations() too late

Author: franzpoeschel

include/openPMD/Iteration.hpp

@@ -28,7 +28,10 @@
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
 
+#include <cstdint>
+#include <deque>
 #include <optional>
+#include <tuple>
 
 namespace openPMD
 {
@@ -282,14 +285,49 @@ class Iteration : public Attributable
     void readGorVBased(std::string const &groupPath, bool beginStep);
     void read_impl(std::string const &groupPath);
 
+    /**
+     * Status after beginning an IO step. Currently includes:
+     * * The advance status (OK, OVER, RANDOMACCESS)
+     * * The opened iterations, in case the snapshot attribute is found
+     */
+    struct BeginStepStatus
+    {
+        using AvailableIterations_t = std::optional<std::deque<uint64_t> >;
+
+        AdvanceStatus stepStatus{};
+        /*
+         * If the iteration attribute `snapshot` is present, the value of that
+         * attribute. Otherwise empty.
+         */
+        AvailableIterations_t iterationsInOpenedStep;
+
+        /*
+         * Most of the time, the AdvanceStatus part of this struct is what we
+         * need, so let's make it easy to access.
+         */
+        inline operator AdvanceStatus() const
+        {
+            return stepStatus;
+        }
+
+        /*
+         * Support for std::tie()
+         */
+        inline operator std::tuple<AdvanceStatus &, AvailableIterations_t &>()
+        {
+            return std::tuple<AdvanceStatus &, AvailableIterations_t &>{
+                stepStatus, iterationsInOpenedStep};
+        }
+    };
+
     /**
      * @brief Begin an IO step on the IO file (or file-like object)
      *        containing this iteration. In case of group-based iteration
      *        layout, this will be the complete Series.
      *
-     * @return AdvanceStatus
+     * @return BeginStepStatus
      */
-    AdvanceStatus beginStep(bool reread);
+    BeginStepStatus beginStep(bool reread);
 
     /**
      * @brief End an IO step on the IO file (or file-like object)

include/openPMD/ReadIterations.hpp

@@ -23,6 +23,8 @@
 #include "openPMD/Iteration.hpp"
 #include "openPMD/Series.hpp"
 
+#include <deque>
+#include <iostream>
 #include <optional>
 
 namespace openPMD
@@ -54,7 +56,8 @@ class SeriesIterator
     using maybe_series_t = std::optional<Series>;
 
     maybe_series_t m_series;
-    iteration_index_t m_currentIteration = 0;
+    std::deque<iteration_index_t> m_iterationsInCurrentStep;
+    uint64_t m_currentIteration{};
 
 public:
     //! construct the end() iterator
@@ -71,6 +74,21 @@ class SeriesIterator
     bool operator!=(SeriesIterator const &other) const;
 
     static SeriesIterator end();
+
+private:
+    inline bool setCurrentIteration()
+    {
+        if (m_iterationsInCurrentStep.empty())
+        {
+            std::cerr << "[ReadIterations] Encountered a step without "
+                         "iterations. Closing the Series."
+                      << std::endl;
+            *this = end();
+            return false;
+        }
+        m_currentIteration = *m_iterationsInCurrentStep.begin();
+        return true;
+    }
 };
 
 /**

include/openPMD/Series.hpp

@@ -37,6 +37,7 @@
 #include <mpi.h>
 #endif
 
+#include <deque>
 #include <map>
 #include <optional>
 #include <set>
@@ -585,8 +586,10 @@ OPENPMD_private
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
      * Iterations/Records/Record Components etc.
+     * If series.iterations contains the attribute `snapshot`, returns its
+     * value.
      */
-    void readGorVBased(bool init = true);
+    std::optional<std::deque<uint64_t> > readGorVBased(bool init = true);
     void readBase();
     std::string iterationFilename(uint64_t i);
 

src/Iteration.cpp

@@ -566,8 +566,9 @@ void Iteration::read_impl(std::string const &groupPath)
     readAttributes(ReadMode::FullyReread);
 }
 
-AdvanceStatus Iteration::beginStep(bool reread)
+auto Iteration::beginStep(bool reread) -> BeginStepStatus
 {
+    BeginStepStatus res;
     using IE = IterationEncoding;
     auto series = retrieveSeries();
     // Initialize file with this to quiet warnings
@@ -583,11 +584,17 @@ AdvanceStatus Iteration::beginStep(bool reread)
         file = &series.get();
         break;
     }
+
     AdvanceStatus status = series.advance(
         AdvanceMode::BEGINSTEP, *file, series.indexOf(*this), *this);
-    if (status != AdvanceStatus::OK)
+    switch (status)
     {
-        return status;
+    case AdvanceStatus::OVER:
+        res.stepStatus = status;
+        return res;
+    case AdvanceStatus::OK:
+    case AdvanceStatus::RANDOMACCESS:
+        break;
     }
 
     // re-read -> new datasets might be available
@@ -606,6 +613,7 @@ AdvanceStatus Iteration::beginStep(bool reread)
             auto oldType = this->IOHandler()->m_frontendAccess;
             auto newType =
                 const_cast<Access *>(&this->IOHandler()->m_frontendAccess);
+            res.iterationsInOpenedStep = series.readGorVBased(false);
             *newType = Access::READ_WRITE;
             series.readGorVBased(false);
             *newType = oldType;
@@ -619,7 +627,8 @@ AdvanceStatus Iteration::beginStep(bool reread)
         }
     }
 
-    return status;
+    res.stepStatus = status;
+    return res;
 }
 
 void Iteration::endStep()
@@ -641,6 +650,7 @@ void Iteration::endStep()
     }
     // @todo filebased check
     series.advance(AdvanceMode::ENDSTEP, *file, series.indexOf(*this), *this);
+    series.get().m_currentlyActiveIterations.clear();
 }
 
 StepStatus Iteration::getStepStatus()