Step-Based iteration layout (#855)

* Step-based iteration layout

* Add eager parsing test

Datasets wth changing dimensions require re-parsing

* Fully re-read an Iteration in stepping mode

* Remove %V shorthand to select variable-based layout

* Hijack some other tests for variable-based encoding

* Refine re-parsing

When re-parsing a Series, we must keep old handles valid, but we might
need to delete stale map members.

* Test that attributes don't occur in the wrong step

Test still failing

* Specifiy more precisely when to re-read attributes

* Fix ADIOS1 bug: Wrong datatype reported for unitDimension

* Rename __step__ -> snapshot

* Code cleanup and in-code documentation

Author: franzpoeschel

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -32,6 +32,7 @@
 #include "openPMD/auxiliary/Option.hpp"
 #include "openPMD/backend/Writable.hpp"
 #include "openPMD/config.hpp"
+#include "openPMD/IterationEncoding.hpp"
 
 #if openPMD_HAVE_ADIOS2
 #    include <adios2.h>
@@ -224,6 +225,11 @@ class ADIOS2IOHandlerImpl
 
 private:
     adios2::ADIOS m_ADIOS;
+    /*
+     * If the iteration encoding is variableBased, we default to using the
+     * 2021_02_09 schema since it allows mutable attributes.
+     */
+    IterationEncoding m_iterationEncoding = IterationEncoding::groupBased;
     /**
      * The ADIOS2 engine type, to be passed to adios2::IO::SetEngine
      */
@@ -1386,7 +1392,7 @@ friend class ADIOS2IOHandlerImpl;
         }
         catch( ... )
         {
-            std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
+             std::cerr << "[~ADIOS2IOHandler] An error occurred." << std::endl;
         }
     }
 

include/openPMD/IO/AbstractIOHandlerImpl.hpp

@@ -241,6 +241,7 @@ class AbstractIOHandlerImpl
    *
    * The operation should overwrite existing file positions, even when the Writable was already marked written.
    * The path parameters.path may contain multiple levels (e.g. first/second/third/). This path should be relative (i.e. it should not start with a slash "/").
+   * The number of levels may be zero, i.e. parameters.path may be an empty string.
    * The Writables file position should correspond to the complete opened path (i.e. first/second/third/ should be assigned to the Writables file position).
    * The Writable should be marked written when the operation completes successfully.
    */

include/openPMD/IO/IOTask.hpp

@@ -25,6 +25,7 @@
 #include "openPMD/backend/Attribute.hpp"
 #include "openPMD/ChunkInfo.hpp"
 #include "openPMD/Dataset.hpp"
+#include "openPMD/IterationEncoding.hpp"
 
 #include <memory>
 #include <map>
@@ -108,7 +109,8 @@ template<>
 struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_FILE > : public AbstractParameter
 {
     Parameter() = default;
-    Parameter(Parameter const & p) : AbstractParameter(), name(p.name) {}
+    Parameter(Parameter const & p) :
+        AbstractParameter(), name(p.name), encoding(p.encoding) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -118,13 +120,15 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::CREATE_FILE > : public AbstractPa
     }
 
     std::string name = "";
+    IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
 template<>
 struct OPENPMDAPI_EXPORT Parameter< Operation::OPEN_FILE > : public AbstractParameter
 {
     Parameter() = default;
-    Parameter(Parameter const & p) : AbstractParameter(), name(p.name) {}
+    Parameter(Parameter const & p) :
+        AbstractParameter(), name(p.name), encoding(p.encoding) {}
 
     std::unique_ptr< AbstractParameter >
     clone() const override
@@ -134,6 +138,12 @@ struct OPENPMDAPI_EXPORT Parameter< Operation::OPEN_FILE > : public AbstractPara
     }
 
     std::string name = "";
+    /*
+     * The backends might need to ensure availability of certain features
+     * for some iteration encodings, e.g. availability of ADIOS steps for
+     * variableBased encoding.
+     */
+    IterationEncoding encoding = IterationEncoding::groupBased;
 };
 
 template<>

include/openPMD/Iteration.hpp

@@ -156,13 +156,31 @@ class Iteration : public LegacyAttributable
 
     struct DeferredParseAccess
     {
-        std::string index;
+        /**
+         * The group path within /data containing this iteration.
+         * Example: "1" for iteration 1, "" in variable-based iteration
+         * encoding.
+         */
+        std::string path;
+        /**
+         * The iteration index as accessed by the user in series.iterations[i]
+         */
+        uint64_t iteration = 0;
+        /**
+         * If this iteration is part of a Series with file-based layout.
+         * (Group- and variable-based parsing shares the same code logic.)
+         */
         bool fileBased = false;
+        /**
+         * If fileBased == true, the file name (without file path) of the file
+         * containing this iteration.
+         */
         std::string filename;
     };
 
     void flushFileBased(std::string const&, uint64_t);
     void flushGroupBased(uint64_t);
+    void flushVariableBased(uint64_t);
     void flush();
     void deferParseAccess( DeferredParseAccess );
     /*
@@ -177,11 +195,15 @@ class Iteration : public LegacyAttributable
      * allow for those different control flows.
      * Finally, read_impl() is called which contains the common parsing
      * logic for an iteration.
+     * 
+     * reread() reads again an Iteration that has been previously read.
+     * Calling it on an Iteration not yet parsed is an error.
      *
      */
     void read();
+    void reread( std::string const & path );
     void readFileBased( std::string filePath, std::string const & groupPath );
-    void readGroupBased( std::string const & groupPath );
+    void readGorVBased( std::string const & groupPath );
     void read_impl( std::string const & groupPath );
 
     /**

include/openPMD/IterationEncoding.hpp

@@ -31,7 +31,7 @@ namespace openPMD
  */
 enum class IterationEncoding
 {
-    fileBased, groupBased
+    fileBased, groupBased, variableBased
 };
 
 std::ostream&

include/openPMD/RecordComponent.hpp

@@ -286,18 +286,17 @@ class RecordComponent : public BaseRecordComponent
     bool dirtyRecursive() const;
 
 protected:
-    /**
-     * Make sure to parse a RecordComponent only once.
-     */
-    std::shared_ptr< bool > hasBeenRead = std::make_shared< bool >( false );
+
     /**
      * The same std::string that the parent class would pass as parameter to
      * RecordComponent::flush().
      * This is stored only upon RecordComponent::flush() if
      * AbstractIOHandler::flushLevel is set to FlushLevel::SkeletonOnly
      * (for use by the Span<T>-based overload of RecordComponent::storeChunk()).
+     * @todo Merge functionality with ownKeyInParent?
      */
     std::shared_ptr< std::string > m_name = std::make_shared< std::string >();
+
 }; // RecordComponent
 } // namespace openPMD
 

include/openPMD/Series.hpp

@@ -260,6 +260,9 @@ class SeriesImpl : public AttributableImpl
      */
     IterationEncoding iterationEncoding() const;
     /** Set the <A HREF="https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#iterations-and-time-series">encoding style</A> for multiple iterations in this series.
+     * A preview on the <A HREF="https://github.com/openPMD/openPMD-standard/pull/250">openPMD 2.0 variable-based iteration encoding</A> can be activated with this call.
+     * Making full use of the variable-based iteration encoding requires (1) explicit support by the backend (available only in ADIOS2) and (2) use of the openPMD streaming API.
+     * In other backends and without the streaming API, only one iteration/snapshot may be written in the variable-based encoding, making this encoding a good choice for single-snapshot data dumps.
      *
      * @param   iterationEncoding   Desired <A HREF="https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#iterations-and-time-series">encoding style</A> for multiple iterations in this series.
      * @return  Reference to modified series.
@@ -342,13 +345,20 @@ class SeriesImpl : public AttributableImpl
 
     std::unique_ptr< ParsedInput > parseInput(std::string);
     void init(std::shared_ptr< AbstractIOHandler >, std::unique_ptr< ParsedInput >);
-    void initDefaults();
+    void initDefaults( IterationEncoding );
     std::future< void > flush_impl(
         iterations_iterator begin,
         iterations_iterator end,
         FlushLevel );
     void flushFileBased( iterations_iterator begin, iterations_iterator end );
-    void flushGroupBased( iterations_iterator begin, iterations_iterator end );
+    /*
+     * Group-based and variable-based iteration layouts share a lot of logic
+     * (realistically, the variable-based iteration layout only throws out
+     *  one layer in the hierarchy).
+     * As a convention, methods that deal with both layouts are called
+     * .*GorVBased, short for .*GroupOrVariableBased
+     */
+    void flushGorVBased( iterations_iterator begin, iterations_iterator end );
     void flushMeshesPath();
     void flushParticlesPath();
     void readFileBased( );
@@ -357,11 +367,8 @@ class SeriesImpl : public AttributableImpl
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
      * Iterations/Records/Record Components etc.
-     * Re-parsing of objects that have already been parsed is not implemented
-     * as of yet. Such a facility will be required upon implementing things such
-     * as resizable datasets.
      */
-    void readGroupBased( bool init = true );
+    void readGorVBased( bool init = true );
     void readBase();
     std::string iterationFilename( uint64_t i );
     void openIteration( uint64_t index, Iteration iteration );

include/openPMD/backend/Attributable.hpp

@@ -215,7 +215,24 @@ class AttributableImpl
     std::vector< std::string > myPath() const;
 
     void flushAttributes();
-    void readAttributes();
+    enum ReadMode {
+        /**
+         * Don't read an attribute from the backend if it has been previously
+         * read.
+         */
+        IgnoreExisting,
+        /**
+         * Read all the attributes that the backend has to offer and override
+         * if it has been read previously.
+         */
+        OverrideExisting,
+        /**
+         * Remove all attributes that have been read previously and read
+         * everything that the backend currently has to offer.
+         */
+        FullyReread
+    };
+    void readAttributes( ReadMode );
 
     /** Retrieve the value of a floating point Attribute of user-defined precision with ensured type-safety.
      *

include/openPMD/backend/Container.hpp

@@ -23,10 +23,12 @@
 #include "openPMD/backend/Attributable.hpp"
 
 #include <initializer_list>
-#include <type_traits>
-#include <stdexcept>
 #include <map>
+#include <set>
+#include <stdexcept>
 #include <string>
+#include <type_traits>
+#include <utility>
 
 // expose private and protected members for invasive testing
 #ifndef OPENPMD_protected
@@ -106,14 +108,16 @@ class Container : public LegacyAttributable
     static_assert(
         std::is_base_of< AttributableImpl, T >::value,
         "Type of container element must be derived from Writable");
-    using InternalContainer = T_container;
 
     friend class Iteration;
     friend class ParticleSpecies;
     friend class internal::SeriesData;
     friend class SeriesImpl;
     friend class Series;
 
+protected:
+    using InternalContainer = T_container;
+
 public:
     using key_type = typename InternalContainer::key_type;
     using mapped_type = typename InternalContainer::mapped_type;
@@ -321,6 +325,72 @@ class Container : public LegacyAttributable
     }
 
     std::shared_ptr< InternalContainer > m_container;
+
+    /**
+     * This class wraps a Container and forwards operator[]() and at() to it.
+     * It remembers the keys used for accessing. Upon going out of scope, all
+     * keys not yet accessed are removed from the Container.
+     * Note that the container is stored by non-owning reference, thus
+     * requiring that the original Container stay in scope while using this
+     * class.
+     */
+    class EraseStaleEntries
+    {
+        std::set< key_type > m_accessedKeys;
+        /*
+         * Note: Putting a copy here leads to weird bugs due to destructors
+         * being called too eagerly upon destruction.
+         * Should be avoidable by extending the frontend redesign to the
+         * Container class template
+         * (https://github.com/openPMD/openPMD-api/pull/886)
+         */
+        Container & m_originalContainer;
+
+    public:
+        explicit EraseStaleEntries( Container & container_in )
+            : m_originalContainer( container_in )
+        {
+        }
+
+        template< typename K >
+        mapped_type & operator[]( K && k )
+        {
+            m_accessedKeys.insert( k ); // copy
+            return m_originalContainer[ std::forward< K >( k ) ];
+        }
+
+        template< typename K >
+        mapped_type & at( K && k )
+        {
+            m_accessedKeys.insert( k ); // copy
+            return m_originalContainer.at( std::forward< K >( k ) );
+        }
+
+        ~EraseStaleEntries()
+        {
+            auto & map = *m_originalContainer.m_container;
+            using iterator_t = typename InternalContainer::const_iterator;
+            std::vector< iterator_t > deleteMe;
+            deleteMe.reserve( map.size() - m_accessedKeys.size() );
+            for( iterator_t it = map.begin(); it != map.end(); ++it )
+            {
+                auto lookup = m_accessedKeys.find( it->first );
+                if( lookup == m_accessedKeys.end() )
+                {
+                    deleteMe.push_back( it );
+                }
+            }
+            for( auto & it : deleteMe )
+            {
+                map.erase( it );
+            }
+        }
+    };
+
+    EraseStaleEntries eraseStaleEntries()
+    {
+        return EraseStaleEntries( *this );
+    }
 };
 
 } // openPMD

include/openPMD/backend/PatchRecordComponent.hpp

@@ -75,7 +75,6 @@ class PatchRecordComponent : public BaseRecordComponent
     void read();
 
     std::shared_ptr< std::queue< IOTask > > m_chunks;
-    std::shared_ptr< bool > hasBeenRead = std::make_shared< bool >( false );
 
     /**
      * @brief Check recursively whether this RecordComponent is dirty.