Lazy parsing

First attempt at an implementation of lazy parsing

todo:
* file-based layout
* iterator access

Basic file-based deferred parsing

Remove changes to Container class

Read eagerly by default

Read iteration upon Iteration::open

Expose to frontend and add some tests

Use Builder Pattern for Series

Author: franzpoeschel

CMakeLists.txt

@@ -348,6 +348,7 @@ set(CORE_SOURCE
         src/Record.cpp
         src/RecordComponent.cpp
         src/Series.cpp
+        src/SeriesBuilder.cpp
         src/version.cpp
         src/WriteIterations.cpp
         src/auxiliary/Date.cpp

include/openPMD/Iteration.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/auxiliary/Option.hpp"
 #include "openPMD/auxiliary/Variant.hpp"
 #include "openPMD/backend/Attributable.hpp"
 #include "openPMD/backend/Container.hpp"
@@ -31,6 +32,11 @@
 
 namespace openPMD
 {
+namespace traits
+{
+struct AccessIteration;
+}
+
 /** @brief  Logical compilation of data from one snapshot (e.g. a single simulation cycle).
  *
  * @see https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath
@@ -46,6 +52,7 @@ class Iteration : public LegacyAttributable
     friend class SeriesImpl;
     friend class WriteIterations;
     friend class SeriesIterator;
+    friend struct traits::AccessIteration;
 
 public:
     Iteration( Iteration const & ) = default;
@@ -153,10 +160,21 @@ class Iteration : public LegacyAttributable
 private:
     Iteration();
 
+    struct DeferredRead
+    {
+        std::string index;
+        bool fileBased = false;
+        std::string filename;
+    };
+
     void flushFileBased(std::string const&, uint64_t);
     void flushGroupBased(uint64_t);
     void flush();
+    void deferRead( DeferredRead );
     void read();
+    void readFileBased( std::string filePath, std::string const & groupPath );
+    void readGroupBased( std::string const & groupPath );
+    void read_impl( std::string const & groupPath );
 
     /**
      * @brief Whether an iteration has been closed yet.
@@ -194,6 +212,10 @@ class Iteration : public LegacyAttributable
     std::shared_ptr< StepStatus > m_stepStatus =
         std::make_shared< StepStatus >( StepStatus::NoStep );
 
+    std::shared_ptr< auxiliary::Option< DeferredRead > > m_deferredRead =
+        std::make_shared< auxiliary::Option< DeferredRead > >(
+            auxiliary::Option< DeferredRead >() );
+
     /**
      * @brief Begin an IO step on the IO file (or file-like object)
      *        containing this iteration. In case of group-based iteration
@@ -252,11 +274,33 @@ class Iteration : public LegacyAttributable
      * @param w The Writable representing the parent.
      */
     virtual void linkHierarchy(Writable& w);
+
+    void accessLazily()
+    {
+        if( IOHandler()->m_frontendAccess == Access::CREATE )
+        {
+            return;
+        }
+        auto oldAccess = IOHandler()->m_frontendAccess;
+        auto newAccess =
+            const_cast< Access * >( &IOHandler()->m_frontendAccess );
+        *newAccess = Access::READ_WRITE;
+        try
+        {
+            read();
+        }
+        catch( ... )
+        {
+            *newAccess = oldAccess;
+            throw;
+        }
+        *newAccess = oldAccess;
+    }
 };  // Iteration
 
-extern template
-float
-Iteration::time< float >() const;
+using Iterations_t = Container< Iteration, uint64_t >;
+
+extern template float Iteration::time< float >() const;
 
 extern template
 double

include/openPMD/Series.hpp

@@ -69,6 +69,7 @@ class SeriesData : public AttributableData
     friend class openPMD::SeriesImpl;
     friend class openPMD::Iteration;
     friend class openPMD::Series;
+    friend class SeriesInternal;
 
 public:
     explicit SeriesData() = default;
@@ -81,7 +82,7 @@ class SeriesData : public AttributableData
 
     virtual ~SeriesData() = default;
 
-    Container< Iteration, uint64_t > iterations{};
+    Iterations_t iterations{};
 
 OPENPMD_private :
     auxiliary::Option< WriteIterations > m_writeIterations;
@@ -99,6 +100,7 @@ OPENPMD_private :
      * one among both flags.
      */
     StepStatus m_stepStatus = StepStatus::NoStep;
+    bool m_parseLazily = false;
 }; // SeriesData
 
 class SeriesInternal;
@@ -312,8 +314,7 @@ class SeriesImpl : public AttributableImpl
     static constexpr char const * const BASEPATH = "/data/%T/";
 
     struct ParsedInput;
-    using iterations_t = decltype(internal::SeriesData::iterations);
-    using iterations_iterator = iterations_t::iterator;
+    using iterations_iterator = Iterations_t::iterator;
 
     internal::SeriesData * m_series;
 
@@ -337,6 +338,7 @@ class SeriesImpl : public AttributableImpl
     void flushMeshesPath();
     void flushParticlesPath();
     void readFileBased( );
+    void readOneIterationFileBased( std::string const & filePath );
     /**
      * Note on re-parsing of a Series:
      * If init == false, the parsing process will seek for new
@@ -347,7 +349,6 @@ class SeriesImpl : public AttributableImpl
      */
     void readGroupBased( bool init = true );
     void readBase();
-    void read();
     std::string iterationFilename( uint64_t i );
     void openIteration( uint64_t index, Iteration iteration );
 
@@ -393,13 +394,15 @@ class SeriesInternal : public SeriesData, public SeriesImpl
         std::string const & filepath,
         Access at,
         MPI_Comm comm,
-        std::string const & options = "{}" );
+        std::string const & options = "{}",
+        bool parseLazily = false );
 #endif
 
     SeriesInternal(
         std::string const & filepath,
         Access at,
-        std::string const & options = "{}" );
+        std::string const & options = "{}",
+        bool parseLazily = false );
     // @todo make AttributableImpl<>::linkHierarchy non-virtual
     virtual ~SeriesInternal();
 };
@@ -423,17 +426,22 @@ class Series : public SeriesImpl
         std::string const & filepath,
         Access at,
         MPI_Comm comm,
-        std::string const & options = "{}" );
+        std::string const & options = "{}",
+        bool parseLazily = false );
 #endif
 
+    /*
+     * @todo Think it's time for a SeriesBuilder.
+     */
     Series(
         std::string const & filepath,
         Access at,
-        std::string const & options = "{}" );
+        std::string const & options = "{}",
+        bool parseLazily = false );
 
     virtual ~Series() = default;
 
-    Container< Iteration, uint64_t > iterations;
+    Iterations_t iterations;
 
     /**
      * @brief Entry point to the reading end of the streaming API.

include/openPMD/SeriesBuilder.hpp

@@ -0,0 +1,65 @@
+/* Copyright 2017-2021 Franz Poeschel
+ *
+ * This file is part of openPMD-api.
+ *
+ * openPMD-api is free software: you can redistribute it and/or modify
+ * it under the terms of of either the GNU General Public License or
+ * the GNU Lesser General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * openPMD-api is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License and the GNU Lesser General Public License
+ * for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * and the GNU Lesser General Public License along with openPMD-api.
+ * If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#pragma once
+
+#include "openPMD/Series.hpp"
+
+#include "openPMD/auxiliary/Option.hpp"
+#include "openPMD/config.hpp"
+
+#if openPMD_HAVE_MPI
+
+#include <mpi.h>
+
+#endif
+
+namespace openPMD
+{
+class SeriesBuilder
+{
+private:
+    std::string m_filePath;
+    std::string m_jsonOptions = "{}";
+    Access m_access =
+        Access::READ_ONLY; // use the most careful one for a default
+    bool m_parseLazily = false;
+#if openPMD_HAVE_MPI
+    auxiliary::Option< MPI_Comm > m_comm;
+#endif
+
+public:
+    explicit SeriesBuilder() = default;
+
+    Series build();
+
+    operator Series();
+
+    SeriesBuilder & filePath( std::string );
+    SeriesBuilder & options( std::string );
+    SeriesBuilder & access( Access );
+    SeriesBuilder & parseEagerly();
+    SeriesBuilder & parseLazily();
+#if openPMD_HAVE_MPI
+    SeriesBuilder & comm( MPI_Comm );
+#endif
+};
+}

include/openPMD/WriteIterations.hpp

@@ -49,19 +49,19 @@ class WriteIterations : private Container< Iteration, uint64_t >
     friend class Series;
 
 private:
-    using iterations_t = Container< Iteration, uint64_t >;
     struct SharedResources
     {
-        iterations_t iterations;
+        Iterations_t iterations;
         auxiliary::Option< uint64_t > currentlyOpen;
 
-        SharedResources( iterations_t );
+        SharedResources( Iterations_t );
         ~SharedResources();
     };
 
-    using key_type = typename iterations_t::key_type;
-    using value_type = typename iterations_t::key_type;
-    WriteIterations( iterations_t );
+    using key_type = typename Iterations_t::key_type;
+    // @todo: this is wrong
+    using value_type = typename Iterations_t::key_type;
+    WriteIterations( Iterations_t );
     explicit WriteIterations() = default;
     //! Index of the last opened iteration
     std::shared_ptr< SharedResources > shared;

include/openPMD/openPMD.hpp

@@ -27,15 +27,16 @@ namespace openPMD {}
 // IWYU pragma: begin_exports
 #include "openPMD/Dataset.hpp"
 #include "openPMD/Datatype.hpp"
-#include "openPMD/IterationEncoding.hpp"
 #include "openPMD/Iteration.hpp"
+#include "openPMD/IterationEncoding.hpp"
 #include "openPMD/Mesh.hpp"
 #include "openPMD/ParticlePatches.hpp"
 #include "openPMD/ParticleSpecies.hpp"
 #include "openPMD/ReadIterations.hpp"
-#include "openPMD/RecordComponent.hpp"
 #include "openPMD/Record.hpp"
+#include "openPMD/RecordComponent.hpp"
 #include "openPMD/Series.hpp"
+#include "openPMD/SeriesBuilder.hpp"
 #include "openPMD/UnitDimension.hpp"
 #include "openPMD/WriteIterations.hpp"
 

src/Iteration.cpp

@@ -28,6 +28,7 @@
 #include "openPMD/backend/Writable.hpp"
 
 #include <exception>
+#include <iostream>
 #include <tuple>
 
 
@@ -145,6 +146,7 @@ Iteration::close( bool _flush )
 Iteration &
 Iteration::open()
 {
+    accessLazily();
     internal::SeriesInternal * s = &retrieveSeries();
     // figure out my iteration number
     auto begin = s->indexOf( *this );
@@ -291,9 +293,52 @@ Iteration::flush()
     }
 }
 
-void
-Iteration::read()
+void Iteration::deferRead( DeferredRead dr )
+{
+    *m_deferredRead = auxiliary::makeOption< DeferredRead >( std::move( dr ) );
+}
+
+void Iteration::read()
 {
+    if( !m_deferredRead->has_value() )
+    {
+        return;
+    }
+    auto const & deferred = m_deferredRead->get();
+    if( deferred.fileBased )
+    {
+        readFileBased( deferred.filename, deferred.index );
+    }
+    else
+    {
+        readGroupBased( deferred.index );
+    }
+    // reset this thing
+    *m_deferredRead = auxiliary::Option< DeferredRead >();
+}
+
+void Iteration::readFileBased(
+    std::string filePath, std::string const & groupPath )
+{
+    auto & series = retrieveSeries();
+
+    series.readOneIterationFileBased( filePath );
+
+    read_impl( groupPath );
+}
+
+void Iteration::readGroupBased( std::string const & groupPath )
+{
+
+    read_impl(groupPath );
+}
+
+void Iteration::read_impl( std::string const & groupPath )
+{
+    Parameter< Operation::OPEN_PATH > pOpen;
+    pOpen.path = groupPath;
+    IOHandler()->enqueue( IOTask( this, pOpen ) );
+
     using DT = Datatype;
     Parameter< Operation::READ_ATT > aRead;
 
@@ -356,7 +401,6 @@ Iteration::read()
 
     if( hasMeshes )
     {
-        Parameter< Operation::OPEN_PATH > pOpen;
         pOpen.path = s->meshesPath();
         IOHandler()->enqueue(IOTask(&meshes, pOpen));
 
@@ -416,7 +460,6 @@ Iteration::read()
 
     if( hasParticles )
     {
-        Parameter< Operation::OPEN_PATH > pOpen;
         pOpen.path = s->particlesPath();
         IOHandler()->enqueue(IOTask(&particles, pOpen));