Snapshots API: Fill missing bits, harmonize API (#1768)

* Basic untested binding for ::snapshots()

* Add Python bindings for Snapshots

deletion missing

* Implement deletion

* Implement emplace()

could be added for stateful iterator too probably

* Fix Python GIL evasion

* Add new access types, no implementation yet

* Add aliases

* Harmonize write- and read-side API

* Remove IteratorHelpers from CMake

* Move iterator convertors back to helper file

* CI fixes

* Update examples

* Documentation, cleanup

* CI fixes

* Replace .writeIteration() with CREATE_LINEAR + snapshots()

* Same for readIterations

* Replace writeIterations() with snapshots() in parallel tests

* Same for readIterations()

* Same for test subfiles

* Explicitly open Iterations in READ_ONLY with deferred parsing

Author: franzpoeschel

examples/10_streaming_read.cpp

@@ -57,7 +57,6 @@ int main()
 
         // The iteration can be closed in order to help free up resources.
         // The iteration's content will be flushed automatically.
-        // An iteration once closed cannot (yet) be reopened.
         iteration.close();
 
         for (size_t i = 0; i < 3; ++i)

examples/10_streaming_read.py

@@ -20,15 +20,8 @@
     series = io.Series("simData.sst", io.Access_Type.read_linear, config)
 
     # Read all available iterations and print electron position data.
-    # Direct access to iterations is possible via `series.iterations`.
-    # For streaming support, `series.read_iterations()` needs to be used
-    # instead of `series.iterations`.
-    # `Series.write_iterations()` and `Series.read_iterations()` are
-    # intentionally restricted APIs that ensure a workflow which also works
-    # in streaming setups, e.g. an iteration cannot be opened again once
-    # it has been closed.
-    for iteration in series.read_iterations():
-        print("Current iteration {}".format(iteration.iteration_index))
+    for index, iteration in series.snapshots().items():
+        print("Current iteration {}".format(index))
         electronPositions = iteration.particles["e"]["position"]
         loadedChunks = []
         shapes = []

examples/10_streaming_write.cpp

@@ -25,7 +25,12 @@ int main()
     // use QueueFullPolicy = Discard in order to create a situation where from
     // the reader's perspective steps are skipped. This tests the bug reported
     // in https://github.com/openPMD/openPMD-api/issues/1747.
-    Series series = Series("electrons.sst", Access::CREATE, R"(
+    // Create the Series with linear write access, i.e. one Iteration after
+    // the other. The alternative would be random-access where multiple
+    // Iterations can be accessed independently from one another. This more
+    // restricted mode enables performance optimizations in the backends, and
+    // more importantly is compatible with streaming I/O.
+    Series series = Series("electrons.sst", Access::CREATE_LINEAR, R"(
 {
   "adios2": {
     "engine": {
@@ -44,12 +49,7 @@ int main()
     std::shared_ptr<position_t> local_data(
         new position_t[length], [](position_t const *ptr) { delete[] ptr; });
 
-    // Create the Series with synchronous snapshots, i.e. one Iteration after
-    // the other. The alternative would be random-access where multiple
-    // Iterations can be accessed independently from one another. This more
-    // restricted mode enables performance optimizations in the backends, and
-    // more importantly is compatible with streaming I/O.
-    auto iterations = series.snapshots(SnapshotWorkflow::Synchronous);
+    auto iterations = series.snapshots();
     for (size_t i = 0; i < 100; ++i)
     {
         Iteration iteration = iterations[i];

examples/10_streaming_write.py

@@ -20,20 +20,13 @@
 
     # create a series and specify some global metadata
     # change the file extension to .json, .h5 or .bp for regular file writing
-    series = io.Series("simData.sst", io.Access_Type.create, config)
+    series = io.Series("simData.sst", io.Access_Type.create_linear, config)
     series.set_author("Franz Poeschel <f.poeschel@hzdr.de>")
     series.set_software("openPMD-api-python-examples")
 
     # now, write a number of iterations (or: snapshots, time steps)
     for i in range(10):
-        # Direct access to iterations is possible via `series.iterations`.
-        # For streaming support, `series.write_iterations()` needs to be used
-        # instead of `series.iterations`.
-        # `Series.write_iterations()` and `Series.read_iterations()` are
-        # intentionally restricted APIs that ensure a workflow which also works
-        # in streaming setups, e.g. an iteration cannot be opened again once
-        # it has been closed.
-        iteration = series.write_iterations()[i]
+        iteration = series.snapshots()[i]
 
         #######################
         # write electron data #

examples/11_particle_dataframe.py

@@ -36,7 +36,7 @@
 # This "if" is important for distributed dask runs
 if __name__ == "__main__":
     s = io.Series("../samples/git-sample/data%T.h5", io.Access.read_only)
-    electrons = s.iterations[400].particles["electrons"]
+    electrons = s.snapshots()[400].particles["electrons"]
 
     # all particles
     df = electrons.to_df()
@@ -101,7 +101,7 @@
 
     # Meshes
     if found_dask:
-        E = s.iterations[400].meshes["E"]
+        E = s.snapshots()[400].meshes["E"]
         E_x = E["x"]
         E_y = E["y"]
         E_z = E["z"]

examples/12_span_write.cpp

@@ -10,7 +10,7 @@ void span_write(std::string const &filename)
     using namespace openPMD;
     using position_t = double;
     // open file for writing
-    Series series = Series(filename, Access::CREATE);
+    Series series = Series(filename, Access::CREATE_LINEAR);
 
     Datatype datatype = determineDatatype<position_t>();
     constexpr unsigned long length = 10ul;
@@ -19,12 +19,7 @@ void span_write(std::string const &filename)
 
     std::vector<position_t> fallbackBuffer;
 
-    // `Series::writeIterations()` and `Series::readIterations()` are
-    // intentionally restricted APIs that ensure a workflow which also works
-    // in streaming setups, e.g. an iteration cannot be opened again once
-    // it has been closed.
-    // `Series::iterations` can be directly accessed in random-access workflows.
-    WriteIterations iterations = series.writeIterations();
+    Snapshots iterations = series.snapshots();
     for (size_t i = 0; i < 10; ++i)
     {
         Iteration iteration = iterations[i];

examples/12_span_write.py

@@ -3,19 +3,14 @@
 
 
 def span_write(filename):
-    series = io.Series(filename, io.Access_Type.create)
+    series = io.Series(filename, io.Access_Type.create_linear)
 
     datatype = np.dtype("double")
     length = 10
     extent = [length]
     dataset = io.Dataset(datatype, extent)
 
-    # `Series.write_iterations()` and `Series.read_iterations()` are
-    # intentionally restricted APIs that ensure a workflow which also works
-    # in streaming setups, e.g. an iteration cannot be opened again once
-    # it has been closed.
-    # `Series.iterations` can be directly accessed in random-access workflows.
-    iterations = series.write_iterations()
+    iterations = series.snapshots()
     for i in range(12):
         iteration = iterations[i]
         electronPositions = iteration.particles["e"]["position"]

examples/13_write_dynamic_configuration.cpp

@@ -157,7 +157,7 @@ CFG.CHUNKS = [10]
 
     // open file for writing
     Series series =
-        Series("../samples/dynamicConfig.h5", Access::CREATE, defaults);
+        Series("../samples/dynamicConfig.h5", Access::CREATE_LINEAR, defaults);
 
     Datatype datatype = determineDatatype<position_t>();
     constexpr unsigned long length = 10ul;
@@ -166,12 +166,7 @@ CFG.CHUNKS = [10]
     std::shared_ptr<position_t> local_data(
         new position_t[length], [](position_t const *ptr) { delete[] ptr; });
 
-    // `Series::writeIterations()` and `Series::readIterations()` are
-    // intentionally restricted APIs that ensure a workflow which also works
-    // in streaming setups, e.g. an iteration cannot be opened again once
-    // it has been closed.
-    // `Series::iterations` can be directly accessed in random-access workflows.
-    WriteIterations iterations = series.writeIterations();
+    Snapshots iterations = series.snapshots();
     for (size_t i = 0; i < 100; ++i)
     {
         Iteration iteration = iterations[i];

examples/13_write_dynamic_configuration.py

@@ -56,19 +56,16 @@ def main():
 
     # create a series and specify some global metadata
     # change the file extension to .json, .h5 or .bp for regular file writing
-    series = io.Series("../samples/dynamicConfig.bp", io.Access_Type.create,
-                       defaults)
+    series = io.Series("../samples/dynamicConfig.bp",
+                       io.Access_Type.create_linear, defaults)
 
     # now, write a number of iterations (or: snapshots, time steps)
     for i in range(10):
-        # Direct access to iterations is possible via `series.iterations`.
-        # For streaming support, `series.write_iterations()` needs to be used
-        # instead of `series.iterations`.
-        # `Series.write_iterations()` and `Series.read_iterations()` are
-        # intentionally restricted APIs that ensure a workflow which also works
-        # in streaming setups, e.g. an iteration cannot be opened again once
-        # it has been closed.
-        iteration = series.write_iterations()[i]
+        # Direct access to iterations is possible via `series.snapshots()`.
+        # Access.create_linear ensures that at most one
+        # Iteration/Snapshot is active at any given time, which is necessary
+        # supporting ADIOS2 steps, and also enables further optimizations.
+        iteration = series.snapshots()[i]
 
         #######################
         # write electron data #
@@ -134,7 +131,7 @@ def main():
         # After closing the iteration, the readers can see the iteration.
         # It can no longer be modified.
         # If not closing an iteration explicitly, it will be implicitly closed
-        # upon creating the next iteration.
+        # upon creating the next iteration due to the use of create_linear.
         iteration.close()
 
     # The files in 'series' are still open until the object is destroyed, on

examples/14_toml_template.cpp

@@ -33,9 +33,9 @@ void write()
 
     openPMD::Series writeTemplate(
         "../samples/tomlTemplate." + backendEnding(),
-        openPMD::Access::CREATE,
+        openPMD::Access::CREATE_LINEAR,
         config);
-    auto iteration = writeTemplate.writeIterations()[0];
+    auto iteration = writeTemplate.snapshots()[0];
 
     openPMD::Dataset ds{openPMD::Datatype::FLOAT, {5, 5}};