Documentation and testing

Author: franzpoeschel

docs/source/usage/workflow.rst

@@ -1,5 +1,38 @@
 .. _workflow:
 
+Access modes
+============
+
+The openPMD-api distinguishes between a number of different access modes:
+
+* **Create mode**: Used for creating a new Series from scratch.
+  Any file possibly existing in the specified location will be overwritten.
+* **Read-only mode**: Used for reading from an existing Series.
+  No modifications will be made.
+* **Read/Write mode**: Creates a new Series if not existing, otherwise opens an existing Series for reading and writing.
+  New datasets and iterations will be inserted as needed.
+  Not fully supported by all backends:
+  * ADIOS1: Automatically coerced to ... mode @todo
+  * ADIOS2: Automatically coerced to *Create* mode if the file does not exist yet and to *Read-only* mode if it exists.
+    Since this happens on a per-file level, this mode allows to read from existing iterations and write to new iterations at the same time in file-based iteration encoding.
+* **Append mode**: Restricted mode for appending new iterations to an existing Series that is supported by all backends.
+  The API is equivalent to that of the *Create* mode, meaning that no reading is supported whatsoever.
+  If the Series does not exist yet, this behaves equivalently to the *Create* mode.
+  Existing iterations will not be deleted, newly-written iterations will be inserted.
+
+  **Warning:** If writing an iteration that already exists, the behavior is implementation-defined and depends on the chosen backend and iteration encoding:
+
+  * The new iteration might fully replace the old one.
+  * The new iteration might be merged into the old one.
+  * (To be removed in a future update) The old and new iteration might coexist in the resulting dataset.
+
+  We suggest to fully define iterations when using Append mode (i.e. as if using Create mode) to avoid implementation-specific behavior.
+  Appending to an openPMD Series is only supported on a per-iteration level.
+
+  **Warning:** There is no reading involved in using Append mode.
+  It is a user's responsibility to ensure that the appended dataset and the appended-to dataset are compatible with each other.
+  The results of using incompatible backend configurations are undefined.
+
 Workflow
 ========
 

test/SerialIOTest.cpp

@@ -1629,8 +1629,24 @@ void fileBased_write_test(const std::string & backend)
         REQUIRE(o.iterations[5].time< double >() == 5.0);
     }
 
-    // extend existing series with new step and auto-detection of iteration padding
+    if( backend == "bp" )
     {
+        // Append + filebased iteration encoding works for all backends
+        Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, Access::APPEND);
+        // Append mode does not support reading anything that already exists
+        REQUIRE(o.iterations.size() == 0);
+        // write something to trigger opening of the file
+        o.iterations[ 6 ].particles[ "e" ][ "position" ][ "x" ].resetDataset(
+            { Datatype::DOUBLE, { 10 } } );
+        o.iterations[ 6 ]
+            .particles[ "e" ][ "position" ][ "x" ]
+            .makeConstant< double >( 1.0 );
+    }
+    else
+    {
+        // @todo enable a workflow for ADIOS2 where only either reading from or
+        // writing to an iteration works
+        // extend existing series with new step and auto-detection of iteration padding
         Series o = Series("../samples/subdir/serial_fileBased_write%T." + backend, Access::READ_WRITE);
 
         REQUIRE(o.iterations.size() == 5);
@@ -1673,6 +1689,7 @@ void fileBased_write_test(const std::string & backend)
     }
 
     // write with auto-detection and in-consistent padding
+    if( backend != "bp" )
     {
         REQUIRE_THROWS_WITH(Series("../samples/subdir/serial_fileBased_write%T." + backend, Access::READ_WRITE),
             Catch::Equals("Cannot write to a series with inconsistent iteration padding. Please specify '%0<N>T' or open as read-only."));
@@ -4617,3 +4634,167 @@ TEST_CASE( "late_setting_of_iterationencoding", "[serial]" )
     REQUIRE( auxiliary::file_exists(
         "../samples/change_name_and_encoding_10.json" ) );
 }
+
+void append_mode( std::string const & extension )
+{
+    std::string jsonConfig = R"END(
+{
+    "adios2":
+    {
+        "schema": 20210209,
+        "engine":
+        {
+            "usesteps" : true
+        }
+    }
+})END";
+    auto writeSomeIterations = []( WriteIterations && writeIterations,
+                                   std::vector< uint64_t > indices )
+    {
+        for( auto index : indices )
+        {
+            auto it = writeIterations[ index ];
+            auto dataset = it.meshes[ "E" ][ "x" ];
+            dataset.resetDataset( { Datatype::INT, { 1 } } );
+            dataset.makeConstant< int >( 0 );
+            // test that it works without closing too
+            it.close();
+        }
+    };
+    {
+        Series write(
+            "../samples/append." + extension, Access::CREATE, jsonConfig );
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 0, 1 } );
+    }
+    {
+        Series write(
+            "../samples/append." + extension, Access::APPEND, jsonConfig );
+        if( write.backend() == "ADIOS1" )
+        {
+            REQUIRE_THROWS_AS(
+                write.flush(), error::OperationUnsupportedInBackend );
+            // destructor will be noisy now
+            return;
+        }
+
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 2, 3 } );
+        write.flush();
+    }
+    {
+        Series write(
+            "../samples/append." + extension, Access::APPEND, jsonConfig );
+        if( write.backend() == "ADIOS1" )
+        {
+            REQUIRE_THROWS_AS(
+                write.flush(), error::OperationUnsupportedInBackend );
+            // destructor will be noisy now
+            return;
+        }
+
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 4, 3 } );
+        write.flush();
+    }
+    {
+        Series read( "../samples/append." + extension, Access::READ_ONLY );
+        REQUIRE( read.iterations.size() == 5 );
+        /*
+         * Roadmap: for now, reading this should work by ignoring the last
+         * duplicate iteration.
+         * After merging https://github.com/openPMD/openPMD-api/pull/949, we
+         * should see both instances when reading.
+         * Final goal: Read only the last instance.
+         */
+        helper::listSeries( read );
+    }
+}
+
+TEST_CASE( "append_mode", "[serial]" )
+{
+    for( auto const & t : testedFileExtensions() )
+    {
+        append_mode( t );
+    }
+}
+
+void append_mode_filebased( std::string const & extension )
+{
+    std::string jsonConfig = R"END(
+{
+    "adios2":
+    {
+        "schema": 20210209,
+        "engine":
+        {
+            "usesteps" : true
+        }
+    }
+})END";
+    auto writeSomeIterations = []( WriteIterations && writeIterations,
+                                   std::vector< uint64_t > indices ) {
+        for( auto index : indices )
+        {
+            auto it = writeIterations[ index ];
+            auto dataset = it.meshes[ "E" ][ "x" ];
+            dataset.resetDataset( { Datatype::INT, { 1 } } );
+            dataset.makeConstant< int >( 0 );
+            // test that it works without closing too
+            it.close();
+        }
+    };
+    if( auxiliary::directory_exists( "../samples/append" ) )
+    {
+        auxiliary::remove_directory( "../samples/append" );
+    }
+    {
+        Series write(
+            "../samples/append/append_%T." + extension,
+            Access::CREATE,
+            jsonConfig );
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 0, 1 } );
+    }
+    {
+        Series write(
+            "../samples/append/append_%T." + extension,
+            Access::APPEND,
+            jsonConfig );
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 4, 5 } );
+        write.flush();
+    }
+    {
+        Series write(
+            "../samples/append/append_%T." + extension,
+            Access::APPEND,
+            jsonConfig );
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 2, 3 } );
+        write.flush();
+    }
+    {
+        Series write(
+            "../samples/append/append_%T." + extension,
+            Access::APPEND,
+            jsonConfig );
+        // overwrite a previous iteration
+        writeSomeIterations(
+            write.writeIterations(), std::vector< uint64_t >{ 4, 123 } );
+        write.flush();
+    }
+    {
+        Series read(
+            "../samples/append/append_%T." + extension, Access::READ_ONLY );
+        REQUIRE( read.iterations.size() == 7 );
+    }
+}
+
+TEST_CASE( "append_mode_filebased", "[serial]" )
+{
+    for( auto const & t : testedFileExtensions() )
+    {
+        append_mode_filebased( t );
+    }
+}