Fix JSON chunk detection

Former algorithm left some chunks split apart, fix that.

Author: franzpoeschel

include/openPMD/Dataset.hpp

@@ -55,13 +55,14 @@ class Dataset
 
 struct Chunk
 {
-    Offset const offset;
-    Extent const extent;
-    unsigned int const rank;
+    Offset offset;
+    Extent extent;
+    unsigned int rank = 0;
 
     /*
      * If rank is smaller than zero, will be converted to zero.
      */
+    explicit Chunk() = default;
     Chunk( Offset, Extent, int rank );
     Chunk( Offset, Extent );
 

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -331,6 +331,87 @@ namespace openPMD
             }
             return res;
         }
+
+        std::pair< bool, Chunk >
+        mergeChunks( Chunk const & _c1, Chunk const & _c2 )
+        {
+            Chunk const *c1( &_c1 ), *c2( &_c2 );
+            unsigned dimensionality = _c1.extent.size();
+            for( unsigned dim = 0; dim < dimensionality; ++dim )
+            {
+                // check if one chunk is the extension of the other at
+                // dimension dim
+                if( c1->offset[ dim ] > c2->offset[ dim ] )
+                {
+                    std::swap( c1, c2 );
+                }
+                // now, c1 begins at the lower of both offsets
+                // next check, that both chunks border one another exactly
+                if( c2->offset[ dim ] != c1->offset[ dim ] + c1->extent[ dim ] )
+                {
+                    continue;
+                }
+                // we've got a candidate
+                // verify that all other dimensions have equal values
+                auto equalValues = [ dimensionality, dim, c1, c2 ]() {
+                    for( unsigned j = 0; j < dimensionality; ++j )
+                    {
+                        if( j == dim )
+                        {
+                            continue;
+                        }
+                        if( c1->offset[ j ] != c2->offset[ j ] ||
+                            c1->extent[ j ] != c2->extent[ j ] )
+                        {
+                            return false;
+                        }
+                    }
+                    return true;
+                };
+                if( !equalValues() )
+                {
+                    continue;
+                }
+                // we can merge the chunks
+                Offset offset( c1->offset );
+                Extent extent( c1->extent );
+                extent[ dim ] += c2->extent[ dim ];
+                return std::make_pair( true, Chunk( offset, extent ) );
+            }
+            return std::make_pair( false, Chunk() );
+        }
+
+        void
+        mergeChunks( ChunkTable & table )
+        {
+            bool stillChanging;
+            do
+            {
+                stillChanging = false;
+                auto innerLoops = [ &table ]() {
+                    for( auto i = table.begin(); i < table.end(); ++i )
+                    {
+                        for( auto j = i + 1; j < table.end(); ++j )
+                        {
+                            std::pair< bool, Chunk > merged =
+                                mergeChunks( *i, *j );
+                            if( merged.first )
+                            {
+                                // erase order is important due to iterator
+                                // invalidation
+                                table.erase( j );
+                                table.erase( i );
+                                table.emplace_back(
+                                    std::move( merged.second ) );
+                                return true;
+                            }
+                        }
+                    }
+                    return false;
+                };
+                stillChanging = innerLoops();
+            } while( stillChanging );
+        }
     } // namespace
 
     void
@@ -342,6 +423,7 @@ namespace openPMD
         auto filePosition = setAndGetFilePosition( writable );
         auto & j = obtainJsonContents( writable )[ "data" ];
         *parameters.chunks = chunksInJSON( j );
+        mergeChunks( *parameters.chunks );
     }
 
     void JSONIOHandlerImpl::openFile(

test/SerialIOTest.cpp

@@ -100,13 +100,14 @@ TEST_CASE( "available_chunks_test_json", "[serial][json]" )
      *    4 ****
      *    5 ****
      *    6 ****
-     *    7 _**_
-     *    8 _**_
-     *    9 ____
+     *    7 **__
+     *    8 **_*
+     *    9 ___*
      *
-     * Will be read as two chunks:
+     * Will be read as three chunks:
      * 1. (2,0) -- (5,4) (offset -- extent)
-     * 2. (7,1) -- (2,2) (offset -- extent)
+     * 2. (7,0) -- (2,2) (offset -- extent)
+     * 3. (8,3) -- (2,1) (offset -- extent)
      *
      */
     constexpr unsigned height = 10;
@@ -124,8 +125,9 @@ TEST_CASE( "available_chunks_test_json", "[serial][json]" )
         }
         for( unsigned line = 7; line < 9; ++line )
         {
-            E_x.storeChunk( data, { line, 1 }, { 1, 2 } );
+            E_x.storeChunk( data, { line, 0 }, { 1, 2 } );
         }
+        E_x.storeChunk( data, { 8, 3 }, {2, 1 } );
         it0.close();
     }
 
@@ -134,9 +136,10 @@ TEST_CASE( "available_chunks_test_json", "[serial][json]" )
         Iteration it0 = read.iterations[ 0 ];
         auto E_x = it0.meshes[ "E" ][ "x" ];
         ChunkTable table = E_x.availableChunks();
-        REQUIRE( table.size() == 2 );
-        REQUIRE( table[ 0 ] == Chunk( { 2, 0 }, { 5, 4 }, -1 ) );
-        REQUIRE( table[ 1 ] == Chunk( { 7, 1 }, { 2, 2 }, -1 ) );
+        REQUIRE( table.size() == 3 );
+        REQUIRE( table[ 0 ] == Chunk( { 2, 0 }, { 5, 4 } ) );
+        REQUIRE( table[ 1 ] == Chunk( { 7, 0 }, { 2, 2 } ) );
+        REQUIRE( table[ 2 ] == Chunk( { 8, 3 }, { 2, 1 } ) );
     }
 }
 

test/python/unittest/API/APITest.py

@@ -1516,7 +1516,7 @@ def testCloseIteration(self):
             self.makeCloseIterationRoundTrip(ext)
 
     def makeAvailableChunksRoundTrip(self, ext):
-        if ext == "json" or ext == "h5":
+        if ext == "h5":
             return
         name = "../samples/available_chunks_python." + ext
         write = io.Series(