Write rank table as parallel dataset

Author: franzpoeschel

include/openPMD/Series.hpp

@@ -108,6 +108,9 @@ class SeriesData : public AttributableData
      */
     MPI_Comm m_communicator;
 #endif
+    internal::AttributableInternal m_rankTable;
+    Parameter< Operation::CREATE_DATASET > m_createRankTable;
+    std::vector< Parameter< Operation::WRITE_DATASET > > m_chunks;
 }; // SeriesData
 
 class SeriesInternal;
@@ -188,15 +191,6 @@ class SeriesInterface : public AttributableInterface
     chunk_assignment::RankMeta
     mpiRanksMetaInfo() const;
 
-    /**
-     * @brief Set the Mpi Ranks Meta Info attribute, i.e. a Vector with
-     *        a String per (writing) MPI rank, indicating user-
-     *        defined meta information per rank. Example: host name.
-     *
-     * @return Reference to modified series.
-     */
-    SeriesInterface & setMpiRanksMetaInfo( chunk_assignment::RankMeta );
-
     /**
      * @brief Set the Mpi Ranks Meta Info attribute, i.e. a Vector with
      *        a String per (writing) MPI rank, indicating user-
@@ -414,7 +408,8 @@ class SeriesInterface : public AttributableInterface
     void flushGorVBased( iterations_iterator begin, iterations_iterator end );
     void flushMeshesPath();
     void flushParticlesPath();
-    void readFileBased( );
+    void flushRankTable();
+    void readFileBased();
     void readOneIterationFileBased( std::string const & filePath );
     /**
      * Note on re-parsing of a Series:

include/openPMD/backend/Attributable.hpp

@@ -130,6 +130,7 @@ class AttributableInterface
     friend class Iteration;
     friend class Series;
     friend class SeriesInterface;
+    friend class internal::SeriesInternal;
     friend class Writable;
     friend class WriteIterations;
 
@@ -394,6 +395,20 @@ class AttributableInterface
     virtual void linkHierarchy(Writable& w);
 }; // AttributableInterface
 
+namespace internal
+{
+    class AttributableInternal :
+        public AttributableData,
+        public AttributableInterface
+    {
+    public:
+        AttributableInternal() : AttributableInterface{ this }
+        {
+        }
+        virtual ~AttributableInternal() = default;
+    };
+}
+
 // Alias this as Attributable since this is a public abstract parent class
 // for most of the classes in our object model of the openPMD hierarchy
 using Attributable = AttributableInterface;

src/Iteration.cpp

@@ -212,6 +212,8 @@ Iteration::flushFileBased(std::string const& filename, uint64_t i)
         fCreate.name = filename;
         IOHandler()->enqueue(IOTask(s, fCreate));
 
+        s->flushRankTable();
+
         /* create basePath */
         Parameter< Operation::CREATE_PATH > pCreate;
         pCreate.path = auxiliary::replace_first(s->basePath(), "%T/", "");

src/Series.cpp

@@ -31,6 +31,7 @@
 #include "openPMD/Series.hpp"
 #include "openPMD/version.hpp"
 
+#include <algorithm>
 #include <exception>
 #include <iomanip>
 #include <iostream>
@@ -187,6 +188,7 @@ SeriesInterface::mpiRanksMetaInfo() const
     return res;
 }
 
+#if 0
 SeriesInterface &
 SeriesInterface::setMpiRanksMetaInfo( chunk_assignment::RankMeta rankMeta )
 {
@@ -208,10 +210,12 @@ SeriesInterface::setMpiRanksMetaInfo( chunk_assignment::RankMeta rankMeta )
     setAttribute( "rankMetaInfo", std::move( asContiguousVector ) );
     return *this;
 }
+#endif
 
 SeriesInterface &
 SeriesInterface::setMpiRanksMetaInfo( std::string const & myRankInfo )
 {
+#if 0
     if( auxiliary::starts_with( myRankInfo, '@' ) )
     {
         // read from file
@@ -231,22 +235,66 @@ SeriesInterface::setMpiRanksMetaInfo( std::string const & myRankInfo )
         setAttribute( "rankMetaInfo", rankMeta );
         return *this;
     }
-#if openPMD_HAVE_MPI
+#endif
     auto & series = get();
-    int rank;
+    unsigned long long mySize = myRankInfo.size() + 1; // null character
+#if openPMD_HAVE_MPI
+    int rank, size;
     MPI_Comm_rank( series.m_communicator, &rank );
-    std::vector< std::string > rankMeta =
-        auxiliary::collectStringsTo( series.m_communicator, 0, myRankInfo );
-    if( rank == 0 )
-    {
-        setAttribute( "rankMetaInfo", std::move( rankMeta ) );
-    }
+    MPI_Comm_size( series.m_communicator, &size );
+    // todo char portability
+    unsigned long long maxSize = 0;
+    MPI_Allreduce(
+        &mySize,
+        &maxSize,
+        1,
+        MPI_UNSIGNED_LONG_LONG,
+        MPI_MAX,
+        series.m_communicator );
 #else
-    setAttribute( "rankMetaInfo", std::vector< std::string >{ myRankInfo } );
+    int rank{ 0 }, size{ 1 };
+    unsigned long long maxSize = mySize;
 #endif
+
+    series.m_createRankTable.name = "hostTable";
+    series.m_createRankTable.dtype = Datatype::CHAR;
+    series.m_createRankTable.extent = { uint64_t( size ), uint64_t( maxSize ) };
+
+    std::shared_ptr< char > put{
+        new char[ maxSize ]{}, []( char const * ptr ) { delete[] ptr; } };
+    std::copy_n( myRankInfo.c_str(), mySize, put.get() );
+
+    Parameter< Operation::WRITE_DATASET > writeDataset;
+    writeDataset.dtype = Datatype::CHAR;
+    writeDataset.offset = { uint64_t( rank ), 0 };
+    writeDataset.extent = { 1, maxSize };
+    writeDataset.data = std::move( put );
+    series.m_chunks.push_back( std::move( writeDataset ) );
+
     return *this;
 }
 
+void SeriesInterface::flushRankTable()
+{
+    auto & series = get();
+    if( series.m_chunks.empty() )
+    {
+        return;
+    }
+
+    if( !series.m_rankTable.written() )
+    {
+        IOHandler()->enqueue(
+            IOTask( &series.m_rankTable, series.m_createRankTable ) );
+    }
+    for( auto & writeDataset : series.m_chunks )
+    {
+        IOHandler()->enqueue(
+            IOTask( &series.m_rankTable, std::move( writeDataset ) ) );
+    }
+    series.m_chunks.clear();
+}
+
 std::string
 SeriesInterface::particlesPath() const
 {
@@ -797,6 +845,8 @@ SeriesInterface::flushGorVBased( iterations_iterator begin, iterations_iterator
             fCreate.name = series.m_name;
             fCreate.encoding = iterationEncoding();
             IOHandler()->enqueue(IOTask(this, fCreate));
+
+            flushRankTable();
         }
 
         series.iterations.flush(
@@ -1564,6 +1614,7 @@ SeriesInternal::SeriesInternal(
     auto handler = createIOHandler(
         input->path, at, input->format, comm, std::move( optionsJson ) );
     init( handler, std::move( input ) );
+    m_rankTable.linkHierarchy( writable() );
 }
 #endif