Backend implementation

1) Make backends aware of Append mode
2) In ADIOS1, fix use of namespaces, only use #include statements
   outside of namespaces

Author: franzpoeschel

src/IO/ADIOS/ADIOS1IOHandler.cpp

@@ -319,12 +319,17 @@ ADIOS1IOHandlerImpl::initialize_group(std::string const &name)
     return group;
 }
 
+} // namespace openPMD
+
 #ifndef DOXYGEN_SHOULD_SKIP_THIS
 #define CommonADIOS1IOHandlerImpl ADIOS1IOHandlerImpl
 #include "CommonADIOS1IOHandler.cpp"
 #undef CommonADIOS1IOHandlerImpl
 #endif
 
+namespace openPMD
+{
+
 #else
 ADIOS1IOHandler::ADIOS1IOHandler(std::string path, Access at)
         : AbstractIOHandler(std::move(path), at)

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -22,6 +22,7 @@
 #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
 
 #include "openPMD/Datatype.hpp"
+#include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2IOHandler.hpp"
 #include "openPMD/auxiliary/Environment.hpp"
@@ -292,9 +293,6 @@ void ADIOS2IOHandlerImpl::createFile(
         m_iterationEncoding = parameters.encoding;
         associateWithFile( writable, shared_name );
         this->m_dirty.emplace( shared_name );
-        getFileData( shared_name, IfFileNotOpen::OpenImplicitly ).m_mode =
-            adios2::Mode::Write; // WORKAROUND
-        // ADIOS2 does not yet implement ReadWrite Mode
 
         writable->written = true;
         writable->abstractFilePosition =
@@ -1061,7 +1059,8 @@ void ADIOS2IOHandlerImpl::availableChunks(
 adios2::Mode
 ADIOS2IOHandlerImpl::adios2AccessMode( std::string const & fullPath )
 {
-    switch ( m_handler->m_backendAccess )
+    ( void )fullPath;
+    switch( m_handler->m_backendAccess )
     {
     case Access::CREATE:
         return adios2::Mode::Write;
@@ -1071,21 +1070,16 @@ ADIOS2IOHandlerImpl::adios2AccessMode( std::string const & fullPath )
         if( auxiliary::directory_exists( fullPath ) ||
             auxiliary::file_exists( fullPath ) )
         {
-            std::cerr << "ADIOS2 does currently not yet implement ReadWrite "
-                         "(Append) mode. "
-                      << "Replacing with Read mode." << std::endl;
             return adios2::Mode::Read;
         }
         else
         {
-            std::cerr << "ADIOS2 does currently not yet implement ReadWrite "
-                         "(Append) mode. "
-                      << "Replacing with Write mode." << std::endl;
             return adios2::Mode::Write;
         }
-    default:
-        return adios2::Mode::Undefined;
+    case Access::APPEND:
+        return adios2::Mode::Append;
     }
+    throw std::runtime_error( "Unreachable!" );
 }
 
 auxiliary::TracingJSON ADIOS2IOHandlerImpl::nullvalue = nlohmann::json();
@@ -2304,6 +2298,7 @@ namespace detail
                         delayOpeningTheFirstStep = true;
                         break;
                     case adios2::Mode::Write:
+                    case adios2::Mode::Append:
                         /*
                          * File engines, write mode:
                          * Default for old layout is no steps.
@@ -2463,6 +2458,7 @@ namespace detail
         {
             switch( m_mode )
             {
+            case adios2::Mode::Append:
             case adios2::Mode::Write: {
                 // usesSteps attribute only written upon ::advance()
                 // this makes sure that the attribute is only put in case
@@ -2695,19 +2691,17 @@ namespace detail
             []( BufferedActions & ba, adios2::Engine & eng ) {
                 switch( ba.m_mode )
                 {
-                    case adios2::Mode::Write:
-                        eng.PerformPuts();
-                        break;
-                    case adios2::Mode::Read:
-                        eng.PerformGets();
-                        break;
-                    case adios2::Mode::Append:
-                        // TODO order?
-                        eng.PerformGets();
-                        eng.PerformPuts();
-                        break;
-                    default:
-                        break;
+                case adios2::Mode::Write:
+                case adios2::Mode::Append:
+                    eng.PerformPuts();
+                    break;
+                case adios2::Mode::Read:
+                    eng.PerformGets();
+                    break;
+                default:
+                    throw std::runtime_error(
+                        "[ADIOS2] Unexpected access mode." );
+                    break;
                 }
             },
             writeAttributes,

src/IO/ADIOS/CommonADIOS1IOHandler.cpp

@@ -18,11 +18,16 @@
  * and the GNU Lesser General Public License along with openPMD-api.
  * If not, see <http://www.gnu.org/licenses/>.
  */
+#include "openPMD/auxiliary/Filesystem.hpp"
+#include "openPMD/Error.hpp"
+
 #include <algorithm>
 #include <complex>
-#include <tuple>
 #include <string>
+#include <tuple>
 
+namespace openPMD
+{
 
 void
 CommonADIOS1IOHandlerImpl::close(int64_t fd)
@@ -393,6 +398,15 @@ CommonADIOS1IOHandlerImpl::createFile(Writable* writable,
         if( !auxiliary::ends_with(name, ".bp") )
             name += ".bp";
 
+        if( m_handler->m_backendAccess == Access::APPEND &&
+            auxiliary::file_exists( name ) )
+        {
+            throw error::OperationUnsupportedInBackend(
+                "ADIOS1",
+                "Appending to existing file on disk (use Access::CREATE to "
+                "overwrite)" );
+        }
+
         writable->written = true;
         writable->abstractFilePosition = std::make_shared< ADIOS1FilePosition >("/");
 
@@ -1653,3 +1667,5 @@ CommonADIOS1IOHandlerImpl::listAttributes(Writable* writable,
         *parameters.attributes = std::vector< std::string >(attributes.begin(), attributes.end());
     }
 }
+
+} // namespace openPMD

src/IO/ADIOS/ParallelADIOS1IOHandler.cpp

@@ -348,12 +348,17 @@ ParallelADIOS1IOHandlerImpl::initialize_group(std::string const &name)
     return group;
 }
 
+} // namespace openPMD
+
 #ifndef DOXYGEN_SHOULD_SKIP_THIS
 #define CommonADIOS1IOHandlerImpl ParallelADIOS1IOHandlerImpl
 #include "CommonADIOS1IOHandler.cpp"
 #undef CommonADIOS1IOHandlerImpl
 #endif
 
+namespace openPMD
+{
+
 #else
 #   if openPMD_HAVE_MPI
 ParallelADIOS1IOHandler::ParallelADIOS1IOHandler(std::string path,

src/IO/HDF5/HDF5IOHandler.cpp

@@ -175,15 +175,42 @@ HDF5IOHandlerImpl::createFile(Writable* writable,
         std::string name = m_handler->directory + parameters.name;
         if( !auxiliary::ends_with(name, ".h5") )
             name += ".h5";
-        unsigned flags;
-        if( m_handler->m_backendAccess == Access::CREATE )
+        unsigned flags{};
+        switch( m_handler->m_backendAccess )
+        {
+        case Access::CREATE:
             flags = H5F_ACC_TRUNC;
-        else
+            break;
+        case Access::APPEND:
+            if( auxiliary::file_exists( name ) )
+            {
+                flags = H5F_ACC_RDWR;
+            }
+            else
+            {
+                flags = H5F_ACC_TRUNC;
+            }
+            break;
+        case Access::READ_WRITE:
             flags = H5F_ACC_EXCL;
-        hid_t id = H5Fcreate(name.c_str(),
-                             flags,
-                             H5P_DEFAULT,
-                             m_fileAccessProperty);
+            break;
+        case Access::READ_ONLY:
+            // condition has been checked above
+            throw std::runtime_error( "Control flow error" );
+        }
+
+        hid_t id{};
+        if( flags == H5F_ACC_RDWR )
+        {
+            id = H5Fopen( name.c_str(), flags, m_fileAccessProperty);
+        }
+        else
+        {
+            id = H5Fcreate(name.c_str(),
+                                flags,
+                                H5P_DEFAULT,
+                                m_fileAccessProperty);
+        }
         VERIFY(id >= 0, "[HDF5] Internal error: Failed to create HDF5 file");
 
         writable->written = true;
@@ -554,7 +581,14 @@ HDF5IOHandlerImpl::openFile(
     Access at = m_handler->m_backendAccess;
     if( at == Access::READ_ONLY )
         flags = H5F_ACC_RDONLY;
-    else if( at == Access::READ_WRITE || at == Access::CREATE )
+    /*
+     * Within the HDF5 backend, APPEND and READ_WRITE mode are
+     * equivalent, but the openPMD frontend exposes no reading
+     * functionality in APPEND mode.
+     */
+    else if(
+        at == Access::READ_WRITE || at == Access::CREATE ||
+        at == Access::APPEND )
         flags = H5F_ACC_RDWR;
     else
         throw std::runtime_error("[HDF5] Unknown file Access");

src/IO/JSON/JSONIOHandlerImpl.cpp

@@ -115,7 +115,7 @@ namespace openPMD
                 file.invalidate( );
             }
 
-            std::string const dir( m_handler->directory );
+            std::string const & dir( m_handler->directory );
             if( !auxiliary::directory_exists( dir ) )
             {
                 auto success = auxiliary::create_directories( dir );
@@ -133,6 +133,15 @@ namespace openPMD
             this->m_jsonVals[shared_name] =
                 std::make_shared< nlohmann::json >( );
 
+            // start off with already written contents if available
+            // @todo Do the same in READ_WRITE mode in ::openFile()?
+            auto const & fullName = fullPath(name);
+            if( m_handler->m_backendAccess == Access::APPEND
+                && auxiliary::file_exists(fullName))
+            {
+                std::ifstream handle(fullName);
+                handle >> *this->m_jsonVals[shared_name];
+            }
 
             writable->written = true;
             writable->abstractFilePosition =
@@ -1099,6 +1108,14 @@ namespace openPMD
         {
             case Access::CREATE:
             case Access::READ_WRITE:
+            case Access::APPEND:
+                /*
+                 * Always truncate when writing, we alway write entire JSON
+                 * datasets, never partial ones.
+                 * Within the JSON backend, APPEND and READ_WRITE mode are
+                 * equivalent, but the openPMD frontend exposes no reading
+                 * functionality in APPEND mode.
+                 */
                 fs->open(
                     path,
                     std::ios_base::out | std::ios_base::trunc