Immediate flush mode

CMakeLists.txt

@@ -400,6 +400,7 @@ set(CORE_SOURCE
         src/Format.cpp
         src/Iteration.cpp
         src/IterationEncoding.cpp
+        src/LoadStoreChunk.cpp
         src/Mesh.cpp
         src/ParticlePatches.cpp
         src/ParticleSpecies.cpp
@@ -411,6 +412,7 @@ set(CORE_SOURCE
         src/version.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
+        src/auxiliary/Future.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/JSONMatcher.cpp
         src/auxiliary/Memory.cpp
@@ -426,6 +428,11 @@ set(CORE_SOURCE
         src/backend/PatchRecord.cpp
         src/backend/PatchRecordComponent.cpp
         src/backend/Writable.cpp
+        src/backend/scientific_defaults/ScientificDefaults.cpp
+        src/backend/scientific_defaults/ScientificDefaults_auxiliary.cpp
+        src/backend/scientific_defaults/ProcessParsedAttribute.cpp
+        src/backend/scientific_defaults/AttributeReader.cpp
+        src/backend/scientific_defaults/ConfigAttribute.cpp
         src/auxiliary/OneDimensionalBlockSlicer.cpp
         src/helper/list_series.cpp
         src/snapshots/ContainerImpls.cpp

include/openPMD/Dataset.hpp

@@ -34,6 +34,12 @@ namespace openPMD
 using Extent = std::vector<std::uint64_t>;
 using Offset = std::vector<std::uint64_t>;
 
+struct MemorySelection
+{
+    Offset offset;
+    Extent extent;
+};
+
 class Dataset
 {
     friend class RecordComponent;

include/openPMD/Datatype.tpp

@@ -223,36 +223,52 @@ namespace detail
     template <typename T>
     constexpr bool is_char_v = is_char<T>::value;
 
-    template <typename T_Char1, typename T_Char2>
-    inline bool isSameChar()
+    struct IsChar
     {
-        return
-            // both must be char types
-            is_char_v<T_Char1> && is_char_v<T_Char2> &&
-            // both must have equivalent sign
-            std::is_signed_v<T_Char1> == std::is_signed_v<T_Char2> &&
-            // both must have equivalent size
-            sizeof(T_Char1) == sizeof(T_Char2);
+        template <typename T>
+        static constexpr bool call()
+        {
+            return is_char_v<T>;
+        }
+        template <size_t N>
+        static constexpr bool call()
+        {
+            return false;
+        }
+    };
+
+    constexpr inline bool isChar(Datatype dtype)
+    {
+        return switchType<IsChar>(dtype);
     }
 
-    template <typename T1>
-    struct IsSameChar
+    struct DtypeSize
     {
-        template <typename T2>
-        static bool call()
+        template <typename T>
+        static constexpr size_t call()
         {
-            return isSameChar<T1, T2>();
+            return sizeof(T);
         }
-
-        static constexpr char const *errorMsg = "IsSameChar";
+        static constexpr char const *errorMsg = "DtypeSize";
     };
 
+    constexpr inline size_t dtypeSize(Datatype dtype)
+    {
+        return switchType<DtypeSize>(dtype);
+    }
 } // namespace detail
 
 template <typename T_Char>
 constexpr inline bool isSameChar(Datatype d)
 {
-    return switchType<detail::IsSameChar<T_Char>>(d);
+    return isSameChar(d, determineDatatype<T_Char>());
+}
+
+constexpr bool isSameChar(Datatype d1, Datatype d2)
+{
+    return detail::isChar(d1) && detail::isChar(d2) &&
+        isSigned(d1) == isSigned(d2) &&
+        detail::dtypeSize(d1) == detail::dtypeSize(d2);
 }
 
 namespace detail
@@ -285,11 +301,6 @@ constexpr inline bool isSigned(Datatype d)
     return switchType<detail::IsSigned>(d);
 }
 
-constexpr inline bool isSameChar(Datatype d, Datatype e)
-{
-    return isChar(d) && isChar(e) && isSigned(d) == isSigned(e);
-}
-
 constexpr bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
 {
     return

include/openPMD/IO/ADIOS/ADIOS2File.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadVariables.hpp"
@@ -112,6 +113,7 @@ struct BufferedUniquePtrPut
     std::string name;
     Offset offset;
     Extent extent;
+    std::optional<MemorySelection> memorySelection;
     UniquePtrWithLambda<void> data;
     Datatype dtype = Datatype::UNDEFINED;
 

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -21,6 +21,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
@@ -30,6 +31,7 @@
 #include "openPMD/IO/AbstractIOHandler.hpp"
 #include "openPMD/IO/AbstractIOHandlerImpl.hpp"
 #include "openPMD/IO/AbstractIOHandlerImplCommon.hpp"
+#include "openPMD/IO/AbstractIOHandler_internal.hpp"
 #include "openPMD/IO/FlushParametersInternal.hpp"
 #include "openPMD/IO/IOTask.hpp"
 #include "openPMD/IO/InvalidatableFile.hpp"
@@ -509,6 +511,7 @@ class ADIOS2IOHandlerImpl
     adios2::Variable<T> verifyDataset(
         Offset const &offset,
         Extent const &extent,
+        std::optional<MemorySelection> const &memorySelection,
         adios2::IO &IO,
         adios2::Engine &engine,
         std::string const &varName,
@@ -622,13 +625,26 @@ class ADIOS2IOHandlerImpl
         var.SetSelection(
             {adios2::Dims(offset.begin(), offset.end()),
              adios2::Dims(extent.begin(), extent.end())});
+
+        if (memorySelection.has_value())
+        {
+            var.SetMemorySelection(
+                {adios2::Dims(
+                     memorySelection->offset.begin(),
+                     memorySelection->offset.end()),
+                 adios2::Dims(
+                     memorySelection->extent.begin(),
+                     memorySelection->extent.end())});
+        }
+
         return var;
     }
 
     struct
     {
         bool noGroupBased = false;
         bool blosc2bp5 = false;
+        bool memorySelection = false;
     } printedWarningsAlready;
 }; // ADIOS2IOHandlerImpl
 
@@ -942,7 +958,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
         try
         {
             auto params = internal::defaultParsedFlushParams;
-            this->flush(params);
+            this->flush_impl(params);
         }
         catch (std::exception const &ex)
         {
@@ -962,9 +978,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
 #if openPMD_HAVE_MPI
 
     ADIOS2IOHandler(
-        std::optional<std::unique_ptr<AbstractIOHandler>> initialize_from,
-        std::string path,
-        Access,
+        internal::AbstractIOHandlerInitFrom &&initialize_from,
         MPI_Comm,
         json::TracingJSON options,
         std::string engineType,
@@ -973,9 +987,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
 #endif
 
     ADIOS2IOHandler(
-        std::optional<std::unique_ptr<AbstractIOHandler>> initialize_from,
-        std::string path,
-        Access,
+        internal::AbstractIOHandlerInitFrom &&initialize_from,
         json::TracingJSON options,
         std::string engineType,
         std::string specifiedExtension);
@@ -990,6 +1002,6 @@ class ADIOS2IOHandler : public AbstractIOHandler
         return true;
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 }; // ADIOS2IOHandler
 } // namespace openPMD

include/openPMD/IO/ADIOS/macros.hpp

@@ -41,6 +41,29 @@
 #define openPMD_HAVE_ADIOS2_BP5 0
 #endif
 
+namespace openPMD
+{
+namespace detail
+{
+    template <typename Variable, typename SFINAE = void>
+    struct CanTheMemorySelectionBeReset
+    {
+        static constexpr bool value = false;
+    };
+
+    template <typename Variable>
+    struct CanTheMemorySelectionBeReset<
+        Variable,
+        decltype(std::declval<Variable>().SetMemorySelection())>
+    {
+        static constexpr bool value = true;
+    };
+} // namespace detail
+
+constexpr bool CanTheMemorySelectionBeReset =
+    detail::CanTheMemorySelectionBeReset<adios2::Variable<int>>::value;
+} // namespace openPMD
+
 #else
 
 #define openPMD_HAS_ADIOS_2_8 0