Introduce more flexible storeChunk() syntax, use to add ADIOS2 memory selection

.github/workflows/linux.yml

@@ -339,7 +339,7 @@ jobs:
     if: github.event.pull_request.draft == false
     steps:
     - uses: actions/checkout@v6
-    - uses: conda-incubator/setup-miniconda@v3
+    - uses: conda-incubator/setup-miniconda@v4
       name: Setup conda
       with:
         auto-update-conda: true

.github/workflows/source.yml

@@ -36,7 +36,7 @@ jobs:
     runs-on: ubuntu-22.04
     steps:
     - uses: actions/checkout@v6
-    - uses: conda-incubator/setup-miniconda@v3
+    - uses: conda-incubator/setup-miniconda@v4
       name: Setup conda
       with:
         auto-update-conda: true

CMakeLists.txt

@@ -400,6 +400,7 @@ set(CORE_SOURCE
         src/Format.cpp
         src/Iteration.cpp
         src/IterationEncoding.cpp
+        src/LoadStoreChunk.cpp
         src/Mesh.cpp
         src/ParticlePatches.cpp
         src/ParticleSpecies.cpp
@@ -411,6 +412,7 @@ set(CORE_SOURCE
         src/version.cpp
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
+        src/auxiliary/Future.cpp
         src/auxiliary/JSON.cpp
         src/auxiliary/JSONMatcher.cpp
         src/auxiliary/Memory.cpp

cmake/dependencies/toml11.cmake

@@ -64,18 +64,18 @@ set(openPMD_toml11_src ""
     "Local path to toml11 source directory (preferred if set)")
 
 # tarball fetcher
-set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.4.0.tar.gz"
+set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/b32a2fff0d27e1f7522f26a125101500ddb47156.tar.gz"
         CACHE STRING
         "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
-set(openPMD_toml11_tar_hash "SHA256=815bfe6792aa11a13a133b86e7f0f45edc5d71eb78f5fb6686c49c7f792b9049"
+set(openPMD_toml11_tar_hash "SHA256=c82df2d6a437795a1b5ac0b87886d5eb481deb60d6e79adb1688017857347cda"
         CACHE STRING
         "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)")
 
 # Git fetcher
 set(openPMD_toml11_repo "https://github.com/ToruNiina/toml11.git"
     CACHE STRING
     "Repository URI to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
-set(openPMD_toml11_branch "v4.4.0"
+set(openPMD_toml11_branch "b32a2fff0d27e1f7522f26a125101500ddb47156"
     CACHE STRING
     "Repository branch for openPMD_toml11_repo if(openPMD_USE_INTERNAL_TOML11)")
 

docs/source/backends/adios2.rst

@@ -84,7 +84,7 @@ environment variable                    default    description
 ``OPENPMD_ADIOS2_ENGINE``               ``File``   `ADIOS2 engine <https://adios2.readthedocs.io/en/latest/engines/engines.html>`_
 ``OPENPMD_ADIOS2_PRETEND_ENGINE``       *empty*    Pretend that an (unknown) ADIOS2 engine is in fact another one (also see the ``adios2.pretend_engine`` :ref:`parameter <backendconfig-adios2>`).
 ``OPENPMD2_ADIOS2_USE_GROUP_TABLE``     ``0``      Use group table (see below)
-``OPENPMD_ADIOS2_STATS_LEVEL``          ``0``      whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no).
+``OPENPMD_ADIOS2_STATS_LEVEL``          see below  Whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no)
 ``OPENPMD_ADIOS2_ASYNC_WRITE``          ``0``      ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``).
 ``OPENPMD_ADIOS2_BP5_BufferChunkMB``    ``0``      ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation
 ``OPENPMD_ADIOS2_BP5_MaxShmMB``         ``0``      ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation
@@ -98,7 +98,7 @@ Please refer to the `ADIOS2 documentation <https://adios2.readthedocs.io/en/late
 
 Notice that the ADIOS2 backend is alternatively configurable via :ref:`JSON parameters <backendconfig>`.
 
-Due to performance considerations, the ADIOS2 backend configures ADIOS2 not to compute any dataset statistics (Min/Max) by default.
+Due to performance considerations, the ADIOS2 backend configures ADIOS2 not to compute any dataset statistics (Min/Max) by default for ADIOS2 versions up until v2.12.0. Starting with `v2.12.1 <https://github.com/ornladios/ADIOS2/releases/tag/v2.12.1>`_, the performance of statistics computation has been improved, so it is turned on by default.
 Statistics may be activated by setting the :ref:`JSON parameter <backendconfig>` ``adios2.engine.parameters.StatsLevel = "1"``.
 
 The ADIOS2 backend overrides the default unlimited queueing behavior of the SST engine with a more cautious limit of 2 steps that may be held in the queue at one time.

include/openPMD/Dataset.hpp

@@ -34,6 +34,18 @@ namespace openPMD
 using Extent = std::vector<std::uint64_t>;
 using Offset = std::vector<std::uint64_t>;
 
+/** Selection of a region of memory for storing chunks.
+ *
+ * Used to specify a non-contiguous memory region when storing
+ * data chunks. This allows writing data that is not contiguous
+ * in memory.
+ */
+struct MemorySelection
+{
+    Offset offset;
+    Extent extent;
+};
+
 class Dataset
 {
     friend class RecordComponent;

include/openPMD/Datatype.hpp

@@ -420,6 +420,11 @@ inline size_t toBits(Datatype d)
     return toBytes(d) * CHAR_BIT;
 }
 
+/** Check if a Datatype is a signed type
+ *
+ * @param d Datatype to test
+ * @return true if signed type (integer, floating point, complex), else false
+ */
 constexpr bool isSigned(Datatype d);
 
 /** Compare if a Datatype is a vector type
@@ -602,6 +607,13 @@ inline bool isSameFloatingPoint(Datatype d)
     return isSameFloatingPoint(d, determineDatatype<T_FP>());
 }
 
+/** Compare if two Datatypes are equivalent floating point types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are floating point and have same bitness, else
+ * false
+ */
 inline bool isSameFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
@@ -629,6 +641,13 @@ inline bool isSameComplexFloatingPoint(Datatype d)
     return isSameComplexFloatingPoint(d, determineDatatype<T_CFP>());
 }
 
+/** Compare if two Datatypes are equivalent complex floating point types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are complex floating point and have same bitness,
+ * else false
+ */
 inline bool isSameComplexFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
@@ -656,6 +675,13 @@ inline bool isSameInteger(Datatype d)
     return isSameInteger(d, determineDatatype<T_Int>());
 }
 
+/** Compare if two Datatypes are equivalent integer types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are integers, same signedness and same bitness,
+ * else false
+ */
 inline bool isSameInteger(Datatype d1, Datatype d2)
 {
     // template
@@ -708,13 +734,24 @@ constexpr bool isChar(Datatype d)
 template <typename T_Char>
 constexpr bool isSameChar(Datatype d);
 
+/** Compare if two Datatypes are equivalent char types
+ *
+ * @param d1 First Datatype to compare
+ * @param d2 Second Datatype to compare
+ * @return true if both types are chars with same signedness and size, else
+ * false
+ */
 constexpr bool isSameChar(Datatype d1, Datatype d2);
 
 /** Comparison for two Datatypes
  *
  * Besides returning true for the same types, identical implementations on
  * some platforms, e.g. if long and long long are the same or double and
  * long double will also return true.
+ *
+ * @param d First Datatype to compare
+ * @param e Second Datatype to compare
+ * @return true if the datatypes are equivalent
  */
 constexpr bool isSame(openPMD::Datatype d, openPMD::Datatype e);
 
@@ -726,15 +763,34 @@ constexpr bool isSame(openPMD::Datatype d, openPMD::Datatype e);
  */
 Datatype basicDatatype(Datatype dt);
 
+/** Convert a scalar Datatype to its vector variant
+ *
+ * @param dt Scalar Datatype to convert
+ * @return Vector Datatype (e.g., INT becomes VEC_INT)
+ */
 Datatype toVectorType(Datatype dt);
 
+/** Convert a Datatype to its string representation
+ *
+ * @param dt Datatype to convert
+ * @return String representation of the Datatype
+ */
 std::string datatypeToString(Datatype dt);
 
+/** Convert a string to a Datatype
+ *
+ * @param s String representation of a Datatype
+ * @return The corresponding Datatype
+ */
 Datatype stringToDatatype(const std::string &s);
 
-void warnWrongDtype(std::string const &key, Datatype store, Datatype request);
-
-std::ostream &operator<<(std::ostream &, openPMD::Datatype const &);
+/** Stream operator for Datatype
+ *
+ * @param os Output stream
+ * @param dt Datatype to output
+ * @return Reference to the stream
+ */
+std::ostream &operator<<(std::ostream &os, openPMD::Datatype const &dt);
 
 template <typename T>
 constexpr auto datatypeIndex() -> size_t

include/openPMD/Datatype.tpp

@@ -223,36 +223,52 @@ namespace detail
     template <typename T>
     constexpr bool is_char_v = is_char<T>::value;
 
-    template <typename T_Char1, typename T_Char2>
-    inline bool isSameChar()
+    struct IsChar
     {
-        return
-            // both must be char types
-            is_char_v<T_Char1> && is_char_v<T_Char2> &&
-            // both must have equivalent sign
-            std::is_signed_v<T_Char1> == std::is_signed_v<T_Char2> &&
-            // both must have equivalent size
-            sizeof(T_Char1) == sizeof(T_Char2);
+        template <typename T>
+        static constexpr bool call()
+        {
+            return is_char_v<T>;
+        }
+        template <size_t N>
+        static constexpr bool call()
+        {
+            return false;
+        }
+    };
+
+    constexpr inline bool isChar(Datatype dtype)
+    {
+        return switchType<IsChar>(dtype);
     }
 
-    template <typename T1>
-    struct IsSameChar
+    struct DtypeSize
     {
-        template <typename T2>
-        static bool call()
+        template <typename T>
+        static constexpr size_t call()
         {
-            return isSameChar<T1, T2>();
+            return sizeof(T);
         }
-
-        static constexpr char const *errorMsg = "IsSameChar";
+        static constexpr char const *errorMsg = "DtypeSize";
     };
 
+    constexpr inline size_t dtypeSize(Datatype dtype)
+    {
+        return switchType<DtypeSize>(dtype);
+    }
 } // namespace detail
 
 template <typename T_Char>
 constexpr inline bool isSameChar(Datatype d)
 {
-    return switchType<detail::IsSameChar<T_Char>>(d);
+    return isSameChar(d, determineDatatype<T_Char>());
+}
+
+constexpr bool isSameChar(Datatype d1, Datatype d2)
+{
+    return detail::isChar(d1) && detail::isChar(d2) &&
+        isSigned(d1) == isSigned(d2) &&
+        detail::dtypeSize(d1) == detail::dtypeSize(d2);
 }
 
 namespace detail
@@ -285,11 +301,6 @@ constexpr inline bool isSigned(Datatype d)
     return switchType<detail::IsSigned>(d);
 }
 
-constexpr inline bool isSameChar(Datatype d, Datatype e)
-{
-    return isChar(d) && isChar(e) && isSigned(d) == isSigned(e);
-}
-
 constexpr bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
 {
     return

include/openPMD/IO/ADIOS/ADIOS2File.hpp

@@ -20,6 +20,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadAttributes.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2PreloadVariables.hpp"
@@ -107,11 +108,14 @@ struct WriteDataset
     static void call(Params &&...);
 };
 
+/** Buffered put operation with unique pointer */
 struct BufferedUniquePtrPut
 {
     std::string name;
     Offset offset;
     Extent extent;
+    /** Optional memory selection for non-contiguous memory regions */
+    std::optional<MemorySelection> memorySelection;
     UniquePtrWithLambda<void> data;
     Datatype dtype = Datatype::UNDEFINED;
 

include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp

@@ -21,6 +21,7 @@
  */
 #pragma once
 
+#include "openPMD/Dataset.hpp"
 #include "openPMD/Error.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2Auxiliary.hpp"
 #include "openPMD/IO/ADIOS/ADIOS2FilePosition.hpp"
@@ -509,6 +510,7 @@ class ADIOS2IOHandlerImpl
     adios2::Variable<T> verifyDataset(
         Offset const &offset,
         Extent const &extent,
+        std::optional<MemorySelection> const &memorySelection,
         adios2::IO &IO,
         adios2::Engine &engine,
         std::string const &varName,
@@ -622,13 +624,26 @@ class ADIOS2IOHandlerImpl
         var.SetSelection(
             {adios2::Dims(offset.begin(), offset.end()),
              adios2::Dims(extent.begin(), extent.end())});
+
+        if (memorySelection.has_value())
+        {
+            var.SetMemorySelection(
+                {adios2::Dims(
+                     memorySelection->offset.begin(),
+                     memorySelection->offset.end()),
+                 adios2::Dims(
+                     memorySelection->extent.begin(),
+                     memorySelection->extent.end())});
+        }
+
         return var;
     }
 
     struct
     {
         bool noGroupBased = false;
         bool blosc2bp5 = false;
+        bool memorySelection = false;
     } printedWarningsAlready;
 }; // ADIOS2IOHandlerImpl
 
@@ -942,7 +957,7 @@ class ADIOS2IOHandler : public AbstractIOHandler
         try
         {
             auto params = internal::defaultParsedFlushParams;
-            this->flush(params);
+            this->flush_impl(params);
         }
         catch (std::exception const &ex)
         {
@@ -990,6 +1005,6 @@ class ADIOS2IOHandler : public AbstractIOHandler
         return true;
     }
 
-    std::future<void> flush(internal::ParsedFlushParams &) override;
+    std::future<void> flush_impl(internal::ParsedFlushParams &) override;
 }; // ADIOS2IOHandler
 } // namespace openPMD