Patch release 0.17.1

.appveyor.yml

@@ -3,7 +3,7 @@ environment:
     - TARGET_ARCH: x64
       CONDA_PY: 3.10
       CONDA_INSTALL_LOCN: C:\\Miniconda37-x64
-      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2017
+      APPVEYOR_BUILD_WORKER_IMAGE: Visual Studio 2019
       platform: x64
       SHARED: OFF
 
@@ -101,12 +101,12 @@ before_build:
   # Compiler & Generator Selection
   - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2017" set OPENPMD_CMAKE_GENERATOR=Visual Studio 15 2017
   - cmd: if "%APPVEYOR_BUILD_WORKER_IMAGE%"=="Visual Studio 2019" set OPENPMD_CMAKE_GENERATOR=Visual Studio 16 2019
-  - cmd: if "%TARGET_ARCH%"=="x64" set OPENPMD_CMAKE_GENERATOR=%OPENPMD_CMAKE_GENERATOR% Win64
 #  - cmd: if "%TARGET_ARCH%"=="x86" "C:\Program Files (x86)\Microsoft Visual Studio 15.9\VC\vcvarsall.bat" x86
 #  - cmd: if "%TARGET_ARCH%"=="x64" "C:\Program Files (x86)\Microsoft Visual Studio 15.9\VC\vcvarsall.bat" amd64
 
   # CMake configure
-  - cmd: cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".."
+  - cmd: if "%TARGET_ARCH%"=="x64" cmake -G "%OPENPMD_CMAKE_GENERATOR%" -A "%TARGET_ARCH%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".."
+  - cmd: if "%TARGET_ARCH%"=="x86" cmake -G "%OPENPMD_CMAKE_GENERATOR%" -DCMAKE_BUILD_TYPE=%CONFIGURATION% -DBUILD_SHARED_LIBS=%SHARED% -DBUILD_TESTING=ON -DopenPMD_USE_PYTHON=ON -DPython_EXECUTABLE="%CONDA_INSTALL_LOCN%\python.exe" -DCMAKE_INSTALL_PREFIX="%CONDA_INSTALL_LOCN%" -DCMAKE_INSTALL_BINDIR="Library\bin" ".."
 
 build_script:
   - cmd: cmake --build . --config %CONFIGURATION% -j 2

CHANGELOG.rst

@@ -1,7 +1,45 @@
 .. _install-changelog:
 
 Changelog
-=========
+==========
+
+0.17.1
+------
+**Date:** YYYY-MM-DD
+
+Bug Fixes
+"""""""""
+
+- ADIOS2:
+
+  - Flush dirty ADIOS2 files in sorted order (#1868)
+  - Keep written flag consistent across MPI ranks for rankTable (#1869)
+  - Fix parallel deletion (#1858)
+  - Do not flush if the backend does not support Span API (#1863)
+- HDF5:
+
+  - Proper status check (#1870)
+  - Close operations also upon failure (#1866)
+- API:
+
+  - Cache Iteration indexes instead of computing them on the spot (#1860)
+  - Harmonize Datatype equality checks (#1854)
+
+Other
+"""""
+
+- Python:
+
+  - Fix keep_alive specifications, add tests for keep_alive (#1851)
+  - Remove duplicate Python test (#1867)
+- Dependencies:
+
+  - Bump toml11 to 4.4.0, nlohmann_json to 3.12.0 (#1842)
+  - Version bump pybind11 -> 3.0.2 (#1849)
+- CI/Infrastructure:
+
+  - create_directories: preserve sticky and setgid permissions (#1855)
+  - Fix AppVeyor 64bit build (#1832)
 
 0.17.0
 ------

CMakeLists.txt

@@ -837,6 +837,7 @@ if(openPMD_BUILD_TESTING)
         elseif(${test_name} STREQUAL "Core")
             list(APPEND ${out_list}
                 test/Files_Core/automatic_variable_encoding.cpp
+                test/Files_Core/read_nonexistent_attribute.cpp
             )
         endif()
     endmacro()

cmake/dependencies/json.cmake

@@ -56,18 +56,18 @@ set(openPMD_json_src ""
     "Local path to nlohmann_json source directory (preferred if set)")
 
 # tarball fetcher
-set(openPMD_json_tar "https://github.com/nlohmann/json/archive/refs/tags/v3.11.3.tar.gz"
+set(openPMD_json_tar "https://github.com/nlohmann/json/archive/refs/tags/v3.12.0.tar.gz"
     CACHE STRING
     "Remote tarball link to pull and build nlohmann_json from if(openPMD_USE_INTERNAL_JSON)")
-set(openPMD_json_tar_hash "SHA256=0d8ef5af7f9794e3263480193c491549b2ba6cc74bb018906202ada498a79406"
+set(openPMD_json_tar_hash "SHA256=4b92eb0c06d10683f7447ce9406cb97cd4b453be18d7279320f7b2f025c10187"
     CACHE STRING
     "Hash checksum of the tarball of nlohmann_json if(openPMD_USE_INTERNAL_JSON)")
 
 # Git fetcher
 set(openPMD_json_repo "https://github.com/nlohmann/json.git"
     CACHE STRING
     "Repository URI to pull and build nlohmann_json from if(openPMD_USE_INTERNAL_JSON)")
-set(openPMD_json_branch "v3.11.3"
+set(openPMD_json_branch "v3.12.0"
     CACHE STRING
     "Repository branch for openPMD_json_repo if(openPMD_USE_INTERNAL_JSON)")
 

cmake/dependencies/pybind11.cmake

@@ -51,9 +51,9 @@ function(find_pybind11)
         mark_as_advanced(FETCHCONTENT_UPDATES_DISCONNECTED_FETCHEDpybind11)
     elseif(NOT openPMD_USE_INTERNAL_PYBIND11)
         if(openPMD_USE_PYTHON STREQUAL AUTO)
-            find_package(pybind11 2.13.0 CONFIG)
+            find_package(pybind11 3.0.0 CONFIG)
         elseif(openPMD_USE_PYTHON)
-            find_package(pybind11 2.13.0 CONFIG REQUIRED)
+            find_package(pybind11 3.0.0 CONFIG REQUIRED)
         endif()
         if(TARGET pybind11::module)
             message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
@@ -67,18 +67,18 @@ set(openPMD_pybind11_src ""
     "Local path to pybind11 source directory (preferred if set)")
 
 # tarball fetcher
-set(openPMD_pybind11_tar "https://github.com/pybind/pybind11/archive/refs/tags/v2.13.6.tar.gz"
+set(openPMD_pybind11_tar "https://github.com/pybind/pybind11/archive/refs/tags/v3.0.2.tar.gz"
         CACHE STRING
         "Remote tarball link to pull and build pybind11 from if(openPMD_USE_INTERNAL_PYBIND11)")
-set(openPMD_pybind11_tar_hash "SHA256=e08cb87f4773da97fa7b5f035de8763abc656d87d5773e62f6da0587d1f0ec20"
+set(openPMD_pybind11_tar_hash "SHA256=2f20a0af0b921815e0e169ea7fec63909869323581b89d7de1553468553f6a2d"
         CACHE STRING
         "Hash checksum of the tarball of pybind11 if(openPMD_USE_INTERNAL_PYBIND11)")
 
 # Git fetcher
 set(openPMD_pybind11_repo "https://github.com/pybind/pybind11.git"
     CACHE STRING
     "Repository URI to pull and build pybind11 from if(openPMD_USE_INTERNAL_PYBIND11)")
-set(openPMD_pybind11_branch "v2.13.6"
+set(openPMD_pybind11_branch "v3.0.2"
     CACHE STRING
     "Repository branch for openPMD_pybind11_repo if(openPMD_USE_INTERNAL_PYBIND11)")
 

cmake/dependencies/toml11.cmake

@@ -64,18 +64,18 @@ set(openPMD_toml11_src ""
     "Local path to toml11 source directory (preferred if set)")
 
 # tarball fetcher
-set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.2.0.tar.gz"
+set(openPMD_toml11_tar "https://github.com/ToruNiina/toml11/archive/refs/tags/v4.4.0.tar.gz"
         CACHE STRING
         "Remote tarball link to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
-set(openPMD_toml11_tar_hash "SHA256=9287971cd4a1a3992ef37e7b95a3972d1ae56410e7f8e3f300727ab1d6c79c2c"
+set(openPMD_toml11_tar_hash "SHA256=815bfe6792aa11a13a133b86e7f0f45edc5d71eb78f5fb6686c49c7f792b9049"
         CACHE STRING
         "Hash checksum of the tarball of toml11 if(openPMD_USE_INTERNAL_TOML11)")
 
 # Git fetcher
 set(openPMD_toml11_repo "https://github.com/ToruNiina/toml11.git"
     CACHE STRING
     "Repository URI to pull and build toml11 from if(openPMD_USE_INTERNAL_TOML11)")
-set(openPMD_toml11_branch "v3.7.1"
+set(openPMD_toml11_branch "v4.4.0"
     CACHE STRING
     "Repository branch for openPMD_toml11_repo if(openPMD_USE_INTERNAL_TOML11)")
 

include/openPMD/Datatype.hpp

@@ -294,7 +294,8 @@ template <typename T>
 inline constexpr Datatype determineDatatype(T &&val)
 {
     (void)val; // don't need this, it only has a name for Doxygen
-    using T_stripped = std::remove_cv_t<std::remove_reference_t<T>>;
+    using T_stripped =
+        std::remove_extent_t<std::remove_cv_t<std::remove_reference_t<T>>>;
     if constexpr (auxiliary::IsPointer_v<T_stripped>)
     {
         return determineDatatype<auxiliary::IsPointer_t<T_stripped>>();
@@ -419,6 +420,8 @@ inline size_t toBits(Datatype d)
     return toBytes(d) * CHAR_BIT;
 }
 
+constexpr bool isSigned(Datatype d);
+
 /** Compare if a Datatype is a vector type
  *
  * @param d Datatype to test
@@ -595,14 +598,19 @@ inline std::tuple<bool, bool> isInteger()
  */
 template <typename T_FP>
 inline bool isSameFloatingPoint(Datatype d)
+{
+    return isSameFloatingPoint(d, determineDatatype<T_FP>());
+}
+
+inline bool isSameFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
-    bool tt_is_fp = isFloatingPoint<T_FP>();
+    bool tt_is_fp = isFloatingPoint(d1);
 
     // Datatype
-    bool dt_is_fp = isFloatingPoint(d);
+    bool dt_is_fp = isFloatingPoint(d2);
 
-    if (tt_is_fp && dt_is_fp && toBits(d) == toBits(determineDatatype<T_FP>()))
+    if (tt_is_fp && dt_is_fp && toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -617,15 +625,19 @@ inline bool isSameFloatingPoint(Datatype d)
  */
 template <typename T_CFP>
 inline bool isSameComplexFloatingPoint(Datatype d)
+{
+    return isSameComplexFloatingPoint(d, determineDatatype<T_CFP>());
+}
+
+inline bool isSameComplexFloatingPoint(Datatype d1, Datatype d2)
 {
     // template
-    bool tt_is_cfp = isComplexFloatingPoint<T_CFP>();
+    bool tt_is_cfp = isComplexFloatingPoint(d1);
 
     // Datatype
-    bool dt_is_cfp = isComplexFloatingPoint(d);
+    bool dt_is_cfp = isComplexFloatingPoint(d2);
 
-    if (tt_is_cfp && dt_is_cfp &&
-        toBits(d) == toBits(determineDatatype<T_CFP>()))
+    if (tt_is_cfp && dt_is_cfp && toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -640,17 +652,22 @@ inline bool isSameComplexFloatingPoint(Datatype d)
  */
 template <typename T_Int>
 inline bool isSameInteger(Datatype d)
+{
+    return isSameInteger(d, determineDatatype<T_Int>());
+}
+
+inline bool isSameInteger(Datatype d1, Datatype d2)
 {
     // template
     bool tt_is_int, tt_is_sig;
-    std::tie(tt_is_int, tt_is_sig) = isInteger<T_Int>();
+    std::tie(tt_is_int, tt_is_sig) = isInteger(d1);
 
     // Datatype
     bool dt_is_int, dt_is_sig;
-    std::tie(dt_is_int, dt_is_sig) = isInteger(d);
+    std::tie(dt_is_int, dt_is_sig) = isInteger(d2);
 
     if (tt_is_int && dt_is_int && tt_is_sig == dt_is_sig &&
-        toBits(d) == toBits(determineDatatype<T_Int>()))
+        toBits(d1) == toBits(d2))
         return true;
     else
         return false;
@@ -691,46 +708,15 @@ constexpr bool isChar(Datatype d)
 template <typename T_Char>
 constexpr bool isSameChar(Datatype d);
 
+constexpr bool isSameChar(Datatype d1, Datatype d2);
+
 /** Comparison for two Datatypes
  *
  * Besides returning true for the same types, identical implementations on
  * some platforms, e.g. if long and long long are the same or double and
  * long double will also return true.
  */
-inline bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
-{
-    // exact same type
-    if (static_cast<int>(d) == static_cast<int>(e))
-        return true;
-
-    bool d_is_vec = isVector(d);
-    bool e_is_vec = isVector(e);
-
-    // same int
-    bool d_is_int, d_is_sig;
-    std::tie(d_is_int, d_is_sig) = isInteger(d);
-    bool e_is_int, e_is_sig;
-    std::tie(e_is_int, e_is_sig) = isInteger(e);
-    if (d_is_int && e_is_int && d_is_vec == e_is_vec && d_is_sig == e_is_sig &&
-        toBits(d) == toBits(e))
-        return true;
-
-    // same float
-    bool d_is_fp = isFloatingPoint(d);
-    bool e_is_fp = isFloatingPoint(e);
-
-    if (d_is_fp && e_is_fp && d_is_vec == e_is_vec && toBits(d) == toBits(e))
-        return true;
-
-    // same complex floating point
-    bool d_is_cfp = isComplexFloatingPoint(d);
-    bool e_is_cfp = isComplexFloatingPoint(e);
-
-    if (d_is_cfp && e_is_cfp && d_is_vec == e_is_vec && toBits(d) == toBits(e))
-        return true;
-
-    return false;
-}
+constexpr bool isSame(openPMD::Datatype d, openPMD::Datatype e);
 
 /**
  * @brief basicDatatype Strip openPMD Datatype of std::vector, std::array et.

include/openPMD/Datatype.tpp

@@ -25,6 +25,7 @@
 // comment to prevent clang-format from moving this #include up
 // datatype macros may be included and un-included in other headers
 #include "openPMD/DatatypeMacros.hpp"
+#include "openPMD/auxiliary/TypeTraits.hpp"
 
 #include <string>
 #include <type_traits> // std::void_t
@@ -253,6 +254,56 @@ constexpr inline bool isSameChar(Datatype d)
 {
     return switchType<detail::IsSameChar<T_Char>>(d);
 }
+
+namespace detail
+{
+    struct IsSigned
+    {
+        template <typename T>
+        static constexpr bool call()
+        {
+            if constexpr (auxiliary::IsVector_v<T> || auxiliary::IsArray_v<T>)
+            {
+                return call<typename T::value_type>();
+            }
+            else if constexpr (std::is_same_v<T, std::string>)
+            {
+                return call<char>();
+            }
+            else
+            {
+                return std::is_signed_v<T>;
+            }
+        }
+
+        static constexpr char const *errorMsg = "IsSigned";
+    };
+} // namespace detail
+
+constexpr inline bool isSigned(Datatype d)
+{
+    return switchType<detail::IsSigned>(d);
+}
+
+constexpr inline bool isSameChar(Datatype d, Datatype e)
+{
+    return isChar(d) && isChar(e) && isSigned(d) == isSigned(e);
+}
+
+constexpr bool isSame(openPMD::Datatype const d, openPMD::Datatype const e)
+{
+    return
+        // exact same type
+        static_cast<int>(d) == static_cast<int>(e)
+        // same int
+        || isSameInteger(d, e)
+        // same float
+        || isSameFloatingPoint(d, e)
+        // same complex floating point
+        || isSameComplexFloatingPoint(d, e)
+        // same char
+        || isSameChar(d, e);
+}
 } // namespace openPMD
 
 #include "openPMD/UndefDatatypeMacros.hpp"

include/openPMD/IO/AbstractIOHandlerImplCommon.hpp

@@ -28,10 +28,10 @@
 #include "openPMD/auxiliary/StringManip.hpp"
 #include "openPMD/backend/Writable.hpp"
 
+#include <set>
 #include <stdexcept>
 #include <string>
 #include <unordered_map>
-#include <unordered_set>
 
 namespace openPMD
 {
@@ -50,7 +50,10 @@ class AbstractIOHandlerImplCommon : public AbstractIOHandlerImpl
      * without the OS path
      */
     std::unordered_map<Writable *, InvalidatableFile> m_files;
-    std::unordered_set<InvalidatableFile> m_dirty;
+    // MUST be an ordered set in order to consistently flush on different
+    // parallel processes (same logic cant apply to m_files since Writable*
+    // pointers are not predictable)
+    std::set<InvalidatableFile> m_dirty;
 
     enum PossiblyExisting
     {

include/openPMD/IO/IOTask.hpp

@@ -558,6 +558,7 @@ struct OPENPMDAPI_EXPORT
     }
 
     // in parameters
+    bool queryOnly = false; // query if the backend supports this
     Offset offset;
     Extent extent;
     Datatype dtype = Datatype::UNDEFINED;