Merge branch 'dev' into eschnett/julia-bindings

Author: eschnett

.clang-format

@@ -0,0 +1,66 @@
+Language:  Cpp
+
+BasedOnStyle:  LLVM
+AccessModifierOffset: -4
+AlignAfterOpenBracket: AlwaysBreak
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: false
+# AlignEscapedNewlinesLeft: false
+AlignEscapedNewlines: Right
+AlignOperands: false
+AlignTrailingComments: false
+AllowAllArgumentsOnNextLine: true
+AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: true
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: None
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLambdasOnASingleLine: true
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakBeforeMultilineStrings: true
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: false
+BinPackParameters: false
+
+BreakBeforeBraces:  Custom
+BraceWrapping:
+  AfterClass: true
+  AfterControlStatement: true
+  AfterEnum: true
+  AfterExternBlock: true
+  AfterFunction: true
+  AfterNamespace: true
+  AfterStruct: true
+  AfterUnion: true
+  BeforeCatch: true
+  BeforeElse: true
+  # BeforeLambdaBody: true
+  SplitEmptyFunction: false
+  SplitEmptyNamespace: false
+  SplitEmptyRecord: false
+
+BreakConstructorInitializers: BeforeComma
+BreakInheritanceList: BeforeComma
+ColumnLimit: 80
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: true
+ConstructorInitializerIndentWidth:  4
+ContinuationIndentWidth: 4
+FixNamespaceComments: true
+IndentWidth: 4
+NamespaceIndentation: Inner
+PointerAlignment: Right
+SortUsingDeclarations: false
+SpaceAfterCStyleCast: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeParens: ControlStatements
+SpaceInEmptyParentheses: false
+SpacesInAngles: false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard: Auto
+StatementMacros: ["OPENPMD_private", "OPENPMD_protected"]

.clang-tidy

@@ -2,5 +2,5 @@
 # FIXME: all performance-* reports
 # FIXME: all cert-* reports
 # FIXME: all bugprone-* reports
-Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
+Checks: -*,bugprone-*,-bugprone-unhandled-self-assignment,-bugprone-parent-virtual-call,-bugprone-narrowing-conversions,-bugprone-exception-escape,-bugprone-string-literal-with-embedded-nul,cppcoreguidelines-slicing,mpi-*,readability-non-const-parameter,performance-for-range-copy,modernize-*,-modernize-use-trailing-return-type,-modernize-use-bool-literals,-modernize-avoid-c-arrays,-modernize-use-auto,-modernize-return-braced-init-list
 HeaderFilterRegex: '((^(?!\/share\/openPMD\/).*)*include\/openPMD\/.+\.hpp|src\/^(?!binding).+\.cpp$)'

.github/workflows/clang-format/clang-format.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+if (( $# > 0 )); then
+    # received arguments, format those files
+    clang-format-12 -i "$@"
+else
+    # received no arguments, find files on our own
+    find include/ src/ test/ examples/ \
+            -regextype egrep \
+            -type f -regex '.*\.(hpp|cpp|hpp\.in)$' \
+        | xargs clang-format-12 -i
+fi

.pre-commit-config.yaml

@@ -47,7 +47,7 @@ repos:
 
 # Changes tabs to spaces
 - repo: https://github.com/Lucas-C/pre-commit-hooks
-  rev: v1.1.10
+  rev: v1.1.13
   hooks:
   - id: remove-tabs
 
@@ -61,11 +61,16 @@ repos:
 #    files: (\.cmake|CMakeLists.txt)(.in)?$
 
 # C++ formatting
-# clang-format
+# clang-format v12
+#   to run manually, use .github/workflows/clang-format/clang-format.sh
+- repo: https://github.com/pre-commit/mirrors-clang-format
+  rev: v13.0.1
+  hooks:
+  - id: clang-format
 
 # Autoremoves unused Python imports
 - repo: https://github.com/hadialqattan/pycln
-  rev: v1.1.0
+  rev: v1.2.4
   hooks:
   - id: pycln
     name: pycln (python)

CHANGELOG.rst

@@ -17,9 +17,8 @@ Changes to "0.14.0"
 Features
 """"""""
 
-- CMake
-
-  - include internally shipped toml11 v3.7.0 #1148
+- include internally shipped toml11 v3.7.0 #1148
+- pybind11: require version 2.9.1+ #1220
 
 Bug Fixes
 """""""""

CMakeLists.txt

@@ -396,9 +396,9 @@ if(openPMD_USE_PYTHON STREQUAL AUTO)
         if(openPMD_USE_INTERNAL_PYBIND11)
             add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
             set(openPMD_HAVE_PYTHON TRUE)
-            message(STATUS "pybind11: Using INTERNAL version 2.6.2")
+            message(STATUS "pybind11: Using INTERNAL version 2.9.1")
         else()
-            find_package(pybind11 2.6.2 CONFIG)
+            find_package(pybind11 2.9.1 CONFIG)
             if(pybind11_FOUND)
                 set(openPMD_HAVE_PYTHON TRUE)
                 message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
@@ -414,9 +414,9 @@ elseif(openPMD_USE_PYTHON)
     if(openPMD_USE_INTERNAL_PYBIND11)
         add_subdirectory("${openPMD_SOURCE_DIR}/share/openPMD/thirdParty/pybind11")
         set(openPMD_HAVE_PYTHON TRUE)
-        message(STATUS "pybind11: Using INTERNAL version 2.6.2")
+        message(STATUS "pybind11: Using INTERNAL version 2.9.1")
     else()
-        find_package(pybind11 2.6.2 REQUIRED CONFIG)
+        find_package(pybind11 2.9.1 REQUIRED CONFIG)
         set(openPMD_HAVE_PYTHON TRUE)
         message(STATUS "pybind11: Found version '${pybind11_VERSION}'")
     endif()
@@ -736,7 +736,12 @@ if(openPMD_HAVE_PYTHON)
     endif()
     unset(_USE_PY_LTO)
 
-    pybind11_extension(openPMD.py)
+    if(EMSCRIPTEN)
+        set_target_properties(openPMD.py PROPERTIES
+            PREFIX "")
+    else()
+        pybind11_extension(openPMD.py)
+    endif()
     pybind11_strip(openPMD.py)
 
     set_target_properties(openPMD.py PROPERTIES CXX_VISIBILITY_PRESET "hidden"
@@ -941,11 +946,11 @@ if(CMAKE_SOURCE_DIR STREQUAL PROJECT_SOURCE_DIR)
         #                     /usr/lib/llvm-6.0/lib/clang/6.0.0/lib/linux/libclang_rt.ubsan_minimal-x86_64.so
         # at runtime when used with symbol-hidden code (e.g. pybind11 module)
 
-        set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code ${CMAKE_CXX_FLAGS}")
+        set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wextra-semi -Wunreachable-code -Wsign-compare ${CMAKE_CXX_FLAGS}")
     elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Intel")
         set(CMAKE_CXX_FLAGS "-w3 -wd193,383,1572 ${CMAKE_CXX_FLAGS}")
     elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "GNU")
-        set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code ${CMAKE_CXX_FLAGS}")
+        set(CMAKE_CXX_FLAGS "-Wall -Wextra -Wpedantic -Wshadow -Woverloaded-virtual -Wunreachable-code -Wsign-compare ${CMAKE_CXX_FLAGS}")
     elseif("${CMAKE_CXX_COMPILER_ID}" STREQUAL "MSVC")
         # Warning C4503: "decorated name length exceeded, name was truncated"
         # Symbols longer than 4096 chars are truncated (and hashed instead)

NEWS.rst

@@ -11,6 +11,7 @@ Building openPMD-api now requires a compiler that supports C++17 or newer.
 
 Python 3.10 is now supported.
 openPMD-api now depends on `toml11 <https://github.com/ToruNiina/toml11>`__ 3.7.0+.
+pybind11 2.9.1 is now the minimally supported version for Python support.
 
 The following backend-specific members of the ``Dataset`` class have been removed: ``Dataset::setChunkSize()``, ``Dataset::setCompression()``, ``Dataset::setCustomTransform()``, ``Dataset::chunkSize``, ``Dataset::compression``, ``Dataset::transform``.
 They are replaced by backend-specific options in the JSON-based backend configuration.

README.md

@@ -27,7 +27,7 @@ See [the openPMD standard](https://github.com/openPMD/openPMD-standard) for deta
 
 This library provides a reference API for openPMD data handling.
 Since openPMD is a schema (or markup) on top of portable, hierarchical file formats, this library implements various backends such as HDF5, ADIOS1, ADIOS2 and JSON.
-Writing & reading through those backends and their associated files is supported for serial and [MPI-parallel](https://www.mpi-forum.org/docs/) workflows.
+Writing & reading through those backends and their associated files are supported for serial and [MPI-parallel](https://www.mpi-forum.org/docs/) workflows.
 
 ## Usage
 
@@ -104,7 +104,7 @@ Required:
 
 Shipped internally in `share/openPMD/thirdParty/`:
 * [Catch2](https://github.com/catchorg/Catch2) 2.13.4+ ([BSL-1.0](https://github.com/catchorg/Catch2/blob/master/LICENSE.txt))
-* [pybind11](https://github.com/pybind/pybind11) 2.6.2+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
+* [pybind11](https://github.com/pybind/pybind11) 2.9.1+ ([new BSD](https://github.com/pybind/pybind11/blob/master/LICENSE))
 * [NLohmann-JSON](https://github.com/nlohmann/json) 3.9.1+ ([MIT](https://github.com/nlohmann/json/blob/develop/LICENSE.MIT))
 * [toml11](https://github.com/ToruNiina/toml11) 3.7.0+ ([MIT](https://github.com/ToruNiina/toml11/blob/master/LICENSE))
 
@@ -120,7 +120,7 @@ while those can be built either with or without:
 Optional language bindings:
 * Python:
   * Python 3.6 - 3.10
-  * pybind11 2.6.2+
+  * pybind11 2.9.1+
   * numpy 1.15+
   * mpi4py 2.1+ (optional, for MPI)
   * pandas 1.0+ (optional, for dataframes)
@@ -212,6 +212,8 @@ export CMAKE_INTERPROCEDURAL_OPTIMIZATION=OFF
 python3 -m pip install openpmd-api --no-binary openpmd-api
 ```
 
+Additional CMake options can be passed via individual environment variables, which need to be prefixed with `openPMD_CMAKE_`.
+
 ### From Source
 
 [![Source Use Case](https://img.shields.io/badge/use_case-development-brightgreen)](https://cmake.org)
@@ -269,7 +271,7 @@ The following options allow to switch to external installs:
 | CMake Option                    | Values     | Library       | Version |
 |---------------------------------|------------|---------------|---------|
 | `openPMD_USE_INTERNAL_CATCH`    | **ON**/OFF | Catch2        | 2.13.4+ |
-| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11      |  2.6.2+ |
+| `openPMD_USE_INTERNAL_PYBIND11` | **ON**/OFF | pybind11      |  2.9.1+ |
 | `openPMD_USE_INTERNAL_JSON`     | **ON**/OFF | NLohmann-JSON |  3.9.1+ |
 | `openPMD_USE_INTERNAL_TOML11`   | **ON**/OFF | toml11        |  3.7.0+ |
 

docs/source/details/backendconfig.rst

@@ -35,6 +35,7 @@ Parameters that are directly passed through to an external library and not inter
 
 The configuration string may refer to the complete ``openPMD::Series`` or may additionally be specified per ``openPMD::Dataset``, passed in the respective constructors.
 This reflects the fact that certain backend-specific parameters may refer to the whole Series (such as storage engines and their parameters) and others refer to actual datasets (such as compression).
+Dataset-specific configurations are (currently) only available during dataset creation, but not when reading datasets.
 
 A JSON/TOML configuration may either be specified as an inline string that can be parsed as a JSON/TOML object, or  alternatively as a path to a JSON/TOML-formatted text file (only in the constructor of ``openPMD::Series``):
 
@@ -129,6 +130,29 @@ Explanation of the single keys:
   This workaround is enabled on a per-dataset level.
   The workaround can be completely deactivated by specifying ``{"adios2": {"use_span_based_put": true}}`` or it can alternatively be activated indiscriminately for all datasets by specifying ``{"adios2": {"use_span_based_put": false}}``.
 
+Operations specified inside ``adios2.dataset.operators`` will be applied to ADIOS2 datasets in writing as well as in reading.
+Beginning with ADIOS2 2.8.0, this can be used to specify decompressor settings:
+
+.. code-block:: json
+
+  {
+    "adios2": {
+      "dataset": {
+        "operators": [
+          {
+            "type": "blosc",
+            "parameters": {
+              "nthreads": 2
+            }
+          }
+        ]
+      }
+    }
+  }
+
+In older ADIOS2 versions, this specification will be without effect in read mode.
+Dataset-specific configurations are (currently) only possible when creating datasets, not when reading.
+
 Any setting specified under ``adios2.dataset`` is applicable globally as well as on a per-dataset level.
 Any setting under ``adios2.engine`` is applicable globally only.
 

docs/source/dev/buildoptions.rst

@@ -69,7 +69,7 @@ The following options allow to switch to external installs of dependencies:
 CMake Option                      Values      Installs Library       Version
 ================================= =========== ======== ============= ========
 ``openPMD_USE_INTERNAL_CATCH``    **ON**/OFF  No       Catch2         2.13.4+
-``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF  No       pybind11        2.6.2+
+``openPMD_USE_INTERNAL_PYBIND11`` **ON**/OFF  No       pybind11        2.9.1+
 ``openPMD_USE_INTERNAL_JSON``     **ON**/OFF  No       NLohmann-JSON   3.9.1+
 ``openPMD_USE_INTERNAL_TOML11``   **ON**/OFF  No       toml11          3.7.0+
 ================================= =========== ======== ============= ========