diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9b21230ce3..8566ab6beb 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -266,10 +266,11 @@ jobs: env: {CXXFLAGS: -Werror, PKG_CONFIG_PATH: /usr/lib/x86_64-linux-gnu/pkgconfig} run: | share/openPMD/download_samples.sh build - cmake -S . -B build \ - -DopenPMD_USE_PYTHON=ON \ - -DopenPMD_USE_MPI=ON \ - -DopenPMD_USE_HDF5=ON \ + cmake -S . -B build \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=ON \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_FILESYSTEM_HEADER=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 4 ctest --test-dir build --output-on-failure @@ -290,11 +291,12 @@ jobs: env: {CXXFLAGS: -Werror} run: | share/openPMD/download_samples.sh build - cmake -S . -B build \ - -DopenPMD_USE_PYTHON=ON \ - -DopenPMD_USE_MPI=OFF \ - -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_INVASIVE_TESTS=ON \ + cmake -S . -B build \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=OFF \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_INVASIVE_TESTS=ON \ + -DopenPMD_USE_FILESYSTEM_HEADER=ON \ -DPython_EXECUTABLE=$(which python3.10) cmake --build build --parallel 4 ctest --test-dir build --output-on-failure @@ -317,13 +319,14 @@ jobs: env: {CXXFLAGS: -Werror} run: | share/openPMD/download_samples.sh build - cmake -S . -B build \ + cmake -S . -B build \ -DCMAKE_CXX_FLAGS="-Wno-error=stringop-overread" \ - -DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \ - -DopenPMD_USE_PYTHON=ON \ - -DopenPMD_USE_MPI=ON \ - -DopenPMD_USE_HDF5=ON \ - -DopenPMD_USE_ADIOS2=ON \ + -DCMAKE_C_FLAGS="-Wno-error=stringop-overread" \ + -DopenPMD_USE_PYTHON=ON \ + -DopenPMD_USE_MPI=ON \ + -DopenPMD_USE_HDF5=ON \ + -DopenPMD_USE_ADIOS2=ON \ + -DopenPMD_USE_FILESYSTEM_HEADER=ON \ -DopenPMD_USE_INVASIVE_TESTS=ON cmake --build build --parallel 4 ctest --test-dir build --output-on-failure diff --git a/CMakeLists.txt b/CMakeLists.txt index e8a60b7f6c..3533cb29a4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -151,6 +151,8 @@ option(openPMD_USE_INTERNAL_TOML11 "Use internally shipped toml11" ${op option(openPMD_USE_INVASIVE_TESTS "Enable unit tests that modify source code" OFF) option(openPMD_USE_VERIFY "Enable internal VERIFY (assert) macro independent of build type" ON) +option(openPMD_USE_FILESYSTEM_HEADER "Enable filesystem header. May be disabled for old compiler versions." OFF) +mark_as_advanced(openPMD_USE_FILESYSTEM_HEADER) set(CMAKE_CONFIGURATION_TYPES "Release;Debug;MinSizeRel;RelWithDebInfo") if(NOT CMAKE_BUILD_TYPE) @@ -552,6 +554,7 @@ endif() if(openPMD_HAVE_PYTHON) add_library(openPMD.py MODULE src/binding/python/openPMD.cpp + src/binding/python/auxiliary.cpp src/binding/python/Access.cpp src/binding/python/Attributable.cpp src/binding/python/BaseRecordComponent.cpp @@ -732,6 +735,18 @@ if(openPMD_USE_INVASIVE_TESTS) target_compile_definitions(openPMD PRIVATE openPMD_USE_INVASIVE_TESTS=1) endif() +function(set_filesystem_header_for_target target) + if(openPMD_USE_FILESYSTEM_HEADER) + target_compile_definitions(${target} PRIVATE openPMD_USE_FILESYSTEM_HEADER=1) + else() + target_compile_definitions(${target} PRIVATE openPMD_USE_FILESYSTEM_HEADER=0) + endif() +endfunction() +set_filesystem_header_for_target(openPMD) +if(openPMD_HAVE_PYTHON) + set_filesystem_header_for_target(openPMD.py) +endif() + if(openPMD_BUILD_TESTING) # compile Catch2 implementation part separately add_library(CatchRunner ${_openpmd_lib_type} diff --git a/docs/source/dev/buildoptions.rst b/docs/source/dev/buildoptions.rst index e4bda142aa..b2c483f34e 100644 --- a/docs/source/dev/buildoptions.rst +++ b/docs/source/dev/buildoptions.rst @@ -9,24 +9,27 @@ Variants The following options can be added to the ``cmake`` call to control features. CMake controls options with prefixed ``-D``, e.g. ``-DopenPMD_USE_MPI=OFF``: -============================== =============== ======================================================================== -CMake Option Values Description -============================== =============== ======================================================================== -``openPMD_USE_MPI`` **AUTO**/ON/OFF Parallel, Multi-Node I/O for clusters -``openPMD_USE_HDF5`` **AUTO**/ON/OFF HDF5 backend (``.h5`` files) -``openPMD_USE_ADIOS2`` **AUTO**/ON/OFF ADIOS2 backend (``.bp`` files in BP3, BP4 or higher) -``openPMD_USE_PYTHON`` **AUTO**/ON/OFF Enable Python bindings -``openPMD_USE_INVASIVE_TESTS`` ON/**OFF** Enable unit tests that modify source code :sup:`1` -``openPMD_USE_VERIFY`` **ON**/OFF Enable internal VERIFY (assert) macro independent of build type :sup:`2` -``openPMD_INSTALL`` **ON**/OFF Add installation targets -``openPMD_INSTALL_RPATH`` **ON**/OFF Add RPATHs to installed binaries -``Python_EXECUTABLE`` (newest found) Path to Python executable -============================== =============== ======================================================================== +================================= =============== =============================================================================== +CMake Option Values Description +================================= =============== =============================================================================== +``openPMD_USE_MPI`` **AUTO**/ON/OFF Parallel, Multi-Node I/O for clusters +``openPMD_USE_HDF5`` **AUTO**/ON/OFF HDF5 backend (``.h5`` files) +``openPMD_USE_ADIOS2`` **AUTO**/ON/OFF ADIOS2 backend (``.bp`` files in BP3, BP4 or higher) +``openPMD_USE_PYTHON`` **AUTO**/ON/OFF Enable Python bindings +``openPMD_USE_INVASIVE_TESTS`` ON/**OFF** Enable unit tests that modify source code :sup:`1` +``openPMD_USE_VERIFY`` **ON**/OFF Enable internal VERIFY (assert) macro independent of build type :sup:`2` +``openPMD_INSTALL`` **ON**/OFF Add installation targets +``openPMD_INSTALL_RPATH`` **ON**/OFF Add RPATHs to installed binaries +``Python_EXECUTABLE`` (newest found) Path to Python executable +``openPMD_USE_FILESYSTEM_HEADER`` ON/**OFF** In-/Exclude optional features implemented with ```` header :sup:`3` +================================= =============== =============================================================================== :sup:`1` e.g. changes C++ visibility keywords, breaks MSVC :sup:`2` this includes most pre-/post-condition checks, disabling without specific cause is highly discouraged +:sup:`3` currently only used for supporting ``pathlib.Path``-type arguments in the Python API; a manual fallback implementation is used otherwise + Shared or Static ---------------- diff --git a/examples/10_streaming_read.py b/examples/10_streaming_read.py index f33d778842..740b3a769d 100755 --- a/examples/10_streaming_read.py +++ b/examples/10_streaming_read.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import json import sys import openpmd_api as io @@ -18,8 +17,7 @@ print("SST engine not available in ADIOS2.") sys.exit(0) - series = io.Series("simData.sst", io.Access_Type.read_linear, - json.dumps(config)) + series = io.Series("simData.sst", io.Access_Type.read_linear, config) # Read all available iterations and print electron position data. # Direct access to iterations is possible via `series.iterations`. diff --git a/examples/10_streaming_write.py b/examples/10_streaming_write.py index 575079ea6d..ffbfc67d80 100755 --- a/examples/10_streaming_write.py +++ b/examples/10_streaming_write.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -import json import sys import numpy as np @@ -21,8 +20,7 @@ # create a series and specify some global metadata # change the file extension to .json, .h5 or .bp for regular file writing - series = io.Series("simData.sst", io.Access_Type.create, - json.dumps(config)) + series = io.Series("simData.sst", io.Access_Type.create, config) series.set_author("Franz Poeschel ") series.set_software("openPMD-api-python-examples") diff --git a/examples/13_write_dynamic_configuration.py b/examples/13_write_dynamic_configuration.py index a0f7b0c213..4dac58e197 100644 --- a/examples/13_write_dynamic_configuration.py +++ b/examples/13_write_dynamic_configuration.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -import json - import numpy as np import openpmd_api as io @@ -19,6 +17,9 @@ # Alternatively, the location of a JSON/TOML-file on the filesystem can # be passed by adding an at-sign `@` in front of the path # The format will then be recognized by filename extension, i.e. .json or .toml +# In Python, normal Python dictionaries can also be used which will then be +# converted via `json.dumps()` in the Series constructor +# (see below for an example in terms of the Dataset constructor) backend = "adios2" iteration_encoding = "group_based" @@ -105,9 +106,6 @@ def main(): 'dataset': { 'operators': [] } - }, - 'adios1': { - 'dataset': {} } } config['adios2']['dataset'] = { @@ -118,9 +116,6 @@ def main(): } }] } - config['adios1']['dataset'] = { - 'transform': 'blosc:compressor=zlib,shuffle=bit,lvl=1;nometa' - } temperature = iteration.meshes["temperature"] temperature.unit_dimension = {io.Unit_Dimension.theta: 1.0} @@ -129,8 +124,7 @@ def main(): # temperature has no x,y,z components, so skip the last layer: temperature_dataset = temperature # let's say we are in a 3x3 mesh - dataset = io.Dataset(np.dtype("double"), [3, 3]) - dataset.options = json.dumps(config) + dataset = io.Dataset(np.dtype("double"), [3, 3], config) temperature_dataset.reset_dataset(dataset) # temperature is constant local_data = np.arange(i * 9, (i + 1) * 9, dtype=np.dtype("double")) diff --git a/examples/7_extended_write_serial.py b/examples/7_extended_write_serial.py index f63a5c8371..a1409b31ef 100755 --- a/examples/7_extended_write_serial.py +++ b/examples/7_extended_write_serial.py @@ -6,8 +6,6 @@ Authors: Axel Huebl, Fabian Koller License: LGPLv3+ """ -import json - import numpy as np from openpmd_api import (Access, Dataset, Mesh_Record_Component, Series, Unit_Dimension) @@ -104,7 +102,6 @@ # before storing record data, you must specify the dataset once per # component this describes the datatype and shape of data as it should be # written to disk - d = Dataset(partial_mesh.dtype, extent=[2, 5]) dataset_config = { "adios2": { "dataset": { @@ -117,7 +114,7 @@ } } } - d.options = json.dumps(dataset_config) + d = Dataset(partial_mesh.dtype, extent=[2, 5], options=dataset_config) mesh["x"].reset_dataset(d) electrons = cur_it.particles["electrons"] diff --git a/examples/9_particle_write_serial.py b/examples/9_particle_write_serial.py index 10a9557d21..788e7c073a 100644 --- a/examples/9_particle_write_serial.py +++ b/examples/9_particle_write_serial.py @@ -6,19 +6,16 @@ Authors: Axel Huebl License: LGPLv3+ """ -import numpy as np -from openpmd_api import (Access, Dataset, Mesh_Record_Component, Series, - Unit_Dimension) - -SCALAR = Mesh_Record_Component.SCALAR +from pathlib import Path +import numpy as np +from openpmd_api import Access, Dataset, Series, Unit_Dimension if __name__ == "__main__": # open file for writing - f = Series( - "../samples/9_particle_write_serial_py.h5", - Access.create - ) + samples = Path("../samples") + filename = "9_particle_write_serial_py.h5" + f = Series(samples / filename, Access.create) # all required openPMD attributes will be set to reasonable default values # (all ones, all zeros, empty strings,...) diff --git a/include/openPMD/binding/python/Common.hpp b/include/openPMD/binding/python/Common.hpp index 7b42b919d8..c72d72ce83 100644 --- a/include/openPMD/binding/python/Common.hpp +++ b/include/openPMD/binding/python/Common.hpp @@ -26,6 +26,10 @@ #include #include #include + +#if openPMD_USE_FILESYSTEM_HEADER +#include +#endif // not yet used: // pybind11/functional.h // for std::function diff --git a/include/openPMD/binding/python/Mpi.hpp b/include/openPMD/binding/python/Mpi.hpp index dc110e0ca1..5de6077f09 100644 --- a/include/openPMD/binding/python/Mpi.hpp +++ b/include/openPMD/binding/python/Mpi.hpp @@ -43,7 +43,25 @@ struct openPMD_PyMPICommObject }; using openPMD_PyMPIIntracommObject = openPMD_PyMPICommObject; -inline std::variant +struct py_object_to_mpi_comm_error +{ + enum class error_type + { + invalid_data, + is_not_an_mpi_communicator + }; + using error_type = error_type; + + std::string error_msg; + error_type type; + + operator std::string() const + { + return error_msg; + } +}; + +inline std::variant pythonObjectAsMpiComm(pybind11::object &comm) { namespace py = pybind11; @@ -55,10 +73,13 @@ pythonObjectAsMpiComm(pybind11::object &comm) //! - installed: include/mpi4py/mpi4py.MPI_api.h // if( import_mpi4py() < 0 ) { here be dragons } + using e = py_object_to_mpi_comm_error; + using e_t = e::error_type; + if (comm.ptr() == Py_None) - return {"MPI communicator cannot be None."}; + return e{"MPI communicator cannot be None.", e_t::invalid_data}; if (comm.ptr() == nullptr) - return {"MPI communicator is a nullptr."}; + return e{"MPI communicator is a nullptr.", e_t::invalid_data}; // check type string to see if this is mpi4py // __str__ (pretty) @@ -68,15 +89,18 @@ pythonObjectAsMpiComm(pybind11::object &comm) py::str const comm_pystr = py::repr(comm); std::string const comm_str = comm_pystr.cast(); if (comm_str.substr(0, 12) != std::string(" >( comm.get_type())) // TODO add mpi4py version from above import check to error message - return { + return e{ "comm has unexpected type layout in " + comm_str + - " (Mismatched MPI at compile vs. runtime? " - "Breaking mpi4py release?)"}; + " (Mismatched MPI at compile vs. runtime? " + "Breaking mpi4py release?)", + e_t::invalid_data}; // todo other possible implementations: // - pyMPI (inactive since 2008?): import mpi; mpi.WORLD @@ -87,12 +111,13 @@ pythonObjectAsMpiComm(pybind11::object &comm) &((openPMD_PyMPIIntracommObject *)(comm.ptr()))->ob_mpi; if (PyErr_Occurred()) - return {"MPI communicator access error."}; + return e{"MPI communicator access error.", e_t::invalid_data}; if (mpiCommPtr == nullptr) { - return { + return e{ "MPI communicator cast failed. " - "(Mismatched MPI at compile vs. runtime?)"}; + "(Mismatched MPI at compile vs. runtime?)", + e_t::invalid_data}; } return {*mpiCommPtr}; } diff --git a/include/openPMD/binding/python/auxiliary.hpp b/include/openPMD/binding/python/auxiliary.hpp new file mode 100644 index 0000000000..41c439e676 --- /dev/null +++ b/include/openPMD/binding/python/auxiliary.hpp @@ -0,0 +1,68 @@ +/* Copyright 2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#pragma once + +#include "openPMD/binding/python/Common.hpp" + +namespace auxiliary +{ +auto json_dumps(py::object const &obj) -> std::string; + +/* + * Functor is a struct of the form: + * + * struct Functor + * { + * template + * static void call(... any kind of argument ...); + * }; + * + * The variadic parameter pack (Types) specifies types which to supply for T. + * + * ForEachTypeNested::call(...args...) will then + * call Functor::template call() for each type T in T1, T2, ... + * one after another. + */ +template +struct ForEachType; + +template +struct ForEachType +{ + template + static void call(Args &&...args) + { + Functor::template call(args...); + ForEachType::template call( + std::forward(args)...); + } +}; + +template +struct ForEachType +{ + template + static constexpr void call(Args &&...) + { /* no-op */ + } +}; +} // namespace auxiliary diff --git a/src/binding/python/ChunkInfo.cpp b/src/binding/python/ChunkInfo.cpp index a392cdd3e2..a622c1f073 100644 --- a/src/binding/python/ChunkInfo.cpp +++ b/src/binding/python/ChunkInfo.cpp @@ -83,9 +83,10 @@ void init_Chunk(py::module &m) "get_collective", [](host_info::Method const &self, py::object &comm) { auto variant = pythonObjectAsMpiComm(comm); - if (auto errorMsg = std::get_if(&variant)) + if (auto errorMsg = + std::get_if(&variant)) { - throw std::runtime_error("[Series] " + *errorMsg); + throw std::runtime_error("[Series] " + errorMsg->error_msg); } else { diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index 70d85721f2..6a3e70993d 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -22,6 +22,7 @@ #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" +#include "openPMD/binding/python/auxiliary.hpp" #include @@ -29,27 +30,35 @@ void init_Dataset(py::module &m) { auto pyDataset = py::class_(m, "Dataset") + .def(py::init(), py::arg("extent")) .def( - py::init(), + py::init(), py::arg("dtype"), - py::arg("extent")) - .def(py::init(), py::arg("extent")) + py::arg("extent"), + py::arg("options") = "{}") .def( - py::init([](py::dtype dt, Extent const &e) { + py::init([](py::dtype dt, Extent e, std::string options) { auto const d = dtype_from_numpy(std::move(dt)); - return new Dataset{d, e}; + return new Dataset{d, std::move(e), std::move(options)}; }), py::arg("dtype"), - py::arg("extent")) + py::arg("extent"), + py::arg("options") = "{}") .def( - py::init(), + py::init([](Datatype dt, Extent e, py::object const &options) { + auto resolved_options = ::auxiliary::json_dumps(options); + return new Dataset{ + dt, std::move(e), std::move(resolved_options)}; + }), py::arg("dtype"), py::arg("extent"), py::arg("options")) .def( - py::init([](py::dtype dt, Extent e, std::string options) { + py::init([](py::dtype dt, Extent e, py::object const &options) { auto const d = dtype_from_numpy(std::move(dt)); - return new Dataset{d, std::move(e), std::move(options)}; + auto resolved_options = ::auxiliary::json_dumps(options); + return new Dataset{ + d, std::move(e), std::move(resolved_options)}; }), py::arg("dtype"), py::arg("extent"), diff --git a/src/binding/python/Series.cpp b/src/binding/python/Series.cpp index 11ad2ae651..c46d069185 100644 --- a/src/binding/python/Series.cpp +++ b/src/binding/python/Series.cpp @@ -19,17 +19,24 @@ * If not, see . */ #include "openPMD/Series.hpp" +#include "openPMD/Error.hpp" #include "openPMD/IO/Access.hpp" #include "openPMD/Iteration.hpp" #include "openPMD/IterationEncoding.hpp" #include "openPMD/auxiliary/JSON.hpp" +#include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Pickle.hpp" +#include "openPMD/binding/python/auxiliary.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" #include "openPMD/snapshots/StatefulIterator.hpp" -#include "openPMD/binding/python/Common.hpp" #include +#include + +#if openPMD_USE_FILESYSTEM_HEADER +#include +#endif #if openPMD_HAVE_MPI // re-implemented signatures: @@ -41,6 +48,200 @@ #include #include +namespace internal +{ +struct DefineSeriesConstructorPerPathType +{ + static constexpr auto json_cfg_as_string(std::string const &str) + -> std::string const & + { + return str; + } + + static auto json_cfg_as_string(py::object const &obj) -> std::string + { +#if !openPMD_USE_FILESYSTEM_HEADER + auto type_repr = py::repr(obj).cast(); + if (type_repr.substr(0, 9) == "PosixPath" || + type_repr.substr(0, 11) == "WindowsPath") + { + auto casted = obj.attr("__str__")(); + return "@" + py::cast(casted); + } +#endif + return ::auxiliary::json_dumps(obj); + } + +#if openPMD_USE_FILESYSTEM_HEADER + static auto json_cfg_as_string(std::filesystem::path const &cfg_path) + -> std::string + { + return "@" + std::string(cfg_path); + } +#endif + + static constexpr auto filepath_as_string(std::string const &str) + -> std::string const & + { + return str; + } + +#if openPMD_USE_FILESYSTEM_HEADER + static auto filepath_as_string(std::filesystem::path const &path) + -> std::string + { + return path; + } +#else + static auto filepath_as_string(py::object const &path) -> std::string + { + auto type_repr = py::repr(path).cast(); + if (type_repr.substr(0, 9) != "PosixPath" && + type_repr.substr(0, 11) != "WindowsPath") + { + throw error::WrongAPIUsage( + "openpmd.Series constructor: 'filepath' argument may either be " + "a String or pathlib.Path."); + } + auto casted = path.attr("__str__")(); + return py::cast(casted); + } +#endif + + template + static void call(py::class_ &py_class) + { + using PathType = typename std::tuple_element<0, TupleType>::type; + using JsonCfgType = typename std::tuple_element<1, TupleType>::type; + + py_class + .def( + py::init([](PathType const &filepath, + Access at, + JsonCfgType const &options) { + decltype(auto) filepath_ = filepath_as_string(filepath); + decltype(auto) options_ = json_cfg_as_string(options); + py::gil_scoped_release release; + return new Series(filepath_, at, options_); + }), + py::arg("filepath"), + py::arg("access"), + py::arg("options") = "{}", + R"END( +Construct a new Series. Parameters: + +* filepath: The file path, either as a String or as pathlib.Path. +* at: Access mode. +* options: Advanced backend configuration via JSON. + May be specified as: + + 1. a Python object representing the JSON structure, + 2. a JSON-formatted string directly, + 3. a pathlib.Path to a JSON textfile, + 3. as a String-type path to a JSON textfile, prepended by an at sign '@'. + +For details on access modes, JSON/TOML configuration and iteration encoding, +refer to: + +* https://openpmd-api.readthedocs.io/en/latest/usage/workflow.html#access-modes +* https://openpmd-api.readthedocs.io/en/latest/details/backendconfig.html +* https://openpmd-api.readthedocs.io/en/latest/usage/concepts.html#iteration-and-series + +In case of file-based iteration encoding, the file names for each +iteration are determined by an expansion pattern that must be specified. +It takes one out of two possible forms: + +1. Simple form: %T is replaced with the iteration index, e.g. + `simData_%T.bp` becomes `simData_50.bp`. +2. Padded form: e.g. %06T is replaced with the iteration index padded to + at least six digits. `simData_%06T.bp` becomes `simData_000050.bp`. + +The backend is determined: + +1. Explicitly via the JSON/TOML parameter `backend`, e.g. `{"backend": + "adios2"}`. +2. Otherwise implicitly from the filename extension, e.g. + `simData_%T.h5`. + +The filename extension can be replaced with a globbing pattern %E. +It will be replaced with an automatically determined file name extension: + +1. In CREATE mode: The extension is set to a backend-specific default + extension. This requires that the backend is specified via JSON/TOML. +2. In READ_ONLY, READ_WRITE and READ_LINEAR modes: These modes require + that files already exist on disk. The disk will be scanned for files + that match the pattern and the resulting file extension will be used. + If the result is ambiguous or no such file is found, an error is + raised. +3. In APPEND mode: Like (2.), except if no matching file is found. In + that case, the procedure of (1.) is used, owing to the fact that + APPEND mode can be used to create new datasets. + )END") +#if openPMD_HAVE_MPI + .def( + py::init([](PathType const &filepath, + Access at, + py::object &comm, + JsonCfgType const &options) { + decltype(auto) filepath_ = filepath_as_string(filepath); + decltype(auto) options_ = json_cfg_as_string(options); + auto variant = pythonObjectAsMpiComm(comm); + if (auto errorMsg = + std::get_if(&variant)) + { + switch (errorMsg->type) + { + + case py_object_to_mpi_comm_error::error_type:: + invalid_data: + throw std::runtime_error( + "[Series] " + errorMsg->error_msg); + case py_object_to_mpi_comm_error::error_type:: + is_not_an_mpi_communicator: + /* + * Since this overload accepts py::object for an MPI + * communicator, we need to test if this is actually + * an MPI communicator. If not, this is not + * immediately an error, we might have just tried to + * call the wrong overload. Go back to trying the + * other overloads by throwing to pybind. + */ + throw py::reference_cast_error(); + } + throw std::runtime_error("Unreachable!"); + } + else + { + py::gil_scoped_release release; + return new Series( + filepath_, + at, + std::get(variant), + options_); + } + }), + py::arg("filepath"), + py::arg("access"), + py::arg("mpi_communicator"), + py::arg("options") = "{}", + R"END( +Construct a new Series. Parameters: + +* filepath: The file path. +* at: Access mode. +* options: Advanced backend configuration via JSON. + May be specified as a JSON-formatted string directly, or as a path + to a JSON textfile, prepended by an at sign '@'. +* mpi_communicator: The MPI communicator + +For further details, refer to the non-MPI overload. + )END") +#endif + ; + } +}; +} // namespace internal + struct StatefulIteratorPythonAdaptor : LegacyIteratorAdaptor { StatefulIteratorPythonAdaptor(LegacyIteratorAdaptor it) @@ -166,107 +367,28 @@ not possible once it has been closed. // keep handle alive while iterator exists py::keep_alive<0, 1>()); - // `clang-format on/off` doesn't help here. - // Writing this without a macro would lead to a huge diff due to - // clang-format. -#define OPENPMD_AVOID_CLANG_FORMAT auto cl = - OPENPMD_AVOID_CLANG_FORMAT -#undef OPENPMD_AVOID_CLANG_FORMAT - - py::class_(m, "Series") - - .def( - py::init([](std::string const &filepath, - Access at, - std::string const &options) { - py::gil_scoped_release release; - return new Series(filepath, at, options); - }), - py::arg("filepath"), - py::arg("access"), - py::arg("options") = "{}", - R"END( -Construct a new Series. Parameters: - -* filepath: The file path. -* at: Access mode. -* options: Advanced backend configuration via JSON. - May be specified as a JSON-formatted string directly, or as a path - to a JSON textfile, prepended by an at sign '@'. - -For details on access modes, JSON/TOML configuration and iteration encoding, -refer to: - -* https://openpmd-api.readthedocs.io/en/latest/usage/workflow.html#access-modes -* https://openpmd-api.readthedocs.io/en/latest/details/backendconfig.html -* https://openpmd-api.readthedocs.io/en/latest/usage/concepts.html#iteration-and-series - -In case of file-based iteration encoding, the file names for each -iteration are determined by an expansion pattern that must be specified. -It takes one out of two possible forms: - -1. Simple form: %T is replaced with the iteration index, e.g. - `simData_%T.bp` becomes `simData_50.bp`. -2. Padded form: e.g. %06T is replaced with the iteration index padded to - at least six digits. `simData_%06T.bp` becomes `simData_000050.bp`. - -The backend is determined: - -1. Explicitly via the JSON/TOML parameter `backend`, e.g. `{"backend": - "adios2"}`. -2. Otherwise implicitly from the filename extension, e.g. - `simData_%T.h5`. - -The filename extension can be replaced with a globbing pattern %E. -It will be replaced with an automatically determined file name extension: - -1. In CREATE mode: The extension is set to a backend-specific default - extension. This requires that the backend is specified via JSON/TOML. -2. In READ_ONLY, READ_WRITE and READ_LINEAR modes: These modes require - that files already exist on disk. The disk will be scanned for files - that match the pattern and the resulting file extension will be used. - If the result is ambiguous or no such file is found, an error is - raised. -3. In APPEND mode: Like (2.), except if no matching file is found. In - that case, the procedure of (1.) is used, owing to the fact that - APPEND mode can be used to create new datasets. - )END") -#if openPMD_HAVE_MPI - .def( - py::init([](std::string const &filepath, - Access at, - py::object &comm, - std::string const &options) { - auto variant = pythonObjectAsMpiComm(comm); - if (auto errorMsg = std::get_if(&variant)) - { - throw std::runtime_error("[Series] " + *errorMsg); - } - else - { - py::gil_scoped_release release; - return new Series( - filepath, at, std::get(variant), options); - } - }), - py::arg("filepath"), - py::arg("access"), - py::arg("mpi_communicator"), - py::arg("options") = "{}", - R"END( -Construct a new Series. Parameters: - -* filepath: The file path. -* at: Access mode. -* options: Advanced backend configuration via JSON. - May be specified as a JSON-formatted string directly, or as a path - to a JSON textfile, prepended by an at sign '@'. -* mpi_communicator: The MPI communicator - -For further details, refer to the non-MPI overload. - )END") + py::class_ cl(m, "Series"); + ::auxiliary::ForEachType< + ::internal::DefineSeriesConstructorPerPathType, + // First tuple components are eligible types for the path argument + // second component for the config argument + // py::object needs to always come last, as a catch-all pattern +#if openPMD_USE_FILESYSTEM_HEADER + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple, + std::tuple +#else + std::tuple, + std::tuple, + std::tuple, + std::tuple #endif - .def("__bool__", &Series::operator bool) + >::template call(cl); + + cl.def("__bool__", &Series::operator bool) .def("__len__", [](Series const &s) { return s.iterations.size(); }) .def( "__repr__", @@ -477,9 +599,11 @@ users to overwrite default options, while keeping any other ones. std::string const &overwrite, py::object &comm) { auto variant = pythonObjectAsMpiComm(comm); - if (auto errorMsg = std::get_if(&variant)) + if (auto errorMsg = + std::get_if(&variant)) { - throw std::runtime_error("[merge_json] " + *errorMsg); + throw std::runtime_error( + "[merge_json] " + std::string(*errorMsg)); } else { diff --git a/src/binding/python/auxiliary.cpp b/src/binding/python/auxiliary.cpp new file mode 100644 index 0000000000..5e927d1d54 --- /dev/null +++ b/src/binding/python/auxiliary.cpp @@ -0,0 +1,33 @@ +/* Copyright 2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/binding/python/auxiliary.hpp" + +namespace auxiliary +{ +auto json_dumps(py::object const &obj) -> std::string +{ + py::module_ json = py::module_::import("json"); + auto dumps = json.attr("dumps"); + auto dumped = dumps(obj); + return py::cast(dumped); +} +} // namespace auxiliary diff --git a/test/python/unittest/API/APITest.py b/test/python/unittest/API/APITest.py index 1566dcad06..588a723925 100644 --- a/test/python/unittest/API/APITest.py +++ b/test/python/unittest/API/APITest.py @@ -1881,17 +1881,15 @@ def testCloseIteration(self): def makeIteratorRoundTrip(self, backend, file_ending): # write - jsonConfig = """ -{ - "defer_iteration_parsing": true, - "adios2": { - "engine": { - "type": "bp4", - "usesteps": true - } - } -} -""" + jsonConfig = { + "defer_iteration_parsing": True, + "adios2": { + "engine": { + "type": "bp4", + "usesteps": True + } + } + } series = io.Series( "../samples/unittest_serialIterator." + file_ending, io.Access_Type.create, @@ -1985,7 +1983,7 @@ def makeAvailableChunksRoundTrip(self, ext): read = io.Series( name, io.Access_Type.read_only, - options='{"defer_iteration_parsing": true}' + options={"defer_iteration_parsing": True} ) read.iterations[0].open() @@ -2073,51 +2071,49 @@ def testWriteFromTemporary(self): self.writeFromTemporary(ext) def testJsonConfigADIOS2(self): - global_config = """ -{ - "adios2": { - "engine": { - "type": "bp3", - "unused": "parameter", - "parameters": { - "BufferGrowthFactor": "2.0", - "Profile": "On" - } - }, - "unused": "as well", - "dataset": { - "operators": [ - { - "type": "blosc", - "parameters": { - "clevel": "1", - "doshuffle": "BLOSC_BITSHUFFLE" - } + global_config = { + "adios2": { + "engine": { + "type": "bp3", + "unused": "parameter", + "parameters": { + "BufferGrowthFactor": "2.0", + "Profile": "On" + } + }, + "unused": "as well", + "dataset": { + "operators": [ + { + "type": "blosc", + "parameters": { + "clevel": "1", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } + ] + } + } } - ] - } - } -} -""" - local_config = """ -{ - "adios2": { - "unused": "dataset parameter", - "dataset": { - "unused": "too", - "operators": [ - { - "type": "blosc", - "parameters": { - "clevel": "3", - "doshuffle": "BLOSC_BITSHUFFLE" - } + + local_config = { + "adios2": { + "unused": "dataset parameter", + "dataset": { + "unused": "too", + "operators": [ + { + "type": "blosc", + "parameters": { + "clevel": "3", + "doshuffle": "BLOSC_BITSHUFFLE" + } + } + ] + } + } } - ] - } - } -} -""" + if not io.variants['adios2']: return series = io.Series( @@ -2223,6 +2219,26 @@ def testCustomGeometries(self): self.assertEqual(e_chargeDensity.geometry, io.Geometry.other) self.assertEqual(e_chargeDensity.geometry_string, "other") + def testSeriesConstructors(self): + import json + from pathlib import Path + + cfg = {"iteration_encoding": "variable_based"} + cfg_as_string = json.dumps(cfg) + cfg_as_file = "../samples/cfg.json" + with open(cfg_as_file, 'w') as f: + json.dump(cfg, f) + cfg_as_filepath = Path(cfg_as_file) + + series_path = "../samples/series_constructors.json" + series_filepath = Path(series_path) + + for f in [series_path, series_filepath]: + for c in [cfg, cfg_as_string, f"@{cfg_as_file}", cfg_as_filepath]: + # print(f"Creating Series with '{f}'\t'{c}'") + s = io.Series(f, io.Access.create, c) + s.close() + if __name__ == '__main__': unittest.main()