diff --git a/docs/source/analysis/paraview.rst b/docs/source/analysis/paraview.rst index e0c837ca0a..7a2b57bbf0 100644 --- a/docs/source/analysis/paraview.rst +++ b/docs/source/analysis/paraview.rst @@ -44,7 +44,7 @@ The file contains the same string as one would put in an openPMD ``Series("...." .. warning:: As of ParaView 5.11 and older, the axisLabel is not yet read for fields. - See, e.g., `WarpX issue 21162 `__. + See, e.g., `WarpX issue 21162 `__. Please apply rotation of, e.g., ``0 -90 0`` to mesh data where needed. .. warning:: diff --git a/docs/source/backends/adios2.rst b/docs/source/backends/adios2.rst index 55f080494c..16347cf15d 100644 --- a/docs/source/backends/adios2.rst +++ b/docs/source/backends/adios2.rst @@ -78,23 +78,24 @@ The ADIOS2 SST engine for streaming can be picked by specifying the ending ``.ss The following environment variables control ADIOS2 I/O behavior at runtime. Fine-tuning these is especially useful when running at large scale. -===================================== ========== ================================================================================ -environment variable default description -===================================== ========== ================================================================================ -``OPENPMD_ADIOS2_HAVE_PROFILING`` ``1`` Turns on/off profiling information right after a run. -``OPENPMD_ADIOS2_HAVE_METADATA_FILE`` ``1`` Online creation of the adios journal file (``1``: yes, ``0``: no). -``OPENPMD_ADIOS2_NUM_SUBSTREAMS`` ``0`` Number of files to be created, 0 indicates maximum number possible. -``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ -``OPENPMD_ADIOS2_PRETEND_ENGINE`` *empty* Pretend that an (unknown) ADIOS2 engine is in fact another one (also see the ``adios2.pretend_engine`` :ref:`parameter `). -``OPENPMD2_ADIOS2_USE_GROUP_TABLE`` ``0`` Use group table (see below) -``OPENPMD_ADIOS2_STATS_LEVEL`` ``0`` whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no). -``OPENPMD_ADIOS2_ASYNC_WRITE`` ``0`` ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``). -``OPENPMD_ADIOS2_BP5_BufferChunkMB`` ``0`` ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation -``OPENPMD_ADIOS2_BP5_MaxShmMB`` ``0`` ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation -``OPENPMD_ADIOS2_BP5_NumSubFiles`` ``0`` ADIOS2 BP5 engine: num of subfiles -``OPENPMD_ADIOS2_BP5_NumAgg`` ``0`` ADIOS2 BP5 engine: num of aggregators -``OPENPMD_ADIOS2_BP5_TypeAgg`` *empty* ADIOS2 BP5 engine: aggregation type. (EveryoneWrites, EveryoneWritesSerial, TwoLevelShm) -===================================== ========== ================================================================================ +======================================= ========== ================================================================================ +environment variable default description +======================================= ========== ================================================================================ +``OPENPMD_ADIOS2_HAVE_PROFILING`` ``1`` Turns on/off profiling information right after a run. +``OPENPMD_ADIOS2_HAVE_METADATA_FILE`` ``1`` Online creation of the adios journal file (``1``: yes, ``0``: no). +``OPENPMD_ADIOS2_NUM_SUBSTREAMS`` ``0`` Number of files to be created, 0 indicates maximum number possible. +``OPENPMD_ADIOS2_ENGINE`` ``File`` `ADIOS2 engine `_ +``OPENPMD_ADIOS2_PRETEND_ENGINE`` *empty* Pretend that an (unknown) ADIOS2 engine is in fact another one (also see the ``adios2.pretend_engine`` :ref:`parameter `). +``OPENPMD2_ADIOS2_USE_GROUP_TABLE`` ``0`` Use group table (see below) +``OPENPMD_ADIOS2_STATS_LEVEL`` ``0`` whether to generate statistics for variables in ADIOS2. (``1``: yes, ``0``: no). +``OPENPMD_ADIOS2_ASYNC_WRITE`` ``0`` ADIOS2 BP5 engine: 1 means setting "AsyncWrite" in ADIOS2 to "on". Flushes will go to the buffer by default (see ``preferred_flush_target``). +``OPENPMD_ADIOS2_BP5_BufferChunkMB`` ``0`` ADIOS2 BP5 engine: applies when using either EveryoneWrites or EveryoneWritesSerial aggregation +``OPENPMD_ADIOS2_BP5_MaxShmMB`` ``0`` ADIOS2 BP5 engine: applies when using TwoLevelShm aggregation +``OPENPMD_ADIOS2_BP5_NumSubFiles`` ``0`` ADIOS2 BP5 engine: num of subfiles +``OPENPMD_ADIOS2_BP5_NumAgg`` ``0`` ADIOS2 BP5 engine: num of aggregators +``OPENPMD_ADIOS2_BP5_TypeAgg`` *empty* ADIOS2 BP5 engine: aggregation type. (EveryoneWrites, EveryoneWritesSerial, TwoLevelShm) +``OPENPMD_BP5_GROUPENCODING_MAX_STEPS`` ``100`` ADIOS2 BP5 engine: max number of allowed output steps in group encoding. +======================================= ========== ================================================================================ Please refer to the `ADIOS2 documentation `_ for details on I/O tuning. @@ -315,6 +316,21 @@ Rather than by reallocation as in BP4, this is done by appending a new chunk, le The default is to flush to disk (except when specifying ``OPENPMD_ADIOS2_ASYNC_WRITE=1``), but the default ``preferred_flush_target`` can also be specified via JSON/TOML at the ``Series`` level. +The BP5 engine is known to perform extremely bad for group-based encoding with many Iterations, since its design assumes that the metadata structure will be constant across output steps, while group-based encoding will add new variables and attributes for each Iteration. +The openPMD-api will hence cancel operation after 100 written Iterations in group-based encoding for BP5. +Experiments with PIConGPU show that the metadata (!) size grows from 10MB to 1GB when going from 100 to 1000 output steps in this setup. +The environment variable ``OPENPMD_BP5_GROUPENCODING_MAX_STEPS`` may be used to change this limit (specifying the limit as ``0`` will disable the check). + +For workarounds you may follow these guidelines: + +* Use file encoding by including an expansion pattern ``%T`` in the filename. +* If output to a single file is required, then: + + * Use another ADIOS2 engine, recommended is the BP4 engine by selecting file ending ``.bp4``. + * Use another openPMD backend, recommended is HDF5 by selecting file ending ``.h5``. + * (experimental) use variable encoding with BP5, either by using the API call ``Series::setIterationEncoding(IterationEncoding::variableBased)`` / ``Series.iteration_encoding = Iteration_Encoding.variable_based`` or by using the JSON/TOML configuration ``{"iteration_encoding": "variable_based"}`` / ``iteration_encoding = "variable_based"``. + Note that there is at this point no complete read support for variable-encoded outputs. + Known Issues diff --git a/examples/10_streaming_write.cpp b/examples/10_streaming_write.cpp index 2eb825ae4a..d3087ce366 100644 --- a/examples/10_streaming_write.cpp +++ b/examples/10_streaming_write.cpp @@ -20,12 +20,16 @@ int main() } // open file for writing + // use QueueFullPolicy = Discard in order to create a situation where from + // the reader's perspective steps are skipped. This tests the bug reported + // in https://github.com/openPMD/openPMD-api/issues/1747. Series series = Series("electrons.sst", Access::CREATE, R"( { "adios2": { "engine": { "parameters": { - "DataTransport": "WAN" + "DataTransport": "WAN", + "QueueFullPolicy": "Discard" } } } diff --git a/include/openPMD/Dataset.hpp b/include/openPMD/Dataset.hpp index 0032888541..a8851f555e 100644 --- a/include/openPMD/Dataset.hpp +++ b/include/openPMD/Dataset.hpp @@ -64,5 +64,6 @@ class Dataset bool empty() const; std::optional joinedDimension() const; + static std::optional joinedDimension(Extent const &); }; } // namespace openPMD diff --git a/include/openPMD/IO/ADIOS/ADIOS2File.hpp b/include/openPMD/IO/ADIOS/ADIOS2File.hpp index 0bcdaa6131..d52fb4af73 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2File.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2File.hpp @@ -25,6 +25,7 @@ #include "openPMD/IO/IOTask.hpp" #include "openPMD/IO/InvalidatableFile.hpp" #include "openPMD/config.hpp" +#include #if openPMD_HAVE_ADIOS2 #include @@ -414,11 +415,7 @@ class ADIOS2File ADIOS2IOHandlerImpl *m_impl; std::optional m_engine; //! ADIOS engine - /* - * Not all engines support the CurrentStep() call, so we have to - * implement this manually. - */ - size_t m_currentStep = 0; + std::optional m_max_steps_bp5 = std::make_optional(100); /* * ADIOS2 does not give direct access to its internal attribute and diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index db3162a2da..9e3982147f 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -32,6 +32,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" #include @@ -473,20 +474,44 @@ class ADIOS2IOHandlerImpl } } auto joinedDim = joinedDimension(shape); - if (joinedDim.has_value()) + auto make_runtime_error = [&](char const *message) { + std::stringstream s; + s << "[ADIOS2IOHandlerImpl::verifyDataset()] " << message; + s << "\nNote: Variable '" << varName << "' has shape "; + auxiliary::write_vec_to_stream(s, shape) + << ", is accessed from offset "; + auxiliary::write_vec_to_stream(s, offset) << " with extent "; + auxiliary::write_vec_to_stream(s, extent); + if (joinedDim.has_value()) + { + s << " (joined dimension on index " << *joinedDim << ")."; + } + else + { + s << " (no joined dimension)."; + } + return std::runtime_error(s.str()); + }; + if (joinedDim.has_value() || + var.ShapeID() == adios2::ShapeID::JoinedArray) { if (!offset.empty()) { - throw std::runtime_error( - "[ADIOS2] Offset must be an empty vector in case of joined " - "array."); + throw make_runtime_error( + "Offset must be an empty vector in case of joined array."); + } + if (!joinedDim.has_value()) + { + throw make_runtime_error( + "Trying to access a dataset as a non-joined array, but it " + "has previously been array."); } for (unsigned int i = 0; i < actualDim; i++) { if (*joinedDim != i && extent[i] != shape[i]) { - throw std::runtime_error( - "[ADIOS2] store_chunk extent of non-joined dimensions " + throw make_runtime_error( + "store_chunk extent of non-joined dimensions " "must be equivalent to the total extent."); } } @@ -498,8 +523,7 @@ class ADIOS2IOHandlerImpl if (!(joinedDim.has_value() && *joinedDim == i) && offset[i] + extent[i] > shape[i]) { - throw std::runtime_error( - "[ADIOS2] Dataset access out of bounds."); + throw make_runtime_error("Dataset access out of bounds."); } } } diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 731372f9e1..3a8960d6a9 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -98,6 +98,13 @@ struct OPENPMDAPI_EXPORT AbstractParameter virtual std::unique_ptr to_heap() && = 0; + // Used as a tag in some constructors to make callsites explicitly + // acknowledge that joined dimensions will not be automatically resolved + struct I_dont_want_to_use_joined_dimensions_t + {}; + constexpr static I_dont_want_to_use_joined_dimensions_t + I_dont_want_to_use_joined_dimensions{}; + protected: // avoid object slicing // by allow only child classes to use these things for defining their own @@ -327,7 +334,17 @@ template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter { - Parameter() = default; + Parameter(Dataset const &ds) + : extent(ds.extent) + , dtype(ds.dtype) + , options(ds.options) + , joinedDimension(ds.joinedDimension()) + {} + + // default constructor, but callsites need to explicitly acknowledge that + // joined dimensions will not be automatically configured when using it + Parameter(I_dont_want_to_use_joined_dimensions_t) + {} Parameter(Parameter &&) = default; Parameter(Parameter const &) = default; Parameter &operator=(Parameter &&) = default; @@ -362,7 +379,15 @@ template <> struct OPENPMDAPI_EXPORT Parameter : public AbstractParameter { - Parameter() = default; + Parameter(Extent e) : joinedDimension(Dataset::joinedDimension(e)) + { + this->extent = std::move(e); + } + + // default constructor, but callsites need to explicitly acknowledge that + // joined dimensions will not be automatically configured when using it + Parameter(I_dont_want_to_use_joined_dimensions_t) + {} Parameter(Parameter &&) = default; Parameter(Parameter const &) = default; Parameter &operator=(Parameter &&) = default; @@ -375,6 +400,7 @@ struct OPENPMDAPI_EXPORT Parameter } Extent extent = {}; + std::optional joinedDimension; }; template <> diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 542503e806..37dc7376d9 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -309,18 +309,14 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) if (!written()) { auto &rc = get(); - Parameter dCreate; - dCreate.name = rc.m_name; - dCreate.extent = getExtent(); - dCreate.dtype = getDatatype(); - dCreate.joinedDimension = joinedDimension(); if (!rc.m_dataset.has_value()) { throw error::WrongAPIUsage( "[RecordComponent] Must specify dataset type and extent before " "using storeChunk() (see RecordComponent::resetDataset())."); } - dCreate.options = rc.m_dataset.value().options; + Parameter dCreate(rc.m_dataset.value()); + dCreate.name = rc.m_name; IOHandler()->enqueue(IOTask(this, dCreate)); } Parameter getBufferView; diff --git a/include/openPMD/auxiliary/StringManip.hpp b/include/openPMD/auxiliary/StringManip.hpp index 36778f205f..eb3799d3be 100644 --- a/include/openPMD/auxiliary/StringManip.hpp +++ b/include/openPMD/auxiliary/StringManip.hpp @@ -242,5 +242,40 @@ namespace auxiliary }); return std::forward(s); } + + template + auto write_vec_to_stream(Stream &&s, Vec const &vec) -> Stream && + { + if (vec.empty()) + { + s << "[]"; + } + else + { + s << '['; + auto it = vec.begin(); + s << *it++; + auto end = vec.end(); + for (; it != end; ++it) + { + s << ", " << *it; + } + s << ']'; + } + return std::forward(s); + } + + template + auto vec_as_string(Vec const &vec) -> std::string + { + if (vec.empty()) + { + return "[]"; + } + else + { + return write_vec_to_stream(std::stringstream(), vec).str(); + } + } } // namespace auxiliary } // namespace openPMD diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index 08853b4d1e..5d3b56d98a 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -25,6 +25,21 @@ namespace openPMD {} +#if defined(CUDA_VERSION) && !defined(OPENPMD_SKIP_CHECK_ISSUE_1720) +static_assert(__cplusplus < 202002L || CUDA_VERSION >= 12040, R"( +Cannot use the openPMD-api in C++20 projects under a Cuda version lower +than 12.4.0 due to a bug in the implementation of std::variant. +Further information at: +https://github.com/openPMD/openPMD-api/issues/1720 +https://forums.developer.nvidia.com/t/nvcc-c-20-std-variant-complie-failed/270162/5 +This cannot be fixed on our side, please either upgrade to CUDA >= 12.4.0 +or use C++17. +If you think that this assertion is shown wrongly, please apply +'#define OPENPMD_SKIP_CHECK_ISSUE_1720' before including +''. +)"); +#endif + // IWYU pragma: begin_exports #include "openPMD/Dataset.hpp" #include "openPMD/Datatype.hpp" diff --git a/src/Dataset.cpp b/src/Dataset.cpp index c1546e9ef0..a56c566805 100644 --- a/src/Dataset.cpp +++ b/src/Dataset.cpp @@ -68,6 +68,11 @@ bool Dataset::empty() const } std::optional Dataset::joinedDimension() const +{ + return joinedDimension(extent); +} + +std::optional Dataset::joinedDimension(Extent const &extent) { std::optional res; for (size_t i = 0; i < extent.size(); ++i) diff --git a/src/IO/ADIOS/ADIOS2File.cpp b/src/IO/ADIOS/ADIOS2File.cpp index 1fb5bd36a5..c02206dd7d 100644 --- a/src/IO/ADIOS/ADIOS2File.cpp +++ b/src/IO/ADIOS/ADIOS2File.cpp @@ -231,6 +231,7 @@ void ADIOS2File::finalize() m_ADIOS.RemoveIO(m_IOName); } } + m_uniquePtrPuts.clear(); finalized = true; } @@ -323,9 +324,10 @@ namespace size_t ADIOS2File::currentStep() { - if (nonpersistentEngine(m_impl->m_engineType)) + // if (nonpersistentEngine(m_impl->m_engineType)) + if (m_mode == adios2::Mode::ReadRandomAccess) { - return m_currentStep; + return 0; } else { @@ -1108,7 +1110,6 @@ void ADIOS2File::flush_impl(ADIOS2FlushParams flushParams, bool writeLatePuts) } engine.EndStep(); engine.BeginStep(); - // ++m_currentStep; // think we should keep this as the logical step m_uniquePtrPuts.clear(); uncommittedAttributes.clear(); m_updateSpans.clear(); @@ -1193,15 +1194,79 @@ AdvanceStatus ADIOS2File::advance(AdvanceMode mode) uncommittedAttributes.clear(); m_updateSpans.clear(); streamStatus = StreamStatus::OutsideOfStep; - ++m_currentStep; return AdvanceStatus::OK; } case AdvanceMode::BEGINSTEP: { adios2::StepStatus adiosStatus{}; + auto &engine = getEngine(); + + auto check_bp5 = [&]() -> bool { + std::string engineType = engine.Type(); + std::transform( + engineType.begin(), + engineType.end(), + engineType.begin(), + [](unsigned char c) { return std::tolower(c); }); + return engineType == "bp5writer"; + }; + + if (engine.CurrentStep() == 0) + { + int max_steps_from_env = + auxiliary::getEnvNum("OPENPMD_BP5_GROUPENCODING_MAX_STEPS", -1); + if (max_steps_from_env == 0) + { + m_max_steps_bp5 = std::nullopt; + } + else if (max_steps_from_env != -1) + { + m_max_steps_bp5 = + std::make_optional(size_t(max_steps_from_env)); + } + } + + // Check some conditions on which to now cancel operation due to + // unwieldy metadata sizes in BP5 with group encoding + if (this->m_impl->m_handler->m_encoding == + IterationEncoding::groupBased && + this->m_max_steps_bp5.has_value() && + engine.CurrentStep() >= *this->m_max_steps_bp5 && + (this->m_mode == adios2::Mode::Write || + this->m_mode == adios2::Mode::Append) && + check_bp5()) + { + throw error::OperationUnsupportedInBackend( + "ADIOS2", + R"( +Trying to create group-based output with more than )" + + std::to_string(*this->m_max_steps_bp5) + + R"( steps in BP5 engine. +As this engine is not adequate for group encoding, this will create immense +metadata sizes. For more context, check: + +* https://github.com/openPMD/openPMD-api/discussions/1724 +* https://github.com/openPMD/openPMD-api/issues/1457 + +Since this is likely to create unreadable data due to the sheer amount of +metadata, we will cancel the writer now. +Please consider using either of the following instead: + +* file encoding (by including an expansion pattern %T in the filename) +* another ADIOS2 engine (e.g. by selecting file extension .bp4) +* another openPMD backend (e.g. by selecting file extension .h5) +* (experimental) variable encoding (e.g. by `Series::setIterationEncoding()` + or by the JSON config {"iteration_encoding": "variable_based"}). + Note that there is at this point no complete read support for variable-encoded + outputs. + +Use the environment variable OPENPMD_BP5_GROUPENCODING_MAX_STEPS to adjust the +number of allowed steps. Set the value as 0 to disable this check. +Be aware of the performance implications described above.)"); + } if (streamStatus != StreamStatus::DuringStep) { - adiosStatus = getEngine().BeginStep(); + adiosStatus = engine.BeginStep(); } else { diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index 05cf7cd689..e7e1440855 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -866,7 +866,10 @@ namespace detail { template static void call( - adios2::IO &IO, std::string const &variable, Extent const &newShape) + adios2::IO &IO, + std::string const &variable, + Extent const &newShape, + std::optional const &newJoinedDim) { auto var = IO.InquireVariable(variable); if (!var) @@ -881,6 +884,63 @@ namespace detail { dims.push_back(ext); } + auto oldJoinedDim = joinedDimension(var.Shape()); + auto make_runtime_error = [&](char const *message) { + std::stringstream s; + s << "[ADIOS2IOHandlerImpl::extendDataset()] " << message + << "\nNote: Variable '" << variable << "' has old shape "; + auxiliary::write_vec_to_stream(s, var.Shape()); + if (oldJoinedDim.has_value()) + { + s << " (joined dimension on index " << *oldJoinedDim << ")"; + } + else + { + s << " (no joined dimension)"; + } + s << " and is extended to new shape "; + auxiliary::write_vec_to_stream(s, newShape); + if (newJoinedDim.has_value()) + { + s << " (joined dimension on index " << *newJoinedDim + << ")."; + } + else + { + s << " (no joined dimension)."; + } + return std::runtime_error(s.str()); + }; + if (oldJoinedDim.has_value() || + var.ShapeID() == adios2::ShapeID::JoinedArray) + { + if (!oldJoinedDim.has_value()) + { + throw make_runtime_error( + "Inconsistent state of variable: Has shape ID " + "JoinedArray, but its shape contains no value " + "adios2::JoinedDim."); + } + if (newJoinedDim != oldJoinedDim) + { + throw make_runtime_error( + "Variable was previously configured with a joined " + "dimension, so the new dataset extent must keep the " + "joined dimension on that index."); + } + dims[*newJoinedDim] = adios2::JoinedDim; + } + else + { + if (newJoinedDim.has_value()) + { + throw make_runtime_error( + "Variable was not previously configured with a " + "joined dimension, but is now requested to change " + "extent to a joined array."); + } + } + var.SetShape(dims); } @@ -900,7 +960,7 @@ void ADIOS2IOHandlerImpl::extendDataset( auto &filedata = getFileData(file, IfFileNotOpen::ThrowError); Datatype dt = detail::fromADIOS2Type(filedata.m_IO.VariableType(name)); switchAdios2VariableType( - dt, filedata.m_IO, name, parameters.extent); + dt, filedata.m_IO, name, parameters.extent, parameters.joinedDimension); } void ADIOS2IOHandlerImpl::openFile( diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index 7675cc3e07..e42b75488f 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -22,6 +22,7 @@ #include "openPMD/IO/AbstractIOHandlerImpl.hpp" #include "openPMD/auxiliary/Environment.hpp" +#include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -42,29 +43,6 @@ AbstractIOHandlerImpl::AbstractIOHandlerImpl(AbstractIOHandler *handler) namespace { - template - auto vec_as_string(Vec const &vec) -> std::string - { - if (vec.empty()) - { - return "[]"; - } - else - { - std::stringstream res; - res << '['; - auto it = vec.begin(); - res << *it++; - auto end = vec.end(); - for (; it != end; ++it) - { - res << ", " << *it; - } - res << ']'; - return res.str(); - } - } - template struct self_or_invoked { diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 48e24fd89b..a11b7222e9 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -836,6 +836,12 @@ void HDF5IOHandlerImpl::extendDataset( throw std::runtime_error( "[HDF5] Extending an unwritten Dataset is not possible."); + if (parameters.joinedDimension.has_value()) + { + error::throwOperationUnsupportedInBackend( + "HDF5", "Joined Arrays currently only supported in ADIOS2"); + } + auto res = getFile(writable); if (!res) res = getFile(writable->parent); diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index b1046c4602..aadae58b5c 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -360,6 +360,13 @@ void JSONIOHandlerImpl::extendDataset( VERIFY_ALWAYS( access::write(m_handler->m_backendAccess), "[JSON] Cannot extend a dataset in read-only mode.") + + if (parameters.joinedDimension.has_value()) + { + error::throwOperationUnsupportedInBackend( + "JSON", "Joined Arrays currently only supported in ADIOS2"); + } + setAndGetFilePosition(writable); refreshFileFromParent(writable); auto &j = obtainJsonContents(writable); diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index 0387268514..988dc35c41 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -307,12 +307,9 @@ void RecordComponent::flush( } else { - Parameter dCreate; + Parameter dCreate( + rc.m_dataset.value()); dCreate.name = name; - dCreate.extent = getExtent(); - dCreate.dtype = getDatatype(); - dCreate.options = rc.m_dataset.value().options; - dCreate.joinedDimension = joinedDimension(); IOHandler()->enqueue(IOTask(this, dCreate)); } } @@ -337,8 +334,8 @@ void RecordComponent::flush( } else { - Parameter pExtend; - pExtend.extent = rc.m_dataset.value().extent; + Parameter pExtend( + rc.m_dataset.value().extent); IOHandler()->enqueue(IOTask(this, std::move(pExtend))); rc.m_hasBeenExtended = false; } diff --git a/src/Series.cpp b/src/Series.cpp index 63c35e9c57..367f98c630 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -388,7 +388,8 @@ void Series::flushRankTable() { return; } - Parameter param; + Parameter param( + AbstractParameter::I_dont_want_to_use_joined_dimensions); param.name = "rankTable"; param.dtype = Datatype::CHAR; param.extent = {uint64_t(size), uint64_t(maxSize)}; diff --git a/src/backend/BaseRecordComponent.cpp b/src/backend/BaseRecordComponent.cpp index 3f0f1b35c0..2cd4551536 100644 --- a/src/backend/BaseRecordComponent.cpp +++ b/src/backend/BaseRecordComponent.cpp @@ -21,6 +21,7 @@ #include "openPMD/backend/BaseRecordComponent.hpp" #include "openPMD/Error.hpp" #include "openPMD/Iteration.hpp" +#include namespace openPMD { @@ -75,7 +76,7 @@ std::optional BaseRecordComponent::joinedDimension() const } else { - return false; + return std::nullopt; } } diff --git a/src/binding/python/Dataset.cpp b/src/binding/python/Dataset.cpp index 70d85721f2..313daee63e 100644 --- a/src/binding/python/Dataset.cpp +++ b/src/binding/python/Dataset.cpp @@ -79,7 +79,8 @@ void init_Dataset(py::module &m) }) .def_property_readonly( - "joined_dimension", &Dataset::joinedDimension) + "joined_dimension", + py::overload_cast<>(&Dataset::joinedDimension, py::const_)) .def_readonly("extent", &Dataset::extent) .def("extend", &Dataset::extend) .def_readonly("rank", &Dataset::rank) diff --git a/src/binding/python/RecordComponent.cpp b/src/binding/python/RecordComponent.cpp index 5645053f0e..55fc62fc41 100644 --- a/src/binding/python/RecordComponent.cpp +++ b/src/binding/python/RecordComponent.cpp @@ -529,7 +529,7 @@ struct PythonDynamicMemoryView , m_datatype(determineDatatype()) {} - [[nodiscard]] pybind11::memoryview currentView() const; + [[nodiscard]] pybind11::object currentView() const; std::shared_ptr m_dynamicView; ShapeContainer m_arrayShape; @@ -542,30 +542,48 @@ namespace struct GetCurrentView { template - static pybind11::memoryview call(PythonDynamicMemoryView const &dynamicView) + static pybind11::object call(PythonDynamicMemoryView const &dynamicView) { auto span = static_cast *>(dynamicView.m_dynamicView.get()) ->currentBuffer(); - return py::memoryview::from_buffer( - span.data(), - dynamicView.m_arrayShape, - dynamicView.m_strides, - /* readonly = */ false); + if (!span.data()) + { + /* + * Fallback for zero-sized store_chunk calls. + * py::memoryview cannot be used since it checks for nullpointers, + * even when the extent is zero. + * This may sound like an esoteric use case at first, but may happen + * in parallel usage when the chunk distribution ends up assigning + * zero-sized chunks to some rank. + */ + return py::array( + dtype_to_numpy(dynamicView.m_datatype), + dynamicView.m_arrayShape, + dynamicView.m_strides); + } + else + { + return py::memoryview::from_buffer( + span.data(), + dynamicView.m_arrayShape, + dynamicView.m_strides, + /* readonly = */ false); + } } static constexpr char const *errorMsg = "DynamicMemoryView"; }; template <> -pybind11::memoryview +pybind11::object GetCurrentView::call(PythonDynamicMemoryView const &) { throw std::runtime_error("[DynamicMemoryView] Only PODs allowed."); } } // namespace -pybind11::memoryview PythonDynamicMemoryView::currentView() const +pybind11::object PythonDynamicMemoryView::currentView() const { return switchNonVectorType(m_datatype, *this); } diff --git a/test/ParallelIOTest.cpp b/test/ParallelIOTest.cpp index 104db037dd..24242f89be 100644 --- a/test/ParallelIOTest.cpp +++ b/test/ParallelIOTest.cpp @@ -2003,7 +2003,29 @@ void joined_dim(std::string const &ext) patchExtent.store(10); } writeFrom.clear(); + // There seems to be a bug making this flush call necessary, need to fix + it.seriesFlush(); it.close(); + + it = s.writeIterations()[200]; + + // Test issue fixed with + // https://github.com/openPMD/openPMD-api/pull/1740 + + auto bug_dataset = it.particles["flush_multiple_times"]["position"]; + + std::vector buffer(length_of_patch * 2); + std::iota(buffer.begin(), buffer.end(), length_of_patch * 2 * rank); + + bug_dataset.resetDataset({Datatype::INT, {Dataset::JOINED_DIMENSION}}); + bug_dataset.storeChunkRaw(buffer.data(), {}, {length_of_patch}); + it.seriesFlush(); + + bug_dataset.resetDataset({Datatype::INT, {Dataset::JOINED_DIMENSION}}); + bug_dataset.storeChunkRaw( + buffer.data() + length_of_patch, {}, {length_of_patch}); + it.seriesFlush(); + s.close(); } diff --git a/test/SerialIOTest.cpp b/test/SerialIOTest.cpp index 44b6aa2624..bb304a61ff 100644 --- a/test/SerialIOTest.cpp +++ b/test/SerialIOTest.cpp @@ -741,6 +741,22 @@ TEST_CASE("close_and_copy_attributable_test", "[serial]") } } +TEST_CASE("issue_1744_unique_ptrs_at_close_time", "[serial]") +{ +#if openPMD_HAVE_ADIOS2 + openPMD::Series write( + "../samples/issue_1744_unique_ptrs_at_close_time.bp4", + openPMD::Access::CREATE, + R"({"iteration_encoding": "group_based"})"); + std::unique_ptr data_unique(new int[10]); + std::iota(data_unique.get(), data_unique.get() + 10, 0); + auto E_x = write.writeIterations()[0].meshes["E"]["x"]; + E_x.resetDataset({openPMD::Datatype::INT, {10}}); + E_x.storeChunk(std::move(data_unique), {0}, {10}); + write.close(); +#endif +} + #if openPMD_HAVE_ADIOS2 TEST_CASE("close_iteration_throws_test", "[serial]") {