diff --git a/CMakeLists.txt b/CMakeLists.txt index 32007025a6..01ad92365c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -409,9 +409,15 @@ set(CORE_SOURCE src/auxiliary/Filesystem.cpp src/auxiliary/JSON.cpp src/auxiliary/JSONMatcher.cpp + src/auxiliary/Memory.cpp src/auxiliary/Mpi.cpp + src/auxiliary/UniquePtr.cpp + src/auxiliary/Variant.cpp src/backend/Attributable.cpp + src/backend/Attribute.cpp + src/backend/BaseRecord.cpp src/backend/BaseRecordComponent.cpp + src/backend/Container.cpp src/backend/MeshRecordComponent.cpp src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp diff --git a/include/openPMD/Datatype.hpp b/include/openPMD/Datatype.hpp index 24f77218a0..26b31c8458 100644 --- a/include/openPMD/Datatype.hpp +++ b/include/openPMD/Datatype.hpp @@ -99,33 +99,6 @@ enum class Datatype : int */ std::vector openPMD_Datatypes(); -namespace detail -{ - struct bottom - {}; - - // std::variant, but ignore first template parameter - // little trick to avoid trailing commas in the macro expansions below - template - using variant_tail_t = std::variant; -} // namespace detail - -#define OPENPMD_ENUMERATE_TYPES(type) , type - -using dataset_types = - detail::variant_tail_t; - -using non_vector_types = - detail::variant_tail_t; - -using attribute_types = - detail::variant_tail_t; - -#undef OPENPMD_ENUMERATE_TYPES - /** @brief Fundamental equivalence check for two given types T and U. * * This checks whether the fundamental datatype (i.e. that of a single value @@ -451,7 +424,7 @@ inline size_t toBits(Datatype d) * @param d Datatype to test * @return true if vector type, else false */ -inline bool isVector(Datatype d) +constexpr inline bool isVector(Datatype d) { using DT = Datatype; @@ -777,6 +750,12 @@ void warnWrongDtype(std::string const &key, Datatype store, Datatype request); std::ostream &operator<<(std::ostream &, openPMD::Datatype const &); +template +constexpr auto datatypeIndex() -> size_t +{ + return static_cast(static_cast(determineDatatype())); +} + /** * Generalizes switching over an openPMD datatype. * diff --git a/include/openPMD/DatatypeMacros.hpp b/include/openPMD/DatatypeMacros.hpp index d72e86408f..e2c9733429 100644 --- a/include/openPMD/DatatypeMacros.hpp +++ b/include/openPMD/DatatypeMacros.hpp @@ -95,6 +95,27 @@ using openpmd_array_double_7 = std::array; MACRO(std::string) \ MACRO(bool) +#define OPENPMD_FOREACH_VECTOR_DATATYPE(MACRO) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(std::vector>) \ + MACRO(std::vector>) \ + MACRO(std::vector>) \ + MACRO(std::vector) \ + MACRO(std::vector) \ + MACRO(openpmd_array_double_7) + #define OPENPMD_FOREACH_DATASET_DATATYPE(MACRO) \ MACRO(char) \ MACRO(unsigned char) \ diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp index f2280433fd..6c3d499779 100644 --- a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp +++ b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp @@ -36,6 +36,7 @@ #include "openPMD/ThrowError.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/auxiliary/StringManip.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" #include @@ -659,7 +660,7 @@ namespace detail size_t step, adios2::IO &IO, std::string name, - Attribute::resource &resource, + Parameter &, detail::AdiosAttributes const &); template diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 67a6be50ab..4c82cee174 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -26,11 +26,11 @@ #include "openPMD/Streaming.hpp" #include "openPMD/auxiliary/Export.hpp" #include "openPMD/auxiliary/Memory.hpp" -#include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" #include "openPMD/backend/ParsePreference.hpp" +#include #include #include #include @@ -621,7 +621,12 @@ struct OPENPMDAPI_EXPORT IfPossible }; ChangesOverSteps changesOverSteps = ChangesOverSteps::No; - Attribute::resource resource; + // attribute_types + std::any m_resource; + template + void setResource(T val); + template + variant_t const &resource() const; }; template <> @@ -642,8 +647,13 @@ struct OPENPMDAPI_EXPORT std::string name = ""; std::shared_ptr dtype = std::make_shared(); - std::shared_ptr resource = - std::make_shared(); + + // attribute_types + std::shared_ptr m_resource = std::make_shared(); + template + variant_t const &resource() const; + template + void setResource(T val); }; template <> @@ -665,17 +675,14 @@ struct OPENPMDAPI_EXPORT std::string name = ""; std::shared_ptr dtype = std::make_shared(); - struct to_vector_type - { - template - using type = std::vector; - }; - // std::variant, std::vector, ...> - // for all T_i in openPMD::Datatype. - using result_type = typename auxiliary::detail:: - map_variant::type; - - std::shared_ptr resource = std::make_shared(); + // vector_of_attributes_type + std::shared_ptr m_resource = std::make_shared(); + template + variant_t const &resource() const; + template + variant_t &resource(); + template + void setResource(std::vector val); }; template <> diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 3691055985..bfff1e3ae8 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -27,6 +27,7 @@ #include "openPMD/IO/JSON/JSONFilePosition.hpp" #include "openPMD/auxiliary/Filesystem.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/config.hpp" #include @@ -473,7 +474,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl struct AttributeWriter { template - static void call(nlohmann::json &, Attribute::resource const &); + static void call(nlohmann::json &, attribute_types const &); static constexpr char const *errorMsg = "JSON: writeAttribute"; }; diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index ee29a6d7fa..c3c5a3dbc0 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -28,6 +28,9 @@ #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +// comment to prevent this include from being moved by clang-format +#include "openPMD/DatatypeMacros.hpp" + #include #include #include @@ -38,6 +41,7 @@ #include #include #include +#include #include // expose private and protected members for invasive testing @@ -229,8 +233,11 @@ class RecordComponent : public BaseRecordComponent template std::shared_ptr loadChunk(Offset = {0u}, Extent = {-1u}); - using shared_ptr_dataset_types = auxiliary::detail:: - map_variant::type; +#define OPENPMD_ENUMERATE_TYPES(type) , std::shared_ptr + using shared_ptr_dataset_types = auxiliary::detail::variant_tail_t< + auxiliary::detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE( + OPENPMD_ENUMERATE_TYPES)>; +#undef OPENPMD_ENUMERATE_TYPES /** std::variant-based version of allocating loadChunk(Offset, Extent) * @@ -546,4 +553,6 @@ OPENPMD_protected } // namespace openPMD +#include "openPMD/UndefDatatypeMacros.hpp" +// comment to prevent these includes from being moved by clang-format #include "RecordComponent.tpp" diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index 18fe2d98b1..923e3e2e09 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -35,242 +35,6 @@ namespace openPMD { -template -inline RecordComponent &RecordComponent::makeConstant(T value) -{ - if (written()) - throw std::runtime_error( - "A recordComponent can not (yet) be made constant after it has " - "been written."); - - auto &rc = get(); - - rc.m_constantValue = Attribute(value); - rc.m_isConstant = true; - return *this; -} - -template -inline RecordComponent &RecordComponent::makeEmpty(uint8_t dimensions) -{ - return makeEmpty(Dataset(determineDatatype(), Extent(dimensions, 0))); -} - -template -inline std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) -{ - uint8_t dim = getDimensionality(); - - // default arguments - // offset = {0u}: expand to right dim {0u, 0u, ...} - Offset offset = o; - if (o.size() == 1u && o.at(0) == 0u && dim > 1u) - offset = Offset(dim, 0u); - - // extent = {-1u}: take full size - Extent extent(dim, 1u); - if (e.size() == 1u && e.at(0) == -1u) - { - extent = getExtent(); - for (uint8_t i = 0u; i < dim; ++i) - extent[i] -= offset[i]; - } - else - extent = e; - - uint64_t numPoints = 1u; - for (auto const &dimensionSize : extent) - numPoints *= dimensionSize; - -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 11000) || \ - (defined(__apple_build_version__) && __clang_major__ < 14) - auto newData = - std::shared_ptr(new T[numPoints], [](T *p) { delete[] p; }); - loadChunk(newData, offset, extent); - return newData; -#else - auto newData = std::shared_ptr(new T[numPoints]); - loadChunk(newData, offset, extent); - return std::static_pointer_cast(std::move(newData)); -#endif -} - -namespace detail -{ - template - struct do_convert - { - template - static std::optional call(Attribute &attr) - { - if constexpr (std::is_convertible_v) - { - return std::make_optional(attr.get()); - } - else - { - return std::nullopt; - } - } - - static constexpr char const *errorMsg = "is_conversible"; - }; -} // namespace detail - -template -inline void -RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) -{ - Datatype dtype = determineDatatype(data); - /* - * For constant components, we implement type conversion, so there is - * a separate check further below. - * This is especially useful for the short-attribute representation in the - * JSON/TOML backends as they might implicitly turn a LONG into an INT in a - * constant component. The frontend needs to catch such edge cases. - * Ref. `if (constant())` branch. - */ - if (dtype != getDatatype() && !constant()) - if (!isSameInteger(getDatatype()) && - !isSameFloatingPoint(getDatatype()) && - !isSameComplexFloatingPoint(getDatatype()) && - !isSameChar(getDatatype())) - { - std::string const data_type_str = datatypeToString(getDatatype()); - std::string const requ_type_str = - datatypeToString(determineDatatype()); - std::string err_msg = - "Type conversion during chunk loading not yet implemented! "; - err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; - throw std::runtime_error(err_msg); - } - - uint8_t dim = getDimensionality(); - - // default arguments - // offset = {0u}: expand to right dim {0u, 0u, ...} - Offset offset = o; - if (o.size() == 1u && o.at(0) == 0u && dim > 1u) - offset = Offset(dim, 0u); - - // extent = {-1u}: take full size - Extent extent(dim, 1u); - if (e.size() == 1u && e.at(0) == -1u) - { - extent = getExtent(); - for (uint8_t i = 0u; i < dim; ++i) - extent[i] -= offset[i]; - } - else - extent = e; - - if (extent.size() != dim || offset.size() != dim) - { - std::ostringstream oss; - oss << "Dimensionality of chunk (" - << "offset=" << offset.size() << "D, " - << "extent=" << extent.size() << "D) " - << "and record component (" << int(dim) << "D) " - << "do not match."; - throw std::runtime_error(oss.str()); - } - Extent dse = getExtent(); - for (uint8_t i = 0; i < dim; ++i) - if (dse[i] < offset[i] + extent[i]) - throw std::runtime_error( - "Chunk does not reside inside dataset (Dimension on index " + - std::to_string(i) + ". DS: " + std::to_string(dse[i]) + - " - Chunk: " + std::to_string(offset[i] + extent[i]) + ")"); - if (!data) - throw std::runtime_error( - "Unallocated pointer passed during chunk loading."); - - auto &rc = get(); - if (constant()) - { - uint64_t numPoints = 1u; - for (auto const &dimensionSize : extent) - numPoints *= dimensionSize; - - std::optional val = - switchNonVectorType>( - /* from = */ getDatatype(), rc.m_constantValue); - - if (val.has_value()) - { - T *raw_ptr = data.get(); - std::fill(raw_ptr, raw_ptr + numPoints, *val); - } - else - { - std::string const data_type_str = datatypeToString(getDatatype()); - std::string const requ_type_str = - datatypeToString(determineDatatype()); - std::string err_msg = - "Type conversion during chunk loading not possible! "; - err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; - throw error::WrongAPIUsage(err_msg); - } - } - else - { - Parameter dRead; - dRead.offset = offset; - dRead.extent = extent; - dRead.dtype = getDatatype(); - dRead.data = std::static_pointer_cast(data); - rc.push_chunk(IOTask(this, dRead)); - } -} - -template -inline void RecordComponent::loadChunk( - std::shared_ptr ptr, Offset offset, Extent extent) -{ - loadChunk( - std::static_pointer_cast(std::move(ptr)), - std::move(offset), - std::move(extent)); -} - -template -inline void RecordComponent::loadChunkRaw(T *ptr, Offset offset, Extent extent) -{ - loadChunk(auxiliary::shareRaw(ptr), std::move(offset), std::move(extent)); -} - -template -inline void -RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) -{ - if (!data) - throw std::runtime_error( - "Unallocated pointer passed during chunk store."); - Datatype dtype = determineDatatype(data); - - /* std::static_pointer_cast correctly reference-counts the pointer */ - storeChunk( - auxiliary::WriteBuffer(std::static_pointer_cast(data)), - dtype, - std::move(o), - std::move(e)); -} - -template -inline void -RecordComponent::storeChunk(UniquePtrWithLambda data, Offset o, Extent e) -{ - if (!data) - throw std::runtime_error( - "Unallocated pointer passed during chunk store."); - Datatype dtype = determineDatatype<>(data); - - storeChunk( - auxiliary::WriteBuffer{std::move(data).template static_cast_()}, - dtype, - std::move(o), - std::move(e)); -} template inline void @@ -280,22 +44,6 @@ RecordComponent::storeChunk(std::unique_ptr data, Offset o, Extent e) UniquePtrWithLambda(std::move(data)), std::move(o), std::move(e)); } -template -inline void -RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) -{ - storeChunk( - std::static_pointer_cast(std::move(data)), - std::move(o), - std::move(e)); -} - -template -void RecordComponent::storeChunkRaw(T *ptr, Offset offset, Extent extent) -{ - storeChunk(auxiliary::shareRaw(ptr), std::move(offset), std::move(extent)); -} - template inline typename std::enable_if_t< auxiliary::IsContiguousContainer_v> @@ -385,20 +133,6 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) return DynamicMemoryView{std::move(getBufferView), size, *this}; } -template -inline DynamicMemoryView -RecordComponent::storeChunk(Offset offset, Extent extent) -{ - return storeChunk(std::move(offset), std::move(extent), [](size_t size) { -#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 11000) || \ - (defined(__apple_build_version__) && __clang_major__ < 14) - return std::shared_ptr{new T[size], [](auto *ptr) { delete[] ptr; }}; -#else - return std::shared_ptr< T[] >{ new T[ size ] }; -#endif - }); -} - namespace detail { template @@ -421,7 +155,7 @@ namespace detail } // namespace detail template -auto RecordComponent::visit(Args &&...args) +inline auto RecordComponent::visit(Args &&...args) -> decltype(Visitor::template call( std::declval(), std::forward(args)...)) { @@ -430,10 +164,4 @@ auto RecordComponent::visit(Args &&...args) return switchDatasetType>( getDatatype(), *this, std::forward(args)...); } - -template -void RecordComponent::verifyChunk(Offset const &o, Extent const &e) const -{ - verifyChunk(determineDatatype(), o, e); -} } // namespace openPMD diff --git a/include/openPMD/auxiliary/Memory.hpp b/include/openPMD/auxiliary/Memory.hpp index 66ebb5fd34..75199f0985 100644 --- a/include/openPMD/auxiliary/Memory.hpp +++ b/include/openPMD/auxiliary/Memory.hpp @@ -36,136 +36,11 @@ namespace openPMD { namespace auxiliary { - inline std::unique_ptr> - allocatePtr(Datatype dtype, uint64_t numPoints) - { - void *data = nullptr; - std::function del = [](void *) {}; - switch (dtype) - { - using DT = Datatype; - case DT::VEC_STRING: - data = new char *[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_LONG_DOUBLE: - case DT::LONG_DOUBLE: - data = new long double[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::ARR_DBL_7: - case DT::VEC_DOUBLE: - case DT::DOUBLE: - data = new double[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_FLOAT: - case DT::FLOAT: - data = new float[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_CLONG_DOUBLE: - case DT::CLONG_DOUBLE: - data = new std::complex[numPoints]; - del = [](void *p) { - delete[] static_cast *>(p); - }; - break; - case DT::VEC_CDOUBLE: - case DT::CDOUBLE: - data = new std::complex[numPoints]; - del = [](void *p) { - delete[] static_cast *>(p); - }; - break; - case DT::VEC_CFLOAT: - case DT::CFLOAT: - data = new std::complex[numPoints]; - del = [](void *p) { - delete[] static_cast *>(p); - }; - break; - case DT::VEC_SHORT: - case DT::SHORT: - data = new short[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_INT: - case DT::INT: - data = new int[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_LONG: - case DT::LONG: - data = new long[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_LONGLONG: - case DT::LONGLONG: - data = new long long[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_USHORT: - case DT::USHORT: - data = new unsigned short[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_UINT: - case DT::UINT: - data = new unsigned int[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_ULONG: - case DT::ULONG: - data = new unsigned long[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_ULONGLONG: - case DT::ULONGLONG: - data = new unsigned long long[numPoints]; - del = [](void *p) { - delete[] static_cast(p); - }; - break; - case DT::VEC_CHAR: - case DT::CHAR: - data = new char[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_UCHAR: - case DT::UCHAR: - data = new unsigned char[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::VEC_SCHAR: - case DT::SCHAR: - data = new signed char[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::BOOL: - data = new bool[numPoints]; - del = [](void *p) { delete[] static_cast(p); }; - break; - case DT::STRING: - /* user assigns c_str pointer */ - break; - case DT::UNDEFINED: - default: - throw std::runtime_error( - "Unknown Attribute datatype (Pointer allocation)"); - } - - return std::unique_ptr>(data, del); - } + std::unique_ptr> + allocatePtr(Datatype dtype, uint64_t numPoints); - inline std::unique_ptr> - allocatePtr(Datatype dtype, Extent const &e) - { - uint64_t numPoints = 1u; - for (auto const &dimensionSize : e) - numPoints *= dimensionSize; - return allocatePtr(dtype, numPoints); - } + std::unique_ptr> + allocatePtr(Datatype dtype, Extent const &e); /* * A buffer for the WRITE_DATASET task that can either be a std::shared_ptr @@ -177,41 +52,26 @@ namespace auxiliary variant, UniquePtrWithLambda>; EligibleTypes m_buffer; - WriteBuffer() : m_buffer(UniquePtrWithLambda()) - {} + WriteBuffer(); template explicit WriteBuffer(Args &&...args) : m_buffer(std::forward(args)...) {} - WriteBuffer(WriteBuffer &&) = default; + WriteBuffer(WriteBuffer &&) noexcept( + noexcept(EligibleTypes(std::declval()))); WriteBuffer(WriteBuffer const &) = delete; - WriteBuffer &operator=(WriteBuffer &&) = default; + WriteBuffer &operator=(WriteBuffer &&) noexcept(noexcept( + std::declval() = + std::declval())); WriteBuffer &operator=(WriteBuffer const &) = delete; - WriteBuffer const &operator=(std::shared_ptr ptr) - { - m_buffer = std::move(ptr); - return *this; - } + WriteBuffer const &operator=(std::shared_ptr ptr); - WriteBuffer const &operator=(UniquePtrWithLambda ptr) - { - m_buffer = std::move(ptr); - return *this; - } + WriteBuffer const &operator=(UniquePtrWithLambda ptr); - inline void const *get() const - { - return std::visit( - [](auto const &arg) { - // unique_ptr and shared_ptr both have the get() member, so - // we're being sneaky and don't distinguish the types here - return static_cast(arg.get()); - }, - m_buffer); - } + void const *get() const; }; } // namespace auxiliary } // namespace openPMD diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index 526746de89..faf2b29472 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -195,6 +195,14 @@ namespace detail { using type = std::variant<>; }; + + struct bottom + {}; + + // std::variant, but ignore first template parameter + // little trick to avoid trailing commas in the macro expansions below + template + using variant_tail_t = std::variant; } // namespace detail } // namespace openPMD::auxiliary diff --git a/include/openPMD/auxiliary/UniquePtr.hpp b/include/openPMD/auxiliary/UniquePtr.hpp index 6a64763b13..87f3261b45 100644 --- a/include/openPMD/auxiliary/UniquePtr.hpp +++ b/include/openPMD/auxiliary/UniquePtr.hpp @@ -1,3 +1,24 @@ +/* Copyright 2024-2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + #pragma once #include @@ -34,24 +55,9 @@ namespace auxiliary * Default constructor: Use std::default_delete. * This ensures correct destruction of arrays by using delete[]. */ - CustomDelete() - : deleter_type{[]([[maybe_unused]] T_decayed *ptr) { - if constexpr (std::is_void_v) - { - std::cerr << "[Warning] Cannot standard-delete a void-type " - "pointer. Please specify a custom destructor. " - "Will let the memory leak." - << std::endl; - } - else - { - std::default_delete{}(ptr); - } - }} - {} - - CustomDelete(deleter_type func) : deleter_type(std::move(func)) - {} + CustomDelete(); + + CustomDelete(deleter_type func); }; } // namespace auxiliary @@ -84,25 +90,40 @@ class UniquePtrWithLambda public: using T_decayed = std::remove_extent_t; - UniquePtrWithLambda() = default; + UniquePtrWithLambda(); - UniquePtrWithLambda(UniquePtrWithLambda &&) = default; - UniquePtrWithLambda &operator=(UniquePtrWithLambda &&) = default; + UniquePtrWithLambda(UniquePtrWithLambda &&) noexcept; + UniquePtrWithLambda &operator=(UniquePtrWithLambda &&) noexcept; UniquePtrWithLambda(UniquePtrWithLambda const &) = delete; UniquePtrWithLambda &operator=(UniquePtrWithLambda const &) = delete; /** * Conversion constructor from std::unique_ptr with default deleter. + * + * @tparam bare_unique_ptr Equal to std::unique_ptr. Needs to be a + * template parameter since we cannot instantiate + * std::unique_ptr. + * @tparam SFINAE Used to ensure that bare_unique_ptr actually is + * std::unique_ptr. */ - UniquePtrWithLambda(std::unique_ptr); + template < + typename bare_unique_ptr, + typename SFINAE = std::enable_if_t< + std::is_same_v>>> + UniquePtrWithLambda(bare_unique_ptr); /** * Conversion constructor from std::unique_ptr with custom deleter. * * @tparam Del Custom deleter type. + * @tparam SFINAE Used to not compete with the std::unique_ptr + * constructor (without custom deleter). */ - template + template < + typename Del, + typename SFINAE = std::enable_if_t< + !std::is_same_v, std::unique_ptr>>> UniquePtrWithLambda(std::unique_ptr); /** @@ -127,12 +148,7 @@ class UniquePtrWithLambda }; template -UniquePtrWithLambda::UniquePtrWithLambda(std::unique_ptr stdPtr) - : BasePtr{stdPtr.release()} -{} - -template -template +template UniquePtrWithLambda::UniquePtrWithLambda(std::unique_ptr ptr) : BasePtr{ptr.release(), auxiliary::CustomDelete{[&]() { if constexpr (std::is_copy_constructible_v) @@ -155,16 +171,6 @@ UniquePtrWithLambda::UniquePtrWithLambda(std::unique_ptr ptr) }()}} {} -template -UniquePtrWithLambda::UniquePtrWithLambda(T_decayed *ptr) : BasePtr{ptr} -{} - -template -UniquePtrWithLambda::UniquePtrWithLambda( - T_decayed *ptr, std::function deleter) - : BasePtr{ptr, std::move(deleter)} -{} - template template UniquePtrWithLambda UniquePtrWithLambda::static_cast_() && diff --git a/include/openPMD/auxiliary/Variant.hpp b/include/openPMD/auxiliary/Variant.hpp index 06e6606b0a..e5d82a20b1 100644 --- a/include/openPMD/auxiliary/Variant.hpp +++ b/include/openPMD/auxiliary/Variant.hpp @@ -20,6 +20,7 @@ */ #pragma once +#include #include #include #include // IWYU pragma: export @@ -35,7 +36,7 @@ namespace auxiliary * @tparam T Varaidic template argument list of datatypes to be * stored. */ - template + template class Variant { static_assert( @@ -43,35 +44,50 @@ namespace auxiliary "Datatypes to Variant must be supplied as enum."); public: - using resource = std_variant; + struct from_basic_type_tag + {}; + static constexpr from_basic_type_tag from_basic_type = + from_basic_type_tag{}; + struct from_any_tag + {}; + static constexpr from_any_tag from_any = from_any_tag{}; + template + Variant(from_basic_type_tag, U); + + Variant(from_any_tag, std::any); + /** Construct a lightweight wrapper around a generic object that * indicates the concrete datatype of the specific object stored. * * @note Gerneric objects can only generated implicitly if their * datatype is contained in T_DTYPES. - * @param r Generic object to be stored. + * @param u Generic object to be stored. */ - Variant(resource r) : dtype{static_cast(r.index())}, m_data{r} - {} + template + Variant(U u); /** Retrieve a stored specific object of known datatype with ensured * type-safety. * - * @throw std::bad_variant_access if stored object is not of type U. + * @throw std::bad_variant_access if stored object is not of type + * U. * @tparam U Type of the object to be retrieved. * @return Copy of the retrieved object of type U. */ template - U get() const - { - return std::get(m_data); - } + [[nodiscard]] U const &get() const; /** Retrieve the stored generic object. * * @return Copy of the stored generic object. */ - resource getResource() const + template + [[nodiscard]] variant_t const &getVariant() const + { + return *std::any_cast(&m_data); + } + + [[nodiscard]] std::any const &getAny() const { return m_data; } @@ -80,15 +96,12 @@ namespace auxiliary * * @return zero-based index */ - constexpr size_t index() const noexcept - { - return m_data.index(); - } + [[nodiscard]] size_t index() const; T_DTYPES dtype; private: - resource m_data; + std::any m_data; }; /* diff --git a/include/openPMD/backend/Attributable.hpp b/include/openPMD/backend/Attributable.hpp index cc337d8e0d..d34d5bb48f 100644 --- a/include/openPMD/backend/Attributable.hpp +++ b/include/openPMD/backend/Attributable.hpp @@ -27,7 +27,6 @@ #include "openPMD/backend/Writable.hpp" #include -#include #include #include #include diff --git a/include/openPMD/backend/Attribute.hpp b/include/openPMD/backend/Attribute.hpp index 90ae80582b..216528204a 100644 --- a/include/openPMD/backend/Attribute.hpp +++ b/include/openPMD/backend/Attribute.hpp @@ -52,11 +52,20 @@ namespace openPMD /** Variant datatype supporting at least all formats for attributes specified in * the openPMD standard. */ -class Attribute : public auxiliary::Variant + +#define OPENPMD_ENUMERATE_TYPES(type) , type + +class Attribute + : public auxiliary::Variant + +#undef OPENPMD_ENUMERATE_TYPES + { public: - Attribute(resource r) : Variant(std::move(r)) - {} + struct from_any_tag + {}; + static constexpr from_any_tag from_any = from_any_tag{}; /** * Compiler bug: CUDA (nvcc) releases 11.0.3 (v11.0.221), 11.1 (v11.1.105): @@ -69,13 +78,17 @@ class Attribute : public auxiliary::Variant */ #define OPENPMD_ATTRIBUTE_CONSTRUCTOR_FROM_VARIANT(TYPE) \ - Attribute(TYPE val) : Variant(resource(std::move(val))) \ + Attribute(TYPE val) : Variant(Variant::from_basic_type, std::move(val)) \ {} OPENPMD_FOREACH_DATATYPE(OPENPMD_ATTRIBUTE_CONSTRUCTOR_FROM_VARIANT) #undef OPENPMD_ATTRIBUTE_CONSTRUCTOR_FROM_VARIANT + Attribute(from_any_tag, std::any val) + : Variant(Variant::from_any, std::move(val)) + {} + /** Retrieve a stored specific Attribute and cast if convertible. * * @note This performs a static_cast and might introduce precision loss if @@ -101,6 +114,10 @@ class Attribute : public auxiliary::Variant */ template std::optional getOptional() const; + +private: + template + std::variant get_impl() const; }; namespace detail @@ -331,41 +348,6 @@ namespace detail eitherValueOrError); } } // namespace detail - -template -U Attribute::get() const -{ - auto eitherValueOrError = std::visit( - [](auto &&containedValue) -> std::variant { - using containedType = std::decay_t; - return detail::doConvert(&containedValue); - }, - Variant::getResource()); - return std::visit( - [](auto &&containedValue) -> U { - using T = std::decay_t; - if constexpr (std::is_same_v) - { - throw std::move(containedValue); - } - else - { - return std::move(containedValue); - } - }, - std::move(eitherValueOrError)); -} - -template -std::optional Attribute::getOptional() const -{ - return std::visit( - [](auto &&containedValue) -> std::optional { - using containedType = std::decay_t; - return detail::doConvertOptional(&containedValue); - }, - Variant::getResource()); -} } // namespace openPMD #include "openPMD/UndefDatatypeMacros.hpp" diff --git a/include/openPMD/backend/BaseRecord.hpp b/include/openPMD/backend/BaseRecord.hpp index 3eeca4d8d8..6fa8c35574 100644 --- a/include/openPMD/backend/BaseRecord.hpp +++ b/include/openPMD/backend/BaseRecord.hpp @@ -1,4 +1,4 @@ -/* Copyright 2017-2021 Fabian Koller +/* Copyright 2017-2025 Fabian Koller, Franz Poeschel * * This file is part of openPMD-api. * @@ -103,22 +103,10 @@ namespace internal ScalarTuple m_scalarTuple; std::variant m_iterator; - explicit ScalarIterator() = default; + explicit ScalarIterator(); - ScalarIterator(T_BaseRecord *baseRecord) - : m_baseRecordData(&baseRecord->get()) - , m_scalarTuple( - std::make_pair( - RecordComponent::SCALAR, T_RecordComponent(*baseRecord))) - , m_iterator(Right()) - {} - ScalarIterator(T_BaseRecord *baseRecord, Left iterator) - : m_baseRecordData(&baseRecord->get()) - , m_scalarTuple( - std::make_pair( - RecordComponent::SCALAR, T_RecordComponent(*baseRecord))) - , m_iterator(std::move(iterator)) - {} + ScalarIterator(T_BaseRecord *baseRecord, bool is_end); + ScalarIterator(T_BaseRecord *baseRecord, Left iterator); public: /** @@ -165,49 +153,15 @@ namespace internal other.m_iterator)) {} - ScalarIterator &operator++() - { - std::visit( - auxiliary::overloaded{ - [](Left &left) { ++left; }, - [this](Right &) { - m_iterator = m_baseRecordData->m_container.end(); - }}, - m_iterator); - return *this; - } + ScalarIterator &operator++(); - T_Value *operator->() - { - return std::visit( - auxiliary::overloaded{ - [](Left &left) -> T_Value * { return left.operator->(); }, - [this](Right &) -> T_Value * { - /* - * We cannot create this value on the fly since we only - * give out a pointer, so that would be use-after-free. - * Instead, we just keep one value around inside - * BaseRecordData and give it out when needed. - */ - return &m_scalarTuple.value(); - }}, - m_iterator); - } + T_Value *operator->(); - T_Value &operator*() - { - return *operator->(); - } + T_Value &operator*(); - bool operator==(ScalarIterator const &other) const - { - return this->m_iterator == other.m_iterator; - } + bool operator==(ScalarIterator const &other) const; - bool operator!=(ScalarIterator const &other) const - { - return !operator==(other); - } + bool operator!=(ScalarIterator const &other) const; }; } // namespace internal @@ -329,109 +283,20 @@ class BaseRecord } public: - iterator begin() - { - if (get().m_datasetDefined) - { - return makeIterator(); - } - else - { - return makeIterator(T_Container::begin()); - } - } - - const_iterator begin() const - { - if (get().m_datasetDefined) - { - return makeIterator(); - } - else - { - return makeIterator(T_Container::begin()); - } - } - - const_iterator cbegin() const - { - if (get().m_datasetDefined) - { - return makeIterator(); - } - else - { - return makeIterator(T_Container::cbegin()); - } - } - - iterator end() - { - return makeIterator(T_Container::end()); - } - - const_iterator end() const - { - return makeIterator(T_Container::end()); - } - - const_iterator cend() const - { - return makeIterator(T_Container::cend()); - } - - reverse_iterator rbegin() - { - if (get().m_datasetDefined) - { - return makeReverseIterator(); - } - else - { - return makeReverseIterator(this->container().rbegin()); - } - } - - const_reverse_iterator rbegin() const - { - if (get().m_datasetDefined) - { - return makeReverseIterator(); - } - else - { - return makeReverseIterator(this->container().rbegin()); - } - } - - const_reverse_iterator crbegin() const - { - if (get().m_datasetDefined) - { - return makeReverseIterator(); - } - else - { - return makeReverseIterator(this->container().crbegin()); - } - } - - reverse_iterator rend() - { - return makeReverseIterator(this->container().rend()); - } - - const_reverse_iterator rend() const - { - return makeReverseIterator(this->container().rend()); - } - - const_reverse_iterator crend() const - { - return makeReverseIterator(this->container().crend()); - } - - virtual ~BaseRecord() = default; + iterator begin(); + const_iterator begin() const; + const_iterator cbegin() const; + iterator end(); + const_iterator end() const; + const_iterator cend() const; + reverse_iterator rbegin(); + const_reverse_iterator rbegin() const; + const_reverse_iterator crbegin() const; + reverse_iterator rend(); + const_reverse_iterator rend() const; + const_reverse_iterator crend() const; + + virtual ~BaseRecord(); mapped_type &operator[](key_type const &key); mapped_type &operator[](key_type &&key); @@ -508,268 +373,6 @@ class BaseRecord void eraseScalar(); }; // BaseRecord -// implementation - -namespace internal -{ - template - BaseRecordData::BaseRecordData() - { - Attributable impl; - impl.setData({this, [](auto const *) {}}); - impl.setAttribute( - "unitDimension", - std::array{{0., 0., 0., 0., 0., 0., 0.}}); - } -} // namespace internal - -template -BaseRecord::BaseRecord() - : T_Container(Attributable::NoInit()) - , T_RecordComponent(Attributable::NoInit()) -{ - setData(std::make_shared()); -} - -template -auto BaseRecord::operator[](key_type const &key) -> mapped_type & -{ - auto it = this->find(key); - if (it != this->end()) - { - return std::visit( - auxiliary::overloaded{ - [](typename iterator::Left &l) -> mapped_type & { - return l->second; - }, - [this](typename iterator::Right &) -> mapped_type & { - /* - * Do not use the iterator result, as it is a non-owning - * handle - */ - return static_cast(*this); - }}, - it.m_iterator); - } - else - { - bool const keyScalar = (key == RecordComponent::SCALAR); - if ((keyScalar && !Container::empty() && !scalar()) || - (scalar() && !keyScalar)) - throw error::WrongAPIUsage( - "A scalar component can not be contained at the same time as " - "one or more regular components."); - - if (keyScalar) - { - /* - * This activates the RecordComponent API of this object. - */ - T_RecordComponent::get(); - } - mapped_type &ret = keyScalar ? static_cast(*this) - : T_Container:: - operator[](key); - return ret; - } -} - -template -auto BaseRecord::operator[](key_type &&key) -> mapped_type & -{ - auto it = this->find(key); - if (it != this->end()) - { - return std::visit( - auxiliary::overloaded{ - [](typename iterator::Left &l) -> mapped_type & { - return l->second; - }, - [this](typename iterator::Right &) -> mapped_type & { - /* - * Do not use the iterator result, as it is a non-owning - * handle - */ - return static_cast(*this); - }}, - it.m_iterator); - } - else - { - bool const keyScalar = (key == RecordComponent::SCALAR); - if ((keyScalar && !Container::empty() && !scalar()) || - (scalar() && !keyScalar)) - throw error::WrongAPIUsage( - "A scalar component can not be contained at the same time as " - "one or more regular components."); - - if (keyScalar) - { - /* - * This activates the RecordComponent API of this object. - */ - T_RecordComponent::get(); - } - mapped_type &ret = keyScalar ? static_cast(*this) - : T_Container:: - operator[](std::move(key)); - return ret; - } -} - -template -auto BaseRecord::at(key_type const &key) -> mapped_type & -{ - return const_cast( - static_cast const *>(this)->at(key)); -} - -template -auto BaseRecord::at(key_type const &key) const -> mapped_type const & -{ - bool const keyScalar = (key == RecordComponent::SCALAR); - if (keyScalar) - { - if (!get().m_datasetDefined) - { - throw std::out_of_range( - "[at()] Requested scalar entry from non-scalar record."); - } - return static_cast(*this); - } - else - { - return T_Container::at(key); - } -} - -template -auto BaseRecord::erase(key_type const &key) -> size_type -{ - bool const keyScalar = (key == RecordComponent::SCALAR); - size_type res; - if (!keyScalar || (keyScalar && this->at(key).constant())) - res = Container::erase(key); - else - { - res = this->datasetDefined() ? 1 : 0; - eraseScalar(); - } - - if (keyScalar) - { - this->setWritten(false, Attributable::EnqueueAsynchronously::No); - this->writable().abstractFilePosition.reset(); - this->get().m_datasetDefined = false; - } - return res; -} - -template -auto BaseRecord::erase(iterator it) -> iterator -{ - return std::visit( - auxiliary::overloaded{ - [this](typename iterator::Left &left) { - return makeIterator(T_Container::erase(left)); - }, - [this](typename iterator::Right &) { - eraseScalar(); - return end(); - }}, - it.m_iterator); -} - -template -auto BaseRecord::empty() const noexcept -> bool -{ - return !scalar() && T_Container::empty(); -} - -template -auto BaseRecord::find(key_type const &key) -> iterator -{ - auto &r = get(); - if (key == RecordComponent::SCALAR && get().m_datasetDefined) - { - if (r.m_datasetDefined) - { - return begin(); - } - else - { - return end(); - } - } - else - { - return makeIterator(r.m_container.find(key)); - } -} - -template -auto BaseRecord::find(key_type const &key) const -> const_iterator -{ - auto &r = get(); - if (key == RecordComponent::SCALAR && get().m_datasetDefined) - { - if (r.m_datasetDefined) - { - return begin(); - } - else - { - return end(); - } - } - else - { - return makeIterator(r.m_container.find(key)); - } -} - -template -auto BaseRecord::count(key_type const &key) const -> size_type -{ - if (key == RecordComponent::SCALAR) - { - return get().m_datasetDefined ? 1 : 0; - } - else - { - return T_Container::count(key); - } -} - -template -auto BaseRecord::size() const -> size_type -{ - if (scalar()) - { - return 1; - } - else - { - return T_Container::size(); - } -} - -template -auto BaseRecord::clear() -> void -{ - if (Access::READ_ONLY == this->IOHandler()->m_frontendAccess) - throw std::runtime_error( - "Can not clear a container in a read-only Series."); - if (scalar()) - { - eraseScalar(); - } - else - { - T_Container::clear_unchecked(); - } -} - namespace detail { constexpr char const *const NO_SCALAR_INSERT = @@ -777,148 +380,9 @@ namespace detail "records. Use the Record directly as a RecordComponent."; template - void verifyNonscalar(BaseRecord *self) - { - if (self->scalar()) - { - throw error::WrongAPIUsage(NO_SCALAR_INSERT); - } - } + void verifyNonscalar(BaseRecord *self); } // namespace detail -template -auto BaseRecord::insert(value_type const &value) - -> std::pair -{ - detail::verifyNonscalar(this); - auto res = this->container().insert(value); - if (res.first->first == RecordComponent::SCALAR) - { - this->container().erase(res.first); - throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - } - return {makeIterator(std::move(res.first)), res.second}; -} - -template -auto BaseRecord::insert(value_type &&value) -> std::pair -{ - detail::verifyNonscalar(this); - auto res = this->container().insert(std::move(value)); - if (res.first->first == RecordComponent::SCALAR) - { - this->container().erase(res.first); - throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - } - return {makeIterator(std::move(res.first)), res.second}; -} - -template -auto BaseRecord::insert(const_iterator hint, value_type const &value) - -> iterator -{ - detail::verifyNonscalar(this); - auto base_hint = std::visit( - auxiliary::overloaded{ - [](typename const_iterator::Left left) { return left; }, - [this](typename const_iterator::Right) { - return static_cast const *>(this) - ->container() - .begin(); - }}, - hint.m_iterator); - auto res = this->container().insert(base_hint, value); - if (res->first == RecordComponent::SCALAR) - { - this->container().erase(res); - throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - } - return makeIterator(res); -} - -template -auto BaseRecord::insert(const_iterator hint, value_type &&value) - -> iterator -{ - detail::verifyNonscalar(this); - auto base_hint = std::visit( - auxiliary::overloaded{ - [](typename const_iterator::Left left) { return left; }, - [this](typename const_iterator::Right) { - return static_cast const *>(this) - ->container() - .begin(); - }}, - hint.m_iterator); - auto res = this->container().insert(base_hint, std::move(value)); - if (res->first == RecordComponent::SCALAR) - { - this->container().erase(res); - throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - } - return makeIterator(res); -} - -template -template -auto BaseRecord::insert(InputIt first, InputIt last) -> void -{ - detail::verifyNonscalar(this); - this->container().insert(first, last); - /* - * We skip this check as it changes the runtime of this call from - * O(last-first) to O(container().size()). - */ - // for (auto it = this->container().begin(); it != end; ++it) - // { - // if (it->first == RecordComponent::SCALAR) - // { - // this->container().erase(it); - // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - // } - // } -} - -template -auto BaseRecord::insert(std::initializer_list ilist) -> void -{ - detail::verifyNonscalar(this); - this->container().insert(std::move(ilist)); - /* - * We skip this check as it changes the runtime of this call from - * O(last-first) to O(container().size()). - */ - // for (auto it = this->container().begin(); it != end; ++it) - // { - // if (it->first == RecordComponent::SCALAR) - // { - // this->container().erase(it); - // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); - // } - // } -} - -template -auto BaseRecord::swap(BaseRecord &other) -> void -{ - detail::verifyNonscalar(this); - detail::verifyNonscalar(&other); - this->container().swap(other.container()); -} - -template -auto BaseRecord::contains(key_type const &key) const -> bool -{ - if (scalar()) - { - return key == RecordComponent::SCALAR; - } - else - { - return T_Container::contains(key); - } -} - template template auto BaseRecord::emplace(Args &&...args) -> std::pair @@ -933,96 +397,4 @@ auto BaseRecord::emplace(Args &&...args) -> std::pair return {makeIterator(std::move(res.first)), res.second}; } -template -inline unit_representations::AsArray BaseRecord::unitDimension() const -{ - return this->getAttribute("unitDimension") - .template get>(); -} - -template -inline bool BaseRecord::scalar() const -{ - return this->datasetDefined(); -} - -template -inline void BaseRecord::readBase() -{ - using DT = Datatype; - Parameter aRead; - - aRead.name = "unitDimension"; - this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(internal::defaultFlushParams); - if (auto val = - Attribute(*aRead.resource).getOptional>(); - val.has_value()) - this->setAttribute("unitDimension", val.value()); - else - throw std::runtime_error( - "Unexpected Attribute datatype for 'unitDimension'"); - - aRead.name = "timeOffset"; - this->IOHandler()->enqueue(IOTask(this, aRead)); - this->IOHandler()->flush(internal::defaultFlushParams); - if (*aRead.dtype == DT::FLOAT) - this->setAttribute( - "timeOffset", Attribute(*aRead.resource).get()); - else if (*aRead.dtype == DT::DOUBLE) - this->setAttribute( - "timeOffset", Attribute(*aRead.resource).get()); - // conversion cast if a backend reports an integer type - else if (auto val = Attribute(*aRead.resource).getOptional(); - val.has_value()) - this->setAttribute("timeOffset", val.value()); - else - throw std::runtime_error( - "Unexpected Attribute datatype for 'timeOffset'"); -} - -template -inline void BaseRecord::flush( - std::string const &name, internal::FlushParams const &flushParams) -{ - if (!this->written() && this->empty() && !this->datasetDefined()) - throw std::runtime_error( - "A Record can not be written without any contained " - "RecordComponents: " + - name); - - /* - * Defensive programming. Normally, this error should yield as soon as - * possible. - */ - if (scalar() && !T_Container::empty()) - { - throw error::WrongAPIUsage( - "A scalar component can not be contained at the same time as " - "one or more regular components."); - } - - this->flush_impl(name, flushParams); - if (flushParams.flushLevel != FlushLevel::SkeletonOnly) - { - this->setDirty(false); - } - // flush_impl must take care to correctly set the dirty() flag so this - // method doesn't do it -} - -template -void BaseRecord::eraseScalar() -{ - if (this->written()) - { - Parameter dDelete; - dDelete.name = "."; - this->IOHandler()->enqueue(IOTask(this, dDelete)); - this->IOHandler()->flush(internal::defaultFlushParams); - } - auto &data = T_RecordComponent::get(); - data.reset(); - this->writable().abstractFilePosition.reset(); -} } // namespace openPMD diff --git a/include/openPMD/backend/Container.hpp b/include/openPMD/backend/Container.hpp index 8dda5b992b..afe0b25cc6 100644 --- a/include/openPMD/backend/Container.hpp +++ b/include/openPMD/backend/Container.hpp @@ -1,4 +1,4 @@ -/* Copyright 2017-2021 Fabian Koller and Franz Poeschel +/* Copyright 2017-2025 Fabian Koller and Franz Poeschel * * This file is part of openPMD-api. * @@ -66,7 +66,7 @@ namespace internal template < typename T, typename T_key = std::string, - typename T_container = std::map > + typename T_container = std::map> class ContainerData : virtual public AttributableData { public: @@ -99,7 +99,7 @@ namespace internal template < typename T, typename T_key = std::string, - typename T_container = std::map > + typename T_container = std::map> class Container : virtual public Attributable { static_assert( @@ -154,67 +154,25 @@ class Container : virtual public Attributable using const_reverse_iterator = typename InternalContainer::const_reverse_iterator; - iterator begin() noexcept - { - return container().begin(); - } - const_iterator begin() const noexcept - { - return container().begin(); - } - const_iterator cbegin() const noexcept - { - return container().cbegin(); - } + iterator begin() noexcept; + const_iterator begin() const noexcept; + const_iterator cbegin() const noexcept; - iterator end() noexcept - { - return container().end(); - } - const_iterator end() const noexcept - { - return container().end(); - } - const_iterator cend() const noexcept - { - return container().cend(); - } + iterator end() noexcept; + const_iterator end() const noexcept; + const_iterator cend() const noexcept; - reverse_iterator rbegin() noexcept - { - return container().rbegin(); - } - const_reverse_iterator rbegin() const noexcept - { - return container().rbegin(); - } - const_reverse_iterator crbegin() const noexcept - { - return container().crbegin(); - } + reverse_iterator rbegin() noexcept; + const_reverse_iterator rbegin() const noexcept; + const_reverse_iterator crbegin() const noexcept; - reverse_iterator rend() noexcept - { - return container().rend(); - } - const_reverse_iterator rend() const noexcept - { - return container().rend(); - } - const_reverse_iterator crend() const noexcept - { - return container().crend(); - } + reverse_iterator rend() noexcept; + const_reverse_iterator rend() const noexcept; + const_reverse_iterator crend() const noexcept; - bool empty() const noexcept - { - return container().empty(); - } + bool empty() const noexcept; - size_type size() const noexcept - { - return container().size(); - } + size_type size() const noexcept; /** Remove all objects from the container and (if written) from disk. * @@ -222,54 +180,23 @@ class Container : virtual public Attributable * Access::READ_ONLY will throw an exception. * @throws std::runtime_error */ - void clear() - { - if (Access::READ_ONLY == IOHandler()->m_frontendAccess) - throw std::runtime_error( - "Can not clear a container in a read-only Series."); - - clear_unchecked(); - } + void clear(); - std::pair insert(value_type const &value) - { - return container().insert(value); - } - std::pair insert(value_type &&value) - { - return container().insert(value); - } - iterator insert(const_iterator hint, value_type const &value) - { - return container().insert(hint, value); - } - iterator insert(const_iterator hint, value_type &&value) - { - return container().insert(hint, value); - } + std::pair insert(value_type const &value); + std::pair insert(value_type &&value); + iterator insert(const_iterator hint, value_type const &value); + iterator insert(const_iterator hint, value_type &&value); template void insert(InputIt first, InputIt last) { container().insert(first, last); } - void insert(std::initializer_list ilist) - { - container().insert(ilist); - } + void insert(std::initializer_list ilist); - void swap(Container &other) - { - container().swap(other.container()); - } + void swap(Container &other); - mapped_type &at(key_type const &key) - { - return container().at(key); - } - mapped_type const &at(key_type const &key) const - { - return container().at(key); - } + mapped_type &at(key_type const &key); + mapped_type const &at(key_type const &key) const; /** Access the value that is mapped to a key equivalent to key, creating it * if such key does not exist already. @@ -281,37 +208,7 @@ class Container : virtual public Attributable * @throws std::out_of_range error if in READ_ONLY mode and key does not * exist, otherwise key will be created */ - mapped_type &operator[](key_type const &key) - { - auto it = container().find(key); - if (it != container().end()) - return it->second; - else - { - if (IOHandler()->m_seriesStatus != - internal::SeriesStatus::Parsing && - access::readOnly(IOHandler()->m_frontendAccess)) - { - auxiliary::OutOfRangeMsg const out_of_range_msg; - throw std::out_of_range(out_of_range_msg(key)); - } - - T t = T(); - t.linkHierarchy(writable()); - auto &ret = container().insert({key, std::move(t)}).first->second; - if constexpr (std::is_same_v) - { - ret.writable().ownKeyWithinParent = key; - } - else - { - ret.writable().ownKeyWithinParent = std::to_string(key); - } - traits::GenerationPolicy gen; - gen(ret); - return ret; - } - } + mapped_type &operator[](key_type const &key); /** Access the value that is mapped to a key equivalent to key, creating it * if such key does not exist already. * @@ -322,57 +219,17 @@ class Container : virtual public Attributable * @throws std::out_of_range error if in READ_ONLY mode and key does not * exist, otherwise key will be created */ - mapped_type &operator[](key_type &&key) - { - auto it = container().find(key); - if (it != container().end()) - return it->second; - else - { - if (IOHandler()->m_seriesStatus != - internal::SeriesStatus::Parsing && - access::readOnly(IOHandler()->m_frontendAccess)) - { - auxiliary::OutOfRangeMsg out_of_range_msg; - throw std::out_of_range(out_of_range_msg(key)); - } - - T t = T(); - t.linkHierarchy(writable()); - auto &ret = container().insert({key, std::move(t)}).first->second; - if constexpr (std::is_same_v) - { - ret.writable().ownKeyWithinParent = std::move(key); - } - else - { - ret.writable().ownKeyWithinParent = - std::to_string(std::move(key)); - } - traits::GenerationPolicy gen; - gen(ret); - return ret; - } - } + mapped_type &operator[](key_type &&key); - iterator find(key_type const &key) - { - return container().find(key); - } - const_iterator find(key_type const &key) const - { - return container().find(key); - } + iterator find(key_type const &key); + const_iterator find(key_type const &key) const; /** This returns either 1 if the key is found in the container of 0 if not. * * @param key key value of the element to count * @return since keys are unique in this container, returns 0 or 1 */ - size_type count(key_type const &key) const - { - return container().count(key); - } + size_type count(key_type const &key) const; /** Checks if there is an element with a key equivalent to an exiting key in * the container. @@ -380,10 +237,7 @@ class Container : virtual public Attributable * @param key key value of the element to search for * @return true of key is found, else false */ - bool contains(key_type const &key) const - { - return container().find(key) != container().end(); - } + bool contains(key_type const &key) const; /** Remove a single element from the container and (if written) from disk. * @@ -393,39 +247,10 @@ class Container : virtual public Attributable * @param key Key of the element to remove. * @return Number of elements removed (either 0 or 1). */ - size_type erase(key_type const &key) - { - if (Access::READ_ONLY == IOHandler()->m_frontendAccess) - throw std::runtime_error( - "Can not erase from a container in a read-only Series."); - - auto res = container().find(key); - if (res != container().end() && res->second.written()) - { - Parameter pDelete; - pDelete.path = "."; - IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(internal::defaultFlushParams); - } - return container().erase(key); - } + size_type erase(key_type const &key); //! @todo why does const_iterator not work compile with pybind11? - iterator erase(iterator res) - { - if (Access::READ_ONLY == IOHandler()->m_frontendAccess) - throw std::runtime_error( - "Can not erase from a container in a read-only Series."); - - if (res != container().end() && res->second.written()) - { - Parameter pDelete; - pDelete.path = "."; - IOHandler()->enqueue(IOTask(&res->second, pDelete)); - IOHandler()->flush(internal::defaultFlushParams); - } - return container().erase(res); - } + iterator erase(iterator res); //! @todo add also: // virtual iterator erase(const_iterator first, const_iterator last) @@ -440,35 +265,14 @@ class Container : virtual public Attributable OPENPMD_protected // clang-format on - void clear_unchecked() - { - if (written()) - throw std::runtime_error( - "Clearing a written container not (yet) implemented."); - - container().clear(); - } + void clear_unchecked(); virtual void - flush(std::string const &path, internal::FlushParams const &flushParams) - { - if (!written()) - { - Parameter pCreate; - pCreate.path = path; - IOHandler()->enqueue(IOTask(this, pCreate)); - } - - flushAttributes(flushParams); - } + flush(std::string const &path, internal::FlushParams const &flushParams); - Container() : Attributable(NoInit()) - { - setData(std::make_shared()); - } + Container(); - Container(NoInit) : Attributable(NoInit()) - {} + Container(NoInit); public: /* @@ -483,37 +287,13 @@ OPENPMD_protected * multiple times (which could happen in diamond inheritance situations). */ - Container(Container const &other) : Attributable(NoInit()) - { - m_attri = other.m_attri; - m_containerData = other.m_containerData; - } + Container(Container const &other); - Container(Container &&other) noexcept : Attributable(NoInit()) - { - if (other.m_attri) - { - m_attri = std::move(other.m_attri); - } - m_containerData = std::move(other.m_containerData); - } + Container(Container &&other) noexcept; - Container &operator=(Container const &other) - { - m_attri = other.m_attri; - m_containerData = other.m_containerData; - return *this; - } + Container &operator=(Container const &other); - Container &operator=(Container &&other) noexcept - { - if (other.m_attri) - { - m_attri = std::move(other.m_attri); - } - m_containerData = std::move(other.m_containerData); - return *this; - } + Container &operator=(Container &&other) noexcept; }; namespace internal @@ -530,10 +310,10 @@ namespace internal template class EraseStaleEntries { - using BareContainer_t = - typename std::remove_reference::type; - using key_type = typename BareContainer_t::key_type; - using mapped_type = typename BareContainer_t::mapped_type; + static_assert( + std::is_same_v>); + using key_type = typename Container_t::key_type; + using mapped_type = typename Container_t::mapped_type; std::set m_accessedKeys; /* * Note: Putting a copy here leads to weird bugs due to destructors @@ -542,65 +322,26 @@ namespace internal * Container class template * (https://github.com/openPMD/openPMD-api/pull/886) */ - Container_t m_originalContainer; + Container_t &m_originalContainer; public: - explicit EraseStaleEntries(Container_t &container_in) - : m_originalContainer(container_in) - {} + explicit EraseStaleEntries(Container_t &container_in); - explicit EraseStaleEntries(BareContainer_t &&container_in) - : m_originalContainer(std::move(container_in)) - {} + EraseStaleEntries(EraseStaleEntries &&) = delete; + EraseStaleEntries &operator=(EraseStaleEntries &&) = delete; - EraseStaleEntries(EraseStaleEntries &&) = default; - EraseStaleEntries &operator=(EraseStaleEntries &&) = default; + mapped_type &operator[](typename Container_t::key_type const &k); - template - mapped_type &operator[](K &&k) - { - m_accessedKeys.insert(k); // copy - return m_originalContainer[std::forward(k)]; - } - - template - mapped_type &at(K &&k) - { - m_accessedKeys.insert(k); // copy - return m_originalContainer.at(std::forward(k)); - } + mapped_type &at(typename Container_t::key_type const &k); /** * Remove key from the list of accessed keys. * If the key is not accessed after this again, it will be deleted along * with all other unaccessed keys upon destruction. */ - template - void forget(K &&k) - { - m_accessedKeys.erase(std::forward(k)); - } - - ~EraseStaleEntries() - { - auto &map = m_originalContainer.container(); - using iterator_t = - typename BareContainer_t::InternalContainer::const_iterator; - std::vector deleteMe; - deleteMe.reserve(map.size() - m_accessedKeys.size()); - for (iterator_t it = map.begin(); it != map.end(); ++it) - { - auto lookup = m_accessedKeys.find(it->first); - if (lookup == m_accessedKeys.end()) - { - deleteMe.push_back(it); - } - } - for (auto &it : deleteMe) - { - map.erase(it); - } - } + void forget(typename Container_t::key_type const &k); + + ~EraseStaleEntries(); }; } // namespace internal } // namespace openPMD diff --git a/include/openPMD/backend/ContainerImpl.tpp b/include/openPMD/backend/ContainerImpl.tpp new file mode 100644 index 0000000000..0384333e10 --- /dev/null +++ b/include/openPMD/backend/ContainerImpl.tpp @@ -0,0 +1,441 @@ +/* Copyright 2017-2025 Fabian Koller and Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/backend/Container.hpp" + +/* + * Instantiations in src/backend/Container.cpp + * This file exists so that our tests can include the Container class with + * custom instantiations. + */ + +namespace openPMD +{ + +template +auto Container::begin() noexcept -> iterator +{ + return container().begin(); +} +template +auto Container::begin() const noexcept -> const_iterator +{ + return container().begin(); +} +template +auto Container::cbegin() const noexcept -> const_iterator +{ + return container().cbegin(); +} + +template +auto Container::end() noexcept -> iterator +{ + return container().end(); +} +template +auto Container::end() const noexcept -> const_iterator +{ + return container().end(); +} +template +auto Container::cend() const noexcept -> const_iterator +{ + return container().cend(); +} + +template +auto Container::rbegin() noexcept -> reverse_iterator +{ + return container().rbegin(); +} +template +auto Container::rbegin() const noexcept + -> const_reverse_iterator +{ + return container().rbegin(); +} +template +auto Container::crbegin() const noexcept + -> const_reverse_iterator +{ + return container().crbegin(); +} + +template +auto Container::rend() noexcept -> reverse_iterator +{ + return container().rend(); +} +template +auto Container::rend() const noexcept + -> const_reverse_iterator +{ + return container().rend(); +} +template +auto Container::crend() const noexcept + -> const_reverse_iterator +{ + return container().crend(); +} + +template +auto Container::empty() const noexcept -> bool +{ + return container().empty(); +} + +template +auto Container::size() const noexcept -> size_type +{ + return container().size(); +} + +template +auto Container::at(key_type const &key) -> mapped_type & +{ + return container().at(key); +} +template +auto Container::at(key_type const &key) const + -> mapped_type const & +{ + return container().at(key); +} + +template +auto Container::operator[](key_type const &key) + -> mapped_type & +{ + auto it = container().find(key); + if (it != container().end()) + return it->second; + else + { + if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing && + access::readOnly(IOHandler()->m_frontendAccess)) + { + auxiliary::OutOfRangeMsg const out_of_range_msg; + throw std::out_of_range(out_of_range_msg(key)); + } + + T t = T(); + t.linkHierarchy(writable()); + auto &ret = container().insert({key, std::move(t)}).first->second; + if constexpr (std::is_same_v) + { + ret.writable().ownKeyWithinParent = key; + } + else + { + ret.writable().ownKeyWithinParent = std::to_string(key); + } + traits::GenerationPolicy gen; + gen(ret); + return ret; + } +} +template +auto Container::operator[](key_type &&key) + -> mapped_type & +{ + auto it = container().find(key); + if (it != container().end()) + return it->second; + else + { + if (IOHandler()->m_seriesStatus != internal::SeriesStatus::Parsing && + access::readOnly(IOHandler()->m_frontendAccess)) + { + auxiliary::OutOfRangeMsg out_of_range_msg; + throw std::out_of_range(out_of_range_msg(key)); + } + + T t = T(); + t.linkHierarchy(writable()); + auto &ret = container().insert({key, std::move(t)}).first->second; + if constexpr (std::is_same_v) + { + ret.writable().ownKeyWithinParent = std::move(key); + } + else + { + ret.writable().ownKeyWithinParent = std::to_string(std::move(key)); + } + traits::GenerationPolicy gen; + gen(ret); + return ret; + } +} + +template +auto Container::clear() -> void +{ + if (Access::READ_ONLY == IOHandler()->m_frontendAccess) + throw std::runtime_error( + "Can not clear a container in a read-only Series."); + + clear_unchecked(); +} + +template +auto Container::insert(value_type const &value) + -> std::pair +{ + return container().insert(value); +} +template +auto Container::insert(value_type &&value) + -> std::pair +{ + return container().insert(value); +} +template +auto Container::insert( + const_iterator hint, value_type const &value) -> iterator +{ + return container().insert(hint, value); +} +template +auto Container::insert( + const_iterator hint, value_type &&value) -> iterator +{ + return container().insert(hint, value); +} +template +auto Container::insert( + std::initializer_list ilist) -> void +{ + container().insert(ilist); +} + +template +auto Container::swap(Container &other) -> void +{ + container().swap(other.container()); +} + +template +auto Container::find(key_type const &key) -> iterator +{ + return container().find(key); +} +template +auto Container::find(key_type const &key) const + -> const_iterator +{ + return container().find(key); +} + +/** This returns either 1 if the key is found in the container of 0 if not. + * + * @param key key value of the element to count + * @return since keys are unique in this container, returns 0 or 1 + */ +template +auto Container::count(key_type const &key) const + -> size_type +{ + return container().count(key); +} + +/** Checks if there is an element with a key equivalent to an exiting key in + * the container. + * + * @param key key value of the element to search for + * @return true of key is found, else false + */ +template +auto Container::contains(key_type const &key) const + -> bool +{ + return container().find(key) != container().end(); +} + +template +auto Container::erase(key_type const &key) -> size_type +{ + if (Access::READ_ONLY == IOHandler()->m_frontendAccess) + throw std::runtime_error( + "Can not erase from a container in a read-only Series."); + + auto res = container().find(key); + if (res != container().end() && res->second.written()) + { + Parameter pDelete; + pDelete.path = "."; + IOHandler()->enqueue(IOTask(&res->second, pDelete)); + IOHandler()->flush(internal::defaultFlushParams); + } + return container().erase(key); +} + +template +auto Container::erase(iterator res) -> iterator +{ + if (Access::READ_ONLY == IOHandler()->m_frontendAccess) + throw std::runtime_error( + "Can not erase from a container in a read-only Series."); + + if (res != container().end() && res->second.written()) + { + Parameter pDelete; + pDelete.path = "."; + IOHandler()->enqueue(IOTask(&res->second, pDelete)); + IOHandler()->flush(internal::defaultFlushParams); + } + return container().erase(res); +} + +template +auto Container::clear_unchecked() -> void +{ + if (written()) + throw std::runtime_error( + "Clearing a written container not (yet) implemented."); + + container().clear(); +} + +template +auto Container::flush( + std::string const &path, internal::FlushParams const &flushParams) -> void +{ + if (!written()) + { + Parameter pCreate; + pCreate.path = path; + IOHandler()->enqueue(IOTask(this, pCreate)); + } + + flushAttributes(flushParams); +} + +template +Container::Container() : Attributable(NoInit()) +{ + setData(std::make_shared()); +} + +template +Container::Container(NoInit) : Attributable(NoInit()) +{} + +template +Container::Container(Container const &other) + : Attributable(NoInit()) +{ + m_attri = other.m_attri; + m_containerData = other.m_containerData; +} + +template +Container::Container(Container &&other) noexcept + : Attributable(NoInit()) +{ + if (other.m_attri) + { + m_attri = std::move(other.m_attri); + } + m_containerData = std::move(other.m_containerData); +} + +template +auto Container::operator=(Container const &other) + -> Container & +{ + m_attri = other.m_attri; + m_containerData = other.m_containerData; + return *this; +} + +template +auto Container::operator=(Container &&other) noexcept + -> Container & +{ + if (other.m_attri) + { + m_attri = std::move(other.m_attri); + } + m_containerData = std::move(other.m_containerData); + return *this; +} + +namespace internal +{ + template + EraseStaleEntries::EraseStaleEntries(Container_t &container_in) + : m_originalContainer(container_in) + {} + + template + auto EraseStaleEntries::operator[]( + typename Container_t::key_type const &k) -> mapped_type & + { + m_accessedKeys.insert(k); + return m_originalContainer[k]; + } + + template + auto + EraseStaleEntries::at(typename Container_t::key_type const &k) + -> mapped_type & + { + m_accessedKeys.insert(k); + return m_originalContainer.at(k); + } + + /** + * Remove key from the list of accessed keys. + * If the key is not accessed after this again, it will be deleted along + * with all other unaccessed keys upon destruction. + */ + template + auto EraseStaleEntries::forget( + typename Container_t::key_type const &k) -> void + { + m_accessedKeys.erase(k); + } + + template + EraseStaleEntries::~EraseStaleEntries() + { + auto &map = m_originalContainer.container(); + using iterator_t = + typename Container_t::InternalContainer::const_iterator; + std::vector deleteMe; + deleteMe.reserve(map.size() - m_accessedKeys.size()); + for (iterator_t it = map.begin(); it != map.end(); ++it) + { + auto lookup = m_accessedKeys.find(it->first); + if (lookup == m_accessedKeys.end()) + { + deleteMe.push_back(it); + } + } + for (auto &it : deleteMe) + { + map.erase(it); + } + } +} // namespace internal +} // namespace openPMD diff --git a/include/openPMD/backend/Variant_internal.hpp b/include/openPMD/backend/Variant_internal.hpp new file mode 100644 index 0000000000..cfe29c528b --- /dev/null +++ b/include/openPMD/backend/Variant_internal.hpp @@ -0,0 +1,41 @@ +#pragma once + +#define OPENPMD_GUARD_HEADER_AGAINST_PUBLIC_INCLUSION + +#include "openPMD/DatatypeMacros.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" + +#include + +namespace openPMD +{ +namespace detail +{ + struct to_vector_type + { + template + using type = std::vector; + }; +} // namespace detail + +#define OPENPMD_ENUMERATE_TYPES(type) , type + +using dataset_types = auxiliary::detail::variant_tail_t< + auxiliary::detail::bottom OPENPMD_FOREACH_DATASET_DATATYPE( + OPENPMD_ENUMERATE_TYPES)>; + +using non_vector_types = auxiliary::detail::variant_tail_t< + auxiliary::detail::bottom OPENPMD_FOREACH_NONVECTOR_DATATYPE( + OPENPMD_ENUMERATE_TYPES)>; + +using attribute_types = auxiliary::detail::variant_tail_t< + auxiliary::detail::bottom OPENPMD_FOREACH_DATATYPE( + OPENPMD_ENUMERATE_TYPES)>; + +// std::variant, std::vector, ...> +// for all T_i in openPMD::Datatype. +using vector_of_attributes_type = typename auxiliary::detail:: + map_variant::type; + +#undef OPENPMD_ENUMERATE_TYPES +} // namespace openPMD diff --git a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp index 7f2ef89761..b6c68da968 100644 --- a/include/openPMD/benchmark/mpi/MPIBenchmark.hpp +++ b/include/openPMD/benchmark/mpi/MPIBenchmark.hpp @@ -239,7 +239,7 @@ MPIBenchmark::runBenchmark(int rootThread) } for (Datatype dt : datatypes) { - switchType>(dt, exec, res, rootThread); + switchDatasetType>(dt, exec, res, rootThread); } return res; diff --git a/include/openPMD/openPMD.hpp b/include/openPMD/openPMD.hpp index ebe296495b..bc6ea782f7 100644 --- a/include/openPMD/openPMD.hpp +++ b/include/openPMD/openPMD.hpp @@ -80,4 +80,8 @@ If you think that this assertion is shown wrongly, please apply #include "openPMD/config.hpp" #include "openPMD/version.hpp" + +#ifdef OPENPMD_GUARD_HEADER_AGAINST_PUBLIC_INCLUSION +static_assert(false, "Some header was publically included that should not be."); +#endif // IWYU pragma: end_exports diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp index cb9f822875..027a1038ad 100644 --- a/src/IO/ADIOS/ADIOS2IOHandler.cpp +++ b/src/IO/ADIOS/ADIOS2IOHandler.cpp @@ -41,6 +41,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/TypeTraits.hpp" #include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include #include @@ -1343,12 +1344,7 @@ void ADIOS2IOHandlerImpl::readAttribute( } Datatype ret = switchType( - type, - ba.currentStep(), - ba.m_IO, - name, - *parameters.resource, - ba.attributes()); + type, ba.currentStep(), ba.m_IO, name, parameters, ba.attributes()); *parameters.dtype = ret; } @@ -1486,10 +1482,11 @@ namespace std::vector const &preload, adios2::IO &IO, std::string const &name, - Parameter::result_type - &put_result_here) + Parameter &put_result_here) { - auto &res = put_result_here.emplace>(); + put_result_here.setResource(std::vector{}); + auto &res = std::get>( + put_result_here.resource()); res.reserve(preload.size()); for (auto const &p : preload) { @@ -1657,7 +1654,7 @@ void ADIOS2IOHandlerImpl::readAttributeAllsteps( } #endif auto &attributes = ba.attributes(); - switchType(type, preload, IO, name, *param.resource); + switchType(type, preload, IO, name, param); attributes.m_data = std::move(preload); } @@ -2209,12 +2206,12 @@ namespace detail size_t step, adios2::IO &IO, std::string name, - Attribute::resource &resource, + Parameter ¶m, detail::AdiosAttributes const &attributes) { return genericReadAttribute( - [&resource](auto &&value) { - resource = static_cast(value); + [¶m](auto &&value) { + param.setResource(static_cast(value)); }, IO, name, @@ -2265,7 +2262,9 @@ namespace detail return it != filedata.uncommittedAttributes.end(); }; if (AttributeTypes::attributeUnchanged( - IO, fullName, std::get(parameters.resource))) + IO, + fullName, + std::get(parameters.resource()))) { return; } @@ -2312,7 +2311,7 @@ namespace detail } } - auto &value = std::get(parameters.resource); + auto &value = std::get(parameters.resource()); bool modifiable = impl->m_modifiableAttributes == ADIOS2IOHandlerImpl::ModifiableAttributes::Yes || parameters.changesOverSteps != diff --git a/src/IO/AbstractIOHandlerImpl.cpp b/src/IO/AbstractIOHandlerImpl.cpp index faad2f49a9..8360474e1a 100644 --- a/src/IO/AbstractIOHandlerImpl.cpp +++ b/src/IO/AbstractIOHandlerImpl.cpp @@ -26,6 +26,7 @@ #include "openPMD/auxiliary/Environment.hpp" #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -502,16 +503,16 @@ std::future AbstractIOHandlerImpl::flush() void AbstractIOHandlerImpl::readAttributeAllsteps( Writable *w, Parameter ¶m) { - using result_type = Parameter::result_type; + using result_type = vector_of_attributes_type; Parameter param_internal; param_internal.name = param.name; param_internal.dtype = param.dtype; readAttribute(w, param_internal); - *param.resource = std::visit( + param.resource() = std::visit( [](auto &val) -> result_type { return result_type{std::vector{std::move(val)}}; }, - *param_internal.resource); + param_internal.resource()); } void AbstractIOHandlerImpl::setWritten( diff --git a/src/IO/HDF5/HDF5IOHandler.cpp b/src/IO/HDF5/HDF5IOHandler.cpp index 55d99a4196..dc05bed0f1 100644 --- a/src/IO/HDF5/HDF5IOHandler.cpp +++ b/src/IO/HDF5/HDF5IOHandler.cpp @@ -1727,7 +1727,7 @@ void HDF5IOHandlerImpl::writeAttribute( node_id >= 0, "[HDF5] Internal error: Failed to open HDF5 object during attribute " "write"); - Attribute const att(parameters.resource); + Attribute const att(Attribute::from_any, parameters.m_resource); Datatype dtype = parameters.dtype; herr_t status; GetH5DataType getH5DataType({ @@ -2793,8 +2793,7 @@ void HDF5IOHandlerImpl::readAttribute( auto dtype = parameters.dtype; *dtype = a.dtype; - auto resource = parameters.resource; - *resource = a.getResource(); + *parameters.m_resource = a.getAny(); status = H5Aclose(attr_id); if (status != 0) diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp index 50fe180a40..01e99ec55f 100644 --- a/src/IO/IOTask.cpp +++ b/src/IO/IOTask.cpp @@ -19,9 +19,11 @@ * If not, see . */ #include "openPMD/IO/IOTask.hpp" +#include "openPMD/DatatypeMacros.hpp" #include "openPMD/auxiliary/JSONMatcher.hpp" #include "openPMD/auxiliary/JSON_internal.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include // std::cerr #include @@ -226,4 +228,83 @@ IOTask::IOTask(IOTask &&) noexcept = default; IOTask &IOTask::operator=(IOTask const &) = default; IOTask &IOTask::operator=(IOTask &&) noexcept = default; + +template +variant_t const &Parameter::resource() const +{ + if (!m_resource->has_value()) + { + *m_resource = std::make_any(); + } + return *std::any_cast(&*m_resource); +} +template attribute_types const & +Parameter::resource() const; + +template +void Parameter::setResource(T val) +{ + *m_resource = + std::make_any(attribute_types{std::move(val)}); +} + +template +variant_t const &Parameter::resource() const +{ + return *std::any_cast(&m_resource); +} +template attribute_types const & +Parameter::resource() const; + +template +void Parameter::setResource(T val) +{ + m_resource = + std::make_any(attribute_types{std::move(val)}); +} + +template +variant_t const &Parameter::resource() const +{ + if (!m_resource->has_value()) + { + *m_resource = std::make_any(); + } + return *std::any_cast(&*m_resource); +} +template vector_of_attributes_type const & +Parameter::resource() + const; + +template +variant_t &Parameter::resource() +{ + if (!m_resource->has_value()) + { + *m_resource = std::make_any(); + } + return *std::any_cast(&*m_resource); +} + +// ????? +#ifndef DOXYGEN_SHOULD_SKIP_THIS +template vector_of_attributes_type & +Parameter::resource(); +#endif + +template +void Parameter::setResource(std::vector val) +{ + *m_resource = std::make_any( + vector_of_attributes_type{std::move(val)}); +} + +#define OPENPMD_INSTANTIATE(type) \ + template void Parameter::setResource( \ + std::vector); \ + template void Parameter::setResource(type val); \ + template void Parameter::setResource(type val); + +OPENPMD_FOREACH_DATATYPE(OPENPMD_INSTANTIATE) +#undef OPENPMD_INSTANTIATE } // namespace openPMD diff --git a/src/IO/JSON/JSONIOHandlerImpl.cpp b/src/IO/JSON/JSONIOHandlerImpl.cpp index 2add2727bd..2287522d92 100644 --- a/src/IO/JSON/JSONIOHandlerImpl.cpp +++ b/src/IO/JSON/JSONIOHandlerImpl.cpp @@ -1303,7 +1303,8 @@ void JSONIOHandlerImpl::writeAttribute( (*jsonVal)[filePosition->id]["attributes"] = nlohmann::json::object(); } nlohmann::json value; - switchType(parameter.dtype, value, parameter.resource); + switchType( + parameter.dtype, value, parameter.resource()); switch (m_attributeMode.m_mode) { case AttributeMode::Long: @@ -1668,7 +1669,7 @@ void JSONIOHandlerImpl::readAttribute( { Attribute attr = recoverAttributeFromJson(j, name); *parameters.dtype = attr.dtype; - *parameters.resource = attr.getResource(); + *parameters.m_resource = attr.getAny(); } } catch (json::type_error &) @@ -2506,7 +2507,7 @@ void JSONIOHandlerImpl::DatasetReader::call( template void JSONIOHandlerImpl::AttributeWriter::call( - nlohmann::json &value, Attribute::resource const &resource) + nlohmann::json &value, attribute_types const &resource) { CppToJSON ctj; value = ctj(std::get(resource)); @@ -2517,7 +2518,7 @@ void JSONIOHandlerImpl::AttributeReader::call( nlohmann::json const &json, Parameter ¶meters) { JsonToCpp jtc; - *parameters.resource = jtc(json); + parameters.setResource(jtc(json)); } template diff --git a/src/Iteration.cpp b/src/Iteration.cpp index 64cfaa39d0..39ae03fdca 100644 --- a/src/Iteration.cpp +++ b/src/Iteration.cpp @@ -31,6 +31,7 @@ #include "openPMD/auxiliary/StringManip.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/backend/Writable.hpp" #include @@ -330,7 +331,7 @@ void Iteration::flushVariableBased( wAttr.changesOverSteps = Parameter::ChangesOverSteps::IfPossible; wAttr.name = "snapshot"; - wAttr.resource = (unsigned long long)i; + wAttr.setResource(i); wAttr.dtype = Datatype::ULONGLONG; IOHandler()->enqueue(IOTask(this, wAttr)); } @@ -480,13 +481,15 @@ void Iteration::read_impl(std::string const &groupPath) IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) - setDt(Attribute(*aRead.resource).get()); + setDt(Attribute(Attribute::from_any, *aRead.m_resource).get()); else if (*aRead.dtype == DT::DOUBLE) - setDt(Attribute(*aRead.resource).get()); + setDt(Attribute(Attribute::from_any, *aRead.m_resource).get()); else if (*aRead.dtype == DT::LONG_DOUBLE) - setDt(Attribute(*aRead.resource).get()); + setDt(Attribute(Attribute::from_any, *aRead.m_resource) + .get()); // conversion cast if a backend reports an integer type - else if (auto val = Attribute(*aRead.resource).getOptional(); + else if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setDt(val.value()); else @@ -495,19 +498,24 @@ void Iteration::read_impl(std::string const &groupPath) error::Reason::UnexpectedContent, {}, "Unexpected Attribute datatype for 'dt' (expected double, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "time"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::FLOAT) - setTime(Attribute(*aRead.resource).get()); + setTime(Attribute(Attribute::from_any, *aRead.m_resource).get()); else if (*aRead.dtype == DT::DOUBLE) - setTime(Attribute(*aRead.resource).get()); + setTime( + Attribute(Attribute::from_any, *aRead.m_resource).get()); else if (*aRead.dtype == DT::LONG_DOUBLE) - setTime(Attribute(*aRead.resource).get()); + setTime(Attribute(Attribute::from_any, *aRead.m_resource) + .get()); // conversion cast if a backend reports an integer type - else if (auto val = Attribute(*aRead.resource).getOptional(); + else if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setTime(val.value()); else @@ -517,12 +525,15 @@ void Iteration::read_impl(std::string const &groupPath) {}, "Unexpected Attribute datatype for 'time' (expected double, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "timeUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setTimeUnitSI(val.value()); else @@ -532,7 +543,9 @@ void Iteration::read_impl(std::string const &groupPath) {}, "Unexpected Attribute datatype for 'timeUnitSI' (expected double, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); /* Find the root point [Series] of this file, * meshesPath and particlesPath are stored there */ diff --git a/src/Mesh.cpp b/src/Mesh.cpp index c5cdefe483..227a8764a3 100644 --- a/src/Mesh.cpp +++ b/src/Mesh.cpp @@ -430,7 +430,7 @@ void Mesh::flush_impl( void Mesh::read() { - internal::EraseStaleEntries map{*this}; + internal::EraseStaleEntries map{*this}; using DT = Datatype; Parameter aRead; @@ -440,7 +440,9 @@ void Mesh::read() IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::STRING) { - std::string tmpGeometry = Attribute(*aRead.resource).get(); + std::string tmpGeometry = + Attribute(Attribute::from_any, *aRead.m_resource) + .get(); if ("cartesian" == tmpGeometry) setGeometry(Geometry::cartesian); else if ("thetaMode" == tmpGeometry) @@ -459,18 +461,22 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'geometry' (expected a string, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "dataOrder"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (*aRead.dtype == DT::CHAR) setDataOrder( - static_cast(Attribute(*aRead.resource).get())); + static_cast( + Attribute(Attribute::from_any, *aRead.m_resource).get())); else if (*aRead.dtype == DT::STRING) { std::string tmpDataOrder = - Attribute(*aRead.resource).get(); + Attribute(Attribute::from_any, *aRead.m_resource) + .get(); if (tmpDataOrder.size() == 1) setDataOrder(static_cast(tmpDataOrder[0])); else @@ -487,12 +493,14 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'dataOrder' (expected char or " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "axisLabels"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - Attribute a = Attribute(*aRead.resource); + Attribute a = Attribute(Attribute::from_any, *aRead.m_resource); if (auto val = a.getOptional>(); val.has_value()) setAxisLabels(*val); else @@ -502,12 +510,14 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'axisLabels' (expected a vector " "of string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "gridSpacing"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - a = Attribute(*aRead.resource); + a = Attribute(Attribute::from_any, *aRead.m_resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setGridSpacing(a.get>()); else if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) @@ -525,13 +535,15 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'gridSpacing' (expected a " "vector of double, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "gridGlobalOffset"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = - Attribute(*aRead.resource).getOptional>(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional>(); val.has_value()) setGridGlobalOffset(val.value()); else @@ -541,15 +553,17 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'gridGlobalOffset' (expected a " "vector of double, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "gridUnitSI"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); if (IOHandler()->m_standard >= OpenpmdStandard::v_2_0_0) { - if (auto val = - Attribute(*aRead.resource).getOptional>(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional>(); val.has_value()) setGridUnitSIPerDimension(val.value()); else @@ -559,11 +573,15 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'gridUnitSI' " "(expected vector of double, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); } else { - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setGridUnitSI(val.value()); else @@ -573,7 +591,10 @@ void Mesh::read() {}, "Unexpected Attribute datatype for 'gridUnitSI' " "(expected double, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); } if (scalar()) diff --git a/src/ParticleSpecies.cpp b/src/ParticleSpecies.cpp index 4006cc82ba..4debca137c 100644 --- a/src/ParticleSpecies.cpp +++ b/src/ParticleSpecies.cpp @@ -40,7 +40,7 @@ void ParticleSpecies::read() IOHandler()->enqueue(IOTask(this, pList)); IOHandler()->flush(internal::defaultFlushParams); - internal::EraseStaleEntries map{*this}; + internal::EraseStaleEntries map{*this}; Parameter pOpen; Parameter aList; diff --git a/src/RecordComponent.cpp b/src/RecordComponent.cpp index d5420ea3b8..9a5c2b6b04 100644 --- a/src/RecordComponent.cpp +++ b/src/RecordComponent.cpp @@ -27,6 +27,10 @@ #include "openPMD/auxiliary/Memory.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/Variant_internal.hpp" + +// comment so clang-format does not move this +#include "openPMD/DatatypeMacros.hpp" #include #include @@ -57,6 +61,12 @@ namespace internal } } // namespace internal +template +auto resource(T &t) -> attribute_types & +{ + return t.template resource(); +} + RecordComponent::RecordComponent() : BaseRecordComponent(NoInit()) { setData(std::make_shared()); @@ -225,7 +235,7 @@ RecordComponent &RecordComponent::makeEmpty(Dataset d) setDirty(true); if (!written()) { - switchType >( + switchType>( rc.m_dataset.value().dtype, *this); } return *this; @@ -292,7 +302,7 @@ void RecordComponent::flush( Parameter aWrite; aWrite.name = "value"; aWrite.dtype = rc.m_constantValue.dtype; - aWrite.resource = rc.m_constantValue.getResource(); + aWrite.m_resource = rc.m_constantValue.getAny(); if (isVBased) { aWrite.changesOverSteps = Parameter< @@ -302,7 +312,7 @@ void RecordComponent::flush( aWrite.name = "shape"; Attribute a(getExtent()); aWrite.dtype = a.dtype; - aWrite.resource = a.getResource(); + aWrite.m_resource = a.getAny(); if (isVBased) { aWrite.changesOverSteps = Parameter< @@ -329,7 +339,7 @@ void RecordComponent::flush( aWrite.name = "shape"; Attribute a(getExtent()); aWrite.dtype = a.dtype; - aWrite.resource = a.getResource(); + aWrite.m_resource = a.getAny(); if (isVBased) { aWrite.changesOverSteps = Parameter< @@ -399,7 +409,7 @@ void RecordComponent::readBase(bool require_unit_si) IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - Attribute a(*aRead.resource); + Attribute a(Attribute::from_any, *aRead.m_resource); DT dtype = *aRead.dtype; setWritten(false, Attributable::EnqueueAsynchronously::No); switchNonVectorType(dtype, *this, a); @@ -408,11 +418,11 @@ void RecordComponent::readBase(bool require_unit_si) aRead.name = "shape"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - a = Attribute(*aRead.resource); + a = Attribute(Attribute::from_any, *aRead.m_resource); Extent e; // uint64_t check - if (auto val = a.getOptional >(); val.has_value()) + if (auto val = a.getOptional>(); val.has_value()) for (auto const &shape : val.value()) e.push_back(shape); else @@ -455,7 +465,9 @@ void RecordComponent::readBase(bool require_unit_si) {}, "Unexpected Attribute datatype for 'unitSI' (expected double, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + ") in '" + myPath().openPMDPath() + "'."); } } @@ -566,4 +578,330 @@ auto RecordComponent::loadChunkVariant(Offset o, Extent e) { return visit(std::move(o), std::move(e)); } + +template +RecordComponent &RecordComponent::makeConstant(T value) +{ + if (written()) + throw std::runtime_error( + "A recordComponent can not (yet) be made constant after it has " + "been written."); + + auto &rc = get(); + + rc.m_constantValue = Attribute(value); + rc.m_isConstant = true; + return *this; +} + +template +RecordComponent &RecordComponent::makeEmpty(uint8_t dimensions) +{ + return makeEmpty(Dataset(determineDatatype(), Extent(dimensions, 0))); +} + +template +std::shared_ptr RecordComponent::loadChunk(Offset o, Extent e) +{ + uint8_t dim = getDimensionality(); + + // default arguments + // offset = {0u}: expand to right dim {0u, 0u, ...} + Offset offset = o; + if (o.size() == 1u && o.at(0) == 0u && dim > 1u) + offset = Offset(dim, 0u); + + // extent = {-1u}: take full size + Extent extent(dim, 1u); + if (e.size() == 1u && e.at(0) == -1u) + { + extent = getExtent(); + for (uint8_t i = 0u; i < dim; ++i) + extent[i] -= offset[i]; + } + else + extent = e; + + uint64_t numPoints = 1u; + for (auto const &dimensionSize : extent) + numPoints *= dimensionSize; + +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 11000) || \ + (defined(__apple_build_version__) && __clang_major__ < 14) + auto newData = + std::shared_ptr(new T[numPoints], [](T *p) { delete[] p; }); + loadChunk(newData, offset, extent); + return newData; +#else + auto newData = std::shared_ptr[]>( + new std::remove_extent_t[numPoints]); + loadChunk(newData, offset, extent); + return std::static_pointer_cast(std::move(newData)); +#endif +} + +namespace detail +{ + template + struct do_convert + { + template + static std::optional call(Attribute &attr) + { + if constexpr (std::is_convertible_v) + { + return std::make_optional(attr.get()); + } + else + { + return std::nullopt; + } + } + + static constexpr char const *errorMsg = "is_conversible"; + }; +} // namespace detail + +template +void RecordComponent::loadChunk(std::shared_ptr data, Offset o, Extent e) +{ + Datatype dtype = determineDatatype(data); + /* + * For constant components, we implement type conversion, so there is + * a separate check further below. + * This is especially useful for the short-attribute representation in the + * JSON/TOML backends as they might implicitly turn a LONG into an INT in a + * constant component. The frontend needs to catch such edge cases. + * Ref. `if (constant())` branch. + */ + if (dtype != getDatatype() && !constant()) + if (!isSameInteger(getDatatype()) && + !isSameFloatingPoint(getDatatype()) && + !isSameComplexFloatingPoint(getDatatype()) && + !isSameChar(getDatatype())) + { + std::string const data_type_str = datatypeToString(getDatatype()); + std::string const requ_type_str = + datatypeToString(determineDatatype()); + std::string err_msg = + "Type conversion during chunk loading not yet implemented! "; + err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; + throw std::runtime_error(err_msg); + } + + uint8_t dim = getDimensionality(); + + // default arguments + // offset = {0u}: expand to right dim {0u, 0u, ...} + Offset offset = o; + if (o.size() == 1u && o.at(0) == 0u && dim > 1u) + offset = Offset(dim, 0u); + + // extent = {-1u}: take full size + Extent extent(dim, 1u); + if (e.size() == 1u && e.at(0) == -1u) + { + extent = getExtent(); + for (uint8_t i = 0u; i < dim; ++i) + extent[i] -= offset[i]; + } + else + extent = e; + + if (extent.size() != dim || offset.size() != dim) + { + std::ostringstream oss; + oss << "Dimensionality of chunk (" + << "offset=" << offset.size() << "D, " + << "extent=" << extent.size() << "D) " + << "and record component (" << int(dim) << "D) " + << "do not match."; + throw std::runtime_error(oss.str()); + } + Extent dse = getExtent(); + for (uint8_t i = 0; i < dim; ++i) + if (dse[i] < offset[i] + extent[i]) + throw std::runtime_error( + "Chunk does not reside inside dataset (Dimension on index " + + std::to_string(i) + ". DS: " + std::to_string(dse[i]) + + " - Chunk: " + std::to_string(offset[i] + extent[i]) + ")"); + if (!data) + throw std::runtime_error( + "Unallocated pointer passed during chunk loading."); + + auto &rc = get(); + if (constant()) + { + uint64_t numPoints = 1u; + for (auto const &dimensionSize : extent) + numPoints *= dimensionSize; + + std::optional val = + switchNonVectorType>( + /* dt = */ getDatatype(), rc.m_constantValue); + + if (val.has_value()) + { + T *raw_ptr = data.get(); + std::fill(raw_ptr, raw_ptr + numPoints, *val); + } + else + { + std::string const data_type_str = datatypeToString(getDatatype()); + std::string const requ_type_str = + datatypeToString(determineDatatype()); + std::string err_msg = + "Type conversion during chunk loading not possible! "; + err_msg += "Data: " + data_type_str + "; Load as: " + requ_type_str; + throw error::WrongAPIUsage(err_msg); + } + } + else + { + Parameter dRead; + dRead.offset = offset; + dRead.extent = extent; + dRead.dtype = getDatatype(); + dRead.data = std::static_pointer_cast(data); + rc.push_chunk(IOTask(this, dRead)); + } +} + +template +void RecordComponent::loadChunk( + std::shared_ptr ptr, Offset offset, Extent extent) +{ + loadChunk( + std::static_pointer_cast(std::move(ptr)), + std::move(offset), + std::move(extent)); +} + +template +void RecordComponent::loadChunkRaw(T *ptr, Offset offset, Extent extent) +{ + loadChunk(auxiliary::shareRaw(ptr), std::move(offset), std::move(extent)); +} + +template +void RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) +{ + if (!data) + throw std::runtime_error( + "Unallocated pointer passed during chunk store."); + Datatype dtype = determineDatatype(data); + + /* std::static_pointer_cast correctly reference-counts the pointer */ + storeChunk( + auxiliary::WriteBuffer(std::static_pointer_cast(data)), + dtype, + std::move(o), + std::move(e)); +} + +template +void RecordComponent::storeChunk( + UniquePtrWithLambda data, Offset o, Extent e) +{ + if (!data) + throw std::runtime_error( + "Unallocated pointer passed during chunk store."); + Datatype dtype = determineDatatype<>(data); + + storeChunk( + auxiliary::WriteBuffer{std::move(data).template static_cast_()}, + dtype, + std::move(o), + std::move(e)); +} + +template +void RecordComponent::storeChunk(std::shared_ptr data, Offset o, Extent e) +{ + storeChunk( + std::static_pointer_cast(std::move(data)), + std::move(o), + std::move(e)); +} + +template +void RecordComponent::storeChunkRaw(T *ptr, Offset offset, Extent extent) +{ + storeChunk(auxiliary::shareRaw(ptr), std::move(offset), std::move(extent)); +} + +template +DynamicMemoryView RecordComponent::storeChunk(Offset offset, Extent extent) +{ + return storeChunk(std::move(offset), std::move(extent), [](size_t size) { +#if (defined(_LIBCPP_VERSION) && _LIBCPP_VERSION < 11000) || \ + (defined(__apple_build_version__) && __clang_major__ < 14) + return std::shared_ptr{new T[size], [](auto *ptr) { delete[] ptr; }}; +#else + return std::shared_ptr< T[] >{ new T[ size ] }; +#endif + }); +} + +template +void RecordComponent::verifyChunk(Offset const &o, Extent const &e) const +{ + verifyChunk(determineDatatype(), o, e); +} + +// Needed for clang-tidy's peace of mind. +#define OPENPMD_PTR(type) type * +#define OPENPMD_ARRAY(type) type[] + +#define OPENPMD_INSTANTIATE_BASIC(type) \ + template void RecordComponent::loadChunk( \ + std::shared_ptr data, Offset o, Extent e); \ + template void RecordComponent::loadChunk( \ + std::shared_ptr data, Offset o, Extent e); \ + template void RecordComponent::loadChunkRaw( \ + OPENPMD_PTR(type) ptr, Offset offset, Extent extent); \ + template void RecordComponent::verifyChunk( \ + Offset const &o, Extent const &e) const; \ + template DynamicMemoryView RecordComponent::storeChunk( \ + Offset offset, Extent extent); + +#define OPENPMD_INSTANTIATE_CONST_AND_NONCONST(type) \ + template void RecordComponent::storeChunk( \ + std::shared_ptr data, Offset o, Extent e); \ + template void RecordComponent::storeChunk( \ + std::shared_ptr data, Offset o, Extent e); \ + template void RecordComponent::storeChunkRaw( \ + OPENPMD_PTR(type) ptr, Offset offset, Extent extent); + +#define OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT(type) \ + template std::shared_ptr RecordComponent::loadChunk( \ + Offset o, Extent e); \ + template void RecordComponent::storeChunk( \ + UniquePtrWithLambda data, Offset o, Extent e); + +#define OPENPMD_INSTANTIATE_FULLMATRIX(type) \ + template RecordComponent &RecordComponent::makeConstant(type); \ + template RecordComponent &RecordComponent::makeEmpty( \ + uint8_t dimensions); + +#define OPENPMD_INSTANTIATE(type) \ + OPENPMD_INSTANTIATE_BASIC(type) \ + OPENPMD_INSTANTIATE_CONST_AND_NONCONST(type) \ + OPENPMD_INSTANTIATE_CONST_AND_NONCONST(type const) \ + OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT(type) \ + OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT(OPENPMD_ARRAY(type)) \ + OPENPMD_INSTANTIATE_FULLMATRIX(type) \ + OPENPMD_INSTANTIATE_FULLMATRIX(type const) \ + OPENPMD_INSTANTIATE_FULLMATRIX(OPENPMD_ARRAY(type)) \ + OPENPMD_INSTANTIATE_FULLMATRIX(type const[]) + +OPENPMD_FOREACH_NONVECTOR_DATATYPE(OPENPMD_INSTANTIATE) +#undef OPENPMD_INSTANTIATE +#undef OPENPMD_INSTANTIATE_FULLMATRIX +#undef OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT +#undef OPENPMD_INSTANTIATE_CONST_AND_NONCONST +#undef OPENPMD_INSTANTIATE_BASIC +#undef OPENPMD_PTR +#undef OPENPMD_ARRAY + } // namespace openPMD diff --git a/src/Series.cpp b/src/Series.cpp index 04fb8eb0c5..32179cd0bd 100644 --- a/src/Series.cpp +++ b/src/Series.cpp @@ -39,6 +39,7 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Attribute.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/snapshots/ContainerImpls.hpp" #include "openPMD/snapshots/ContainerTraits.hpp" #include "openPMD/snapshots/Snapshots.hpp" @@ -1557,7 +1558,7 @@ void Series::flushMeshesPath() Parameter aWrite; aWrite.name = "meshesPath"; Attribute a = getAttribute("meshesPath"); - aWrite.resource = a.getResource(); + aWrite.m_resource = a.getAny(); aWrite.dtype = a.dtype; IOHandler()->enqueue(IOTask(this, aWrite)); } @@ -1567,7 +1568,7 @@ void Series::flushParticlesPath() Parameter aWrite; aWrite.name = "particlesPath"; Attribute a = getAttribute("particlesPath"); - aWrite.resource = a.getResource(); + aWrite.m_resource = a.getAny(); aWrite.dtype = a.dtype; IOHandler()->enqueue(IOTask(this, aWrite)); } @@ -1816,7 +1817,8 @@ void Series::readOneIterationFileBased(std::string const &filePath) IterationEncoding encoding_out; if (*aRead.dtype == DT::STRING) { - std::string encoding = Attribute(*aRead.resource).get(); + std::string encoding = Attribute(Attribute::from_any, *aRead.m_resource) + .get(); if (encoding == "fileBased") encoding_out = IterationEncoding::fileBased; else if (encoding == "groupBased") @@ -1861,7 +1863,9 @@ void Series::readOneIterationFileBased(std::string const &filePath) throw std::runtime_error( "Unexpected Attribute datatype for 'iterationEncoding' (expected " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -1869,7 +1873,8 @@ void Series::readOneIterationFileBased(std::string const &filePath) if (*aRead.dtype == DT::STRING) { setWritten(false, Attributable::EnqueueAsynchronously::No); - setIterationFormat(Attribute(*aRead.resource).get()); + setIterationFormat(Attribute(Attribute::from_any, *aRead.m_resource) + .get()); setWritten(true, Attributable::EnqueueAsynchronously::No); } else @@ -1879,7 +1884,9 @@ void Series::readOneIterationFileBased(std::string const &filePath) {}, "Unexpected Attribute datatype for 'iterationFormat' (expected " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); Parameter pOpen; std::string version = openPMD(); @@ -1949,7 +1956,8 @@ auto Series::readGorVBased( if (*aRead.dtype == DT::STRING) { std::string encoding = - Attribute(*aRead.resource).get(); + Attribute(Attribute::from_any, *aRead.m_resource) + .get(); if (encoding == "groupBased") series.m_iterationEncoding = IterationEncoding::groupBased; else if (encoding == "variableBased") @@ -2003,7 +2011,10 @@ creating new iterations. {}, "Unexpected Attribute datatype for 'iterationEncoding' " "(expected string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); aRead.name = "iterationFormat"; IOHandler()->enqueue(IOTask(this, aRead)); @@ -2011,7 +2022,8 @@ creating new iterations. if (*aRead.dtype == DT::STRING) { setWritten(false, Attributable::EnqueueAsynchronously::No); - setIterationFormat(Attribute(*aRead.resource).get()); + setIterationFormat(Attribute(Attribute::from_any, *aRead.m_resource) + .get()); setWritten(true, Attributable::EnqueueAsynchronously::No); } else @@ -2021,7 +2033,10 @@ creating new iterations. {}, "Unexpected Attribute datatype for 'iterationFormat' (expected " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); } Parameter pOpen; @@ -2249,7 +2264,8 @@ void Series::readBase() aRead.name = "openPMD"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setOpenPMD(val.value()); else @@ -2259,12 +2275,15 @@ void Series::readBase() {}, "Unexpected Attribute datatype for 'openPMD' (expected string, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "openPMDextension"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) setOpenPMDextension(val.value()); else @@ -2274,12 +2293,15 @@ void Series::readBase() {}, "Unexpected Attribute datatype for 'openPMDextension' (expected " "uint32, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); aRead.name = "basePath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) { if ( // might have been previously initialized in READ_LINEAR access @@ -2304,7 +2326,9 @@ void Series::readBase() {}, "Unexpected Attribute datatype for 'basePath' (expected string, " "found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); Parameter aList; IOHandler()->enqueue(IOTask(this, aList)); @@ -2316,7 +2340,8 @@ void Series::readBase() aRead.name = "meshesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2337,7 +2362,10 @@ void Series::readBase() {}, "Unexpected Attribute datatype for 'meshesPath' (expected " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); } else { @@ -2354,7 +2382,8 @@ void Series::readBase() aRead.name = "particlesPath"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = Attribute(*aRead.resource).getOptional(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); val.has_value()) { /* allow setting the meshes path after completed IO */ @@ -2375,7 +2404,10 @@ void Series::readBase() {}, "Unexpected Attribute datatype for 'particlesPath' (expected " "string, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource) + .dtype) + + ")"); } { // Make sure that the particlesPath does not leak from one iteration @@ -2645,9 +2677,10 @@ void Series::flushStep(bool doFlush) wAttr.changesOverSteps = Parameter::ChangesOverSteps::Yes; wAttr.name = "snapshot"; - wAttr.resource = std::vector{ - series.m_currentlyActiveIterations.begin(), - series.m_currentlyActiveIterations.end()}; + wAttr.setResource( + std::vector{ + series.m_currentlyActiveIterations.begin(), + series.m_currentlyActiveIterations.end()}); series.m_currentlyActiveIterations.clear(); wAttr.dtype = Datatype::VEC_ULONGLONG; IOHandler()->enqueue(IOTask(&series.iterations, wAttr)); @@ -3470,7 +3503,7 @@ auto Series::preparseSnapshots() return res; } }, - *readAttr.resource); + readAttr.resource()); } bool Series::randomAccessSteps() const diff --git a/src/auxiliary/Memory.cpp b/src/auxiliary/Memory.cpp new file mode 100644 index 0000000000..0d37c8c2ca --- /dev/null +++ b/src/auxiliary/Memory.cpp @@ -0,0 +1,190 @@ +/* Copyright 2017-2021 Fabian Koller + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/Memory.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace openPMD::auxiliary +{ +std::unique_ptr> +allocatePtr(Datatype dtype, uint64_t numPoints) +{ + void *data = nullptr; + std::function del = [](void *) {}; + switch (dtype) + { + using DT = Datatype; + case DT::VEC_STRING: + data = new char *[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_LONG_DOUBLE: + case DT::LONG_DOUBLE: + data = new long double[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::ARR_DBL_7: + case DT::VEC_DOUBLE: + case DT::DOUBLE: + data = new double[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_FLOAT: + case DT::FLOAT: + data = new float[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_CLONG_DOUBLE: + case DT::CLONG_DOUBLE: + data = new std::complex[numPoints]; + del = [](void *p) { + delete[] static_cast *>(p); + }; + break; + case DT::VEC_CDOUBLE: + case DT::CDOUBLE: + data = new std::complex[numPoints]; + del = [](void *p) { delete[] static_cast *>(p); }; + break; + case DT::VEC_CFLOAT: + case DT::CFLOAT: + data = new std::complex[numPoints]; + del = [](void *p) { delete[] static_cast *>(p); }; + break; + case DT::VEC_SHORT: + case DT::SHORT: + data = new short[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_INT: + case DT::INT: + data = new int[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_LONG: + case DT::LONG: + data = new long[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_LONGLONG: + case DT::LONGLONG: + data = new long long[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_USHORT: + case DT::USHORT: + data = new unsigned short[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_UINT: + case DT::UINT: + data = new unsigned int[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_ULONG: + case DT::ULONG: + data = new unsigned long[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_ULONGLONG: + case DT::ULONGLONG: + data = new unsigned long long[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_CHAR: + case DT::CHAR: + data = new char[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_UCHAR: + case DT::UCHAR: + data = new unsigned char[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::VEC_SCHAR: + case DT::SCHAR: + data = new signed char[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::BOOL: + data = new bool[numPoints]; + del = [](void *p) { delete[] static_cast(p); }; + break; + case DT::STRING: + /* user assigns c_str pointer */ + break; + case DT::UNDEFINED: + default: + throw std::runtime_error( + "Unknown Attribute datatype (Pointer allocation)"); + } + + return std::unique_ptr>(data, del); +} + +std::unique_ptr> +allocatePtr(Datatype dtype, Extent const &e) +{ + uint64_t numPoints = 1u; + for (auto const &dimensionSize : e) + numPoints *= dimensionSize; + return allocatePtr(dtype, numPoints); +} + +WriteBuffer::WriteBuffer() : m_buffer(UniquePtrWithLambda()) +{} + +WriteBuffer::WriteBuffer(WriteBuffer &&) noexcept( + noexcept(EligibleTypes(std::declval()))) = default; +WriteBuffer &WriteBuffer::operator=(WriteBuffer &&) noexcept(noexcept( + std::declval() = std::declval())) = + default; + +WriteBuffer const &WriteBuffer::operator=(std::shared_ptr ptr) +{ + m_buffer = std::move(ptr); + return *this; +} + +WriteBuffer const &WriteBuffer::operator=(UniquePtrWithLambda ptr) +{ + m_buffer = std::move(ptr); + return *this; +} + +void const *WriteBuffer::get() const +{ + return std::visit( + [](auto const &arg) { + // unique_ptr and shared_ptr both have the get() member, so + // we're being sneaky and don't distinguish the types here + return static_cast(arg.get()); + }, + m_buffer); +} +} // namespace openPMD::auxiliary diff --git a/src/auxiliary/UniquePtr.cpp b/src/auxiliary/UniquePtr.cpp new file mode 100644 index 0000000000..bc9af25ae1 --- /dev/null +++ b/src/auxiliary/UniquePtr.cpp @@ -0,0 +1,109 @@ +/* Copyright 2024-2025 Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/auxiliary/UniquePtr.hpp" + +#include "openPMD/DatatypeMacros.hpp" + +namespace openPMD +{ + +namespace auxiliary +{ + template + CustomDelete::CustomDelete() + : deleter_type{[]([[maybe_unused]] T_decayed *ptr) { + if constexpr (std::is_void_v) + { + std::cerr << "[Warning] Cannot standard-delete a void-type " + "pointer. Please specify a custom destructor. " + "Will let the memory leak." + << std::endl; + } + else + { + std::default_delete{}(ptr); + } + }} + {} + + template + CustomDelete::CustomDelete(deleter_type func) + : deleter_type(std::move(func)) + {} + +#define OPENPMD_INSTANTIATE(type) template class CustomDelete; +// need this for clang-tidy +#define OPENPMD_ARRAY(type) type[] +#define OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT(type) \ + OPENPMD_INSTANTIATE(type) OPENPMD_INSTANTIATE(OPENPMD_ARRAY(type)) + + OPENPMD_FOREACH_DATASET_DATATYPE( + OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT) + OPENPMD_INSTANTIATE(void) +#undef OPENPMD_INSTANTIATE +#undef OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT + +} // namespace auxiliary + +template +UniquePtrWithLambda::UniquePtrWithLambda() = default; + +template +UniquePtrWithLambda::UniquePtrWithLambda(UniquePtrWithLambda &&) noexcept = + default; +template +UniquePtrWithLambda & +UniquePtrWithLambda::operator=(UniquePtrWithLambda &&) noexcept = default; + +template +template +UniquePtrWithLambda::UniquePtrWithLambda(bare_unique_ptr stdPtr) + : BasePtr{stdPtr.release()} +{} + +template +UniquePtrWithLambda::UniquePtrWithLambda(T_decayed *ptr) : BasePtr{ptr} +{} + +template +UniquePtrWithLambda::UniquePtrWithLambda( + T_decayed *ptr, std::function deleter) + : BasePtr{ptr, std::move(deleter)} +{} + +#define OPENPMD_INSTANTIATE(type) \ + template class UniquePtrWithLambda; \ + template UniquePtrWithLambda::UniquePtrWithLambda( \ + std::unique_ptr); + +#define OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT(type) \ + OPENPMD_INSTANTIATE(type) OPENPMD_INSTANTIATE(OPENPMD_ARRAY(type)) + +OPENPMD_FOREACH_DATASET_DATATYPE(OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT) +// Instantiate this directly, do not instantiate the +// `std::unique_ptr`-based constructor. +template class UniquePtrWithLambda; +#undef OPENPMD_INSTANTIATE +#undef OPENPMD_INSTANTIATE_WITH_AND_WITHOUT_EXTENT +#undef OPENPMD_ARRAY + +} // namespace openPMD diff --git a/src/auxiliary/Variant.cpp b/src/auxiliary/Variant.cpp new file mode 100644 index 0000000000..d1d3b5f090 --- /dev/null +++ b/src/auxiliary/Variant.cpp @@ -0,0 +1,65 @@ +#include "openPMD/auxiliary/Variant.hpp" +#include "openPMD/Datatype.hpp" +#include "openPMD/DatatypeMacros.hpp" + +#include +#include + +namespace openPMD::auxiliary +{ +template +template +Variant::Variant(from_basic_type_tag, U u) + : dtype{determineDatatype()} + , m_data{std::make_any>(std::move(u))} +{} + +template +Variant::Variant(from_any_tag, std::any any) + : m_data{std::move(any)} +{ + std::visit( + [this](auto const &val) { + using type = + std::remove_reference_t>; + dtype = determineDatatype(); + }, + getVariant>()); +} + +template +template +Variant::Variant(U u) + : Variant(from_basic_type, std::move(u)) +{} + +template +template +U const &Variant::get() const +{ + using variant_t = std::variant; + variant_t const &v = getVariant(); + return std::get(v); +} + +template +size_t Variant::index() const +{ + return getVariant>().index(); +} + +#define OPENPMD_ENUMERATE_TYPES(type) , type +using VariantInstantiated = + Variant; +template class Variant; +#undef OPENPMD_ENUMERATE_TYPES + +#define OPENPMD_ENUMERATE_INSTANTIATIONS(type) \ + template VariantInstantiated::Variant(from_basic_type_tag, type); \ + template VariantInstantiated::Variant(type); \ + template type const &VariantInstantiated::get() const; +OPENPMD_FOREACH_DATATYPE(OPENPMD_ENUMERATE_INSTANTIATIONS) +#undef OPENPMD_ENUMERATE_INSTANTIATIONS + +} // namespace openPMD::auxiliary diff --git a/src/backend/Attributable.cpp b/src/backend/Attributable.cpp index 82a9d3c4e9..6de68ffcd2 100644 --- a/src/backend/Attributable.cpp +++ b/src/backend/Attributable.cpp @@ -298,7 +298,7 @@ void Attributable::flushAttributes(internal::FlushParams const &flushParams) for (std::string const &att_name : attributes()) { aWrite.name = att_name; - aWrite.resource = getAttribute(att_name).getResource(); + aWrite.m_resource = getAttribute(att_name).getAny(); aWrite.dtype = getAttribute(att_name).dtype; IOHandler()->enqueue(IOTask(this, aWrite)); } @@ -363,7 +363,7 @@ void Attributable::readAttributes(ReadMode mode) << e.what() << ")\n"; continue; } - Attribute a(*aRead.resource); + Attribute a(Attribute::from_any, *aRead.m_resource); auto guardUnitDimension = [this](std::string const &key, auto vector) { if (key == "unitDimension") diff --git a/src/backend/Attribute.cpp b/src/backend/Attribute.cpp new file mode 100644 index 0000000000..3f2b18c57a --- /dev/null +++ b/src/backend/Attribute.cpp @@ -0,0 +1,129 @@ +#include "openPMD/backend/Attribute.hpp" + +#include "openPMD/Datatype.hpp" +#include "openPMD/DatatypeMacros.hpp" +#include "openPMD/auxiliary/TypeTraits.hpp" +#include "openPMD/backend/Variant_internal.hpp" + +#include +#include + +namespace openPMD +{ +// NOLINTNEXTLINE(bugprone-macro-parentheses) +#define OPENPMD_ENUMERATE_TYPES(type) +1 +constexpr static size_t num_datatypes = + 0 OPENPMD_FOREACH_DATATYPE(OPENPMD_ENUMERATE_TYPES); +#undef OPENPMD_ENUMERATE_TYPES + +/* + * These instantiations are somewhat complicated because we want to avoid + * instantiating all possible combinations of (1) requested type and (2) actual + * type. Hence, first define a bool array in eligible_conversions() + * on which types in the Datatype enum are actually convertible to TargetType. + * Only then instantiate. + */ + +template +constexpr auto eligible_conversions() -> std::array +{ + if constexpr ( + auxiliary::IsVector_v || auxiliary::IsArray_v) + { + return eligible_conversions(); + } + else + { + std::array res = {}; +#define OPENPMD_ENUMERATE_TYPES(type) \ + res[datatypeIndex()] = std::is_convertible_v; + OPENPMD_FOREACH_NONVECTOR_DATATYPE(OPENPMD_ENUMERATE_TYPES) +#undef OPENPMD_ENUMERATE_TYPES +#define OPENPMD_ENUMERATE_TYPES(type) \ + res[datatypeIndex()] = res[datatypeIndex()]; + OPENPMD_FOREACH_VECTOR_DATATYPE(OPENPMD_ENUMERATE_TYPES) +#undef OPENPMD_ENUMERATE_TYPES + return res; + } +} + +template +auto Attribute::get_impl() const -> std::variant +{ + constexpr std::array conversions = + eligible_conversions(); + auto variant = Variant::getVariant(); + size_t index = variant.index(); + +#define OPENPMD_ENUMERATE_TYPES(type) \ + case datatypeIndex(): { \ + if constexpr (!conversions[datatypeIndex()]) \ + { \ + std::stringstream error; \ + error << "Cannot convert from " << determineDatatype() \ + << " to " << determineDatatype() << "."; \ + return {std::runtime_error(error.str())}; \ + } \ + else \ + { \ + return detail::doConvert(&std::get(variant)); \ + } \ + break; \ + } + switch (index) + { + OPENPMD_FOREACH_DATATYPE(OPENPMD_ENUMERATE_TYPES) + } +#undef OPENPMD_ENUMERATE_TYPES + + throw std::runtime_error("Unreachable!"); +} + +template +U Attribute::get() const +{ + auto res = get_impl(); + + return std::visit( + [](auto &&containedValue) -> U { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + throw static_cast(containedValue); + } + else + { + return static_cast(containedValue); + } + }, + std::move(res)); +} + +template +std::optional Attribute::getOptional() const +{ + auto res = get_impl(); + + return std::visit( + [](auto &&containedValue) -> std::optional { + using T = std::decay_t; + if constexpr (std::is_same_v) + { + return std::nullopt; + } + else + { + return static_cast(containedValue); + } + }, + std::move(res)); +} + +#define OPENPMD_INSTANTIATE(type) \ + template type Attribute::get() const; \ + template std::optional Attribute::getOptional() const; + +OPENPMD_FOREACH_DATATYPE(OPENPMD_INSTANTIATE) + +#undef OPENPMD_INSTANTIATE +} // namespace openPMD diff --git a/src/backend/BaseRecord.cpp b/src/backend/BaseRecord.cpp new file mode 100644 index 0000000000..52bef0c0f7 --- /dev/null +++ b/src/backend/BaseRecord.cpp @@ -0,0 +1,873 @@ +/* Copyright 2017-2025 Fabian Koller, Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ +#include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +#include + +#define OPENPMD_FORALL_RECORDCOMPONENT_TYPES(MACRO) \ + MACRO(RecordComponent) \ + MACRO(MeshRecordComponent) \ + MACRO(PatchRecordComponent) + +namespace openPMD +{ +namespace internal +{ + template + BaseRecordData::BaseRecordData() + { + Attributable impl; + impl.setData({this, [](auto const *) {}}); + impl.setAttribute( + "unitDimension", + std::array{{0., 0., 0., 0., 0., 0., 0.}}); + } + +#define OPENPMD_INSTANTIATE(recordcomponenttype) \ + template class BaseRecordData< \ + recordcomponenttype, \ + recordcomponenttype::Data_t>; + + OPENPMD_FORALL_RECORDCOMPONENT_TYPES(OPENPMD_INSTANTIATE) + +#undef OPENPMD_INSTANTIATE + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + ScalarIterator:: + ScalarIterator() = default; + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + ScalarIterator:: + ScalarIterator(T_BaseRecord *baseRecord, bool is_end) + : m_baseRecordData(&baseRecord->get()), m_iterator(Right()) + { + if (!is_end) + { + m_scalarTuple.emplace( + RecordComponent::SCALAR, T_RecordComponent(*baseRecord)); + } + } + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + ScalarIterator:: + ScalarIterator(T_BaseRecord *baseRecord, Left iterator) + : m_baseRecordData(&baseRecord->get()), m_iterator(std::move(iterator)) + {} + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + auto ScalarIterator:: + operator++() -> ScalarIterator & + { + std::visit( + auxiliary::overloaded{ + [](Left &left) { ++left; }, + [this](Right &) { m_scalarTuple.reset(); }}, + m_iterator); + return *this; + } + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + auto ScalarIterator:: + operator->() -> T_Value * + { + return std::visit( + auxiliary::overloaded{ + [](Left &left) -> T_Value * { return left.operator->(); }, + [this](Right &) -> T_Value * { + /* + * We cannot create this value on the fly since we only + * give out a pointer, so that would be use-after-free. + * Instead, we just keep one value around inside + * BaseRecordData and give it out when needed. + */ + return &m_scalarTuple.value(); + }}, + m_iterator); + } + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + auto ScalarIterator:: + operator*() -> T_Value & + { + return *operator->(); + } + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + auto ScalarIterator:: + operator==(ScalarIterator const &other) const -> bool + { + return std::visit( + auxiliary::overloaded{ + [&other](Left const &this_left) -> bool { + return std::visit( + auxiliary::overloaded{ + [&this_left](Left const &other_left) -> bool { + return this_left == other_left; + }, + [](Right const &) { return false; }}, + other.m_iterator); + }, + [this, &other](Right const &) -> bool { + return std::visit( + auxiliary::overloaded{ + [](Left const &) -> bool { return false; }, + [this, &other](Right const &) { + return this->m_scalarTuple.has_value() == + other.m_scalarTuple.has_value(); + }}, + other.m_iterator); + }}, + this->m_iterator); + } + + template < + typename T_BaseRecord_, + typename T_BaseRecordData_, + typename T_BaseIterator> + auto ScalarIterator:: + operator!=(ScalarIterator const &other) const -> bool + { + return !operator==(other); + } + +#define INSTANTIATE_ITERATORS_FOR_BASERECORD(baserecordtype) \ + template class internal::ScalarIterator< \ + baserecordtype, \ + baserecordtype::Data_t, \ + baserecordtype::T_Container::InternalContainer::iterator>; \ + template class internal::ScalarIterator< \ + baserecordtype const, \ + baserecordtype::Data_t const, \ + baserecordtype::T_Container::InternalContainer::const_iterator>; \ + template class internal::ScalarIterator< \ + baserecordtype, \ + baserecordtype::Data_t, \ + baserecordtype::T_Container::InternalContainer::reverse_iterator>; \ + template class internal::ScalarIterator< \ + baserecordtype const, \ + baserecordtype::Data_t const, \ + baserecordtype::T_Container::InternalContainer:: \ + const_reverse_iterator>; + +#define OPENPMD_INSTANTIATE(recordcomponenttype) \ + INSTANTIATE_ITERATORS_FOR_BASERECORD(BaseRecord) + + OPENPMD_FORALL_RECORDCOMPONENT_TYPES(OPENPMD_INSTANTIATE) + +#undef OPENPMD_INSTANTIATE +} // namespace internal + +template +auto BaseRecord::begin() -> iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ false); + } + else + { + return makeIterator(T_Container::begin()); + } +} + +template +auto BaseRecord::begin() const -> const_iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ false); + } + else + { + return makeIterator(T_Container::begin()); + } +} + +template +auto BaseRecord::cbegin() const -> const_iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ false); + } + else + { + return makeIterator(T_Container::cbegin()); + } +} + +template +auto BaseRecord::end() -> iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ true); + } + else + { + return makeIterator(T_Container::end()); + } +} + +template +auto BaseRecord::end() const -> const_iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ true); + } + else + { + return makeIterator(T_Container::end()); + } +} + +template +auto BaseRecord::cend() const -> const_iterator +{ + if (get().m_datasetDefined) + { + return makeIterator(/* is_end = */ true); + } + else + { + return makeIterator(T_Container::cend()); + } +} + +template +auto BaseRecord::rbegin() -> reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ false); + } + else + { + return makeReverseIterator(this->container().rbegin()); + } +} + +template +auto BaseRecord::rbegin() const -> const_reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ false); + } + else + { + return makeReverseIterator(this->container().rbegin()); + } +} + +template +auto BaseRecord::crbegin() const -> const_reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ false); + } + else + { + return makeReverseIterator(this->container().crbegin()); + } +} + +template +auto BaseRecord::rend() -> reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ true); + } + else + { + return makeReverseIterator(this->container().rend()); + } +} + +template +auto BaseRecord::rend() const -> const_reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ true); + } + else + { + return makeReverseIterator(this->container().rend()); + } +} + +template +auto BaseRecord::crend() const -> const_reverse_iterator +{ + if (get().m_datasetDefined) + { + return makeReverseIterator(/* is_end = */ true); + } + else + { + return makeReverseIterator(this->container().crend()); + } +} + +template +BaseRecord::BaseRecord() + : T_Container(Attributable::NoInit()) + , T_RecordComponent(Attributable::NoInit()) +{ + setData(std::make_shared()); +} + +template +auto BaseRecord::operator[](key_type const &key) -> mapped_type & +{ + auto it = this->find(key); + if (it != this->end()) + { + return std::visit( + auxiliary::overloaded{ + [](typename iterator::Left &l) -> mapped_type & { + return l->second; + }, + [this](typename iterator::Right &) -> mapped_type & { + /* + * Do not use the iterator result, as it is a non-owning + * handle + */ + return static_cast(*this); + }}, + it.m_iterator); + } + else + { + bool const keyScalar = (key == RecordComponent::SCALAR); + if ((keyScalar && !Container::empty() && !scalar()) || + (scalar() && !keyScalar)) + throw error::WrongAPIUsage( + "A scalar component can not be contained at the same time as " + "one or more regular components."); + + if (keyScalar) + { + /* + * This activates the RecordComponent API of this object. + */ + T_RecordComponent::get(); + } + mapped_type &ret = keyScalar ? static_cast(*this) + : T_Container:: + operator[](key); + return ret; + } +} + +template +auto BaseRecord::operator[](key_type &&key) -> mapped_type & +{ + auto it = this->find(key); + if (it != this->end()) + { + return std::visit( + auxiliary::overloaded{ + [](typename iterator::Left &l) -> mapped_type & { + return l->second; + }, + [this](typename iterator::Right &) -> mapped_type & { + /* + * Do not use the iterator result, as it is a non-owning + * handle + */ + return static_cast(*this); + }}, + it.m_iterator); + } + else + { + bool const keyScalar = (key == RecordComponent::SCALAR); + if ((keyScalar && !Container::empty() && !scalar()) || + (scalar() && !keyScalar)) + throw error::WrongAPIUsage( + "A scalar component can not be contained at the same time as " + "one or more regular components."); + + if (keyScalar) + { + /* + * This activates the RecordComponent API of this object. + */ + T_RecordComponent::get(); + } + mapped_type &ret = keyScalar ? static_cast(*this) + : T_Container:: + operator[](std::move(key)); + return ret; + } +} + +template +auto BaseRecord::at(key_type const &key) -> mapped_type & +{ + return const_cast( + static_cast const *>(this)->at(key)); +} + +template +auto BaseRecord::at(key_type const &key) const -> mapped_type const & +{ + bool const keyScalar = (key == RecordComponent::SCALAR); + if (keyScalar) + { + if (!get().m_datasetDefined) + { + throw std::out_of_range( + "[at()] Requested scalar entry from non-scalar record."); + } + return static_cast(*this); + } + else + { + return T_Container::at(key); + } +} + +template +auto BaseRecord::erase(key_type const &key) -> size_type +{ + bool const keyScalar = (key == RecordComponent::SCALAR); + size_type res; + if (!keyScalar || (keyScalar && this->at(key).constant())) + res = Container::erase(key); + else + { + res = this->datasetDefined() ? 1 : 0; + eraseScalar(); + } + + if (keyScalar) + { + this->setWritten(false, Attributable::EnqueueAsynchronously::No); + this->writable().abstractFilePosition.reset(); + this->get().m_datasetDefined = false; + } + return res; +} + +template +auto BaseRecord::erase(iterator it) -> iterator +{ + return std::visit( + auxiliary::overloaded{ + [this](typename iterator::Left &left) { + return makeIterator(T_Container::erase(left)); + }, + [this](typename iterator::Right &) { + eraseScalar(); + return end(); + }}, + it.m_iterator); +} + +template +auto BaseRecord::empty() const noexcept -> bool +{ + return !scalar() && T_Container::empty(); +} + +template +auto BaseRecord::find(key_type const &key) -> iterator +{ + auto &r = get(); + if (r.m_datasetDefined) + { + if (key == RecordComponent::SCALAR) + { + return begin(); + } + else + { + return end(); + } + } + else if (key == RecordComponent::SCALAR) + { + return end(); + } + else + { + return makeIterator(r.m_container.find(key)); + } +} + +template +auto BaseRecord::find(key_type const &key) const -> const_iterator +{ + auto &r = get(); + if (r.m_datasetDefined) + { + if (key == RecordComponent::SCALAR) + { + return begin(); + } + else + { + return end(); + } + } + else if (key == RecordComponent::SCALAR) + { + return end(); + } + else + { + return makeIterator(r.m_container.find(key)); + } +} + +template +auto BaseRecord::count(key_type const &key) const -> size_type +{ + if (key == RecordComponent::SCALAR) + { + return get().m_datasetDefined ? 1 : 0; + } + else + { + return T_Container::count(key); + } +} + +template +auto BaseRecord::size() const -> size_type +{ + if (scalar()) + { + return 1; + } + else + { + return T_Container::size(); + } +} + +template +auto BaseRecord::clear() -> void +{ + if (Access::READ_ONLY == this->IOHandler()->m_frontendAccess) + throw std::runtime_error( + "Can not clear a container in a read-only Series."); + if (scalar()) + { + eraseScalar(); + } + else + { + T_Container::clear_unchecked(); + } +} + +namespace detail +{ + template + void verifyNonscalar(BaseRecord *self) + { + if (self->scalar()) + { + throw error::WrongAPIUsage(NO_SCALAR_INSERT); + } + } + +// Needed for clang-tidy +#define OPENPMD_APPLY_TEMPLATE(template_, type) template_ + +#define OPENPMD_INSTANTIATE(recordcomponenttype) \ + template void \ + verifyNonscalar( \ + BaseRecord *); + OPENPMD_FORALL_RECORDCOMPONENT_TYPES(OPENPMD_INSTANTIATE) +#undef OPENPMD_INSTANTIATE +#undef OPENPMD_APPLY_TEMPLATE +} // namespace detail + +template +auto BaseRecord::insert(value_type const &value) + -> std::pair +{ + detail::verifyNonscalar(this); + auto res = this->container().insert(value); + if (res.first->first == RecordComponent::SCALAR) + { + this->container().erase(res.first); + throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + } + return {makeIterator(std::move(res.first)), res.second}; +} + +template +auto BaseRecord::insert(value_type &&value) -> std::pair +{ + detail::verifyNonscalar(this); + auto res = this->container().insert(std::move(value)); + if (res.first->first == RecordComponent::SCALAR) + { + this->container().erase(res.first); + throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + } + return {makeIterator(std::move(res.first)), res.second}; +} + +template +auto BaseRecord::insert(const_iterator hint, value_type const &value) + -> iterator +{ + detail::verifyNonscalar(this); + auto base_hint = std::visit( + auxiliary::overloaded{ + [](typename const_iterator::Left left) { return left; }, + [this](typename const_iterator::Right) { + return static_cast const *>(this) + ->container() + .begin(); + }}, + hint.m_iterator); + auto res = this->container().insert(base_hint, value); + if (res->first == RecordComponent::SCALAR) + { + this->container().erase(res); + throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + } + return makeIterator(res); +} + +template +auto BaseRecord::insert(const_iterator hint, value_type &&value) + -> iterator +{ + detail::verifyNonscalar(this); + auto base_hint = std::visit( + auxiliary::overloaded{ + [](typename const_iterator::Left left) { return left; }, + [this](typename const_iterator::Right) { + return static_cast const *>(this) + ->container() + .begin(); + }}, + hint.m_iterator); + auto res = this->container().insert(base_hint, std::move(value)); + if (res->first == RecordComponent::SCALAR) + { + this->container().erase(res); + throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + } + return makeIterator(res); +} + +template +template +auto BaseRecord::insert(InputIt first, InputIt last) -> void +{ + detail::verifyNonscalar(this); + this->container().insert(first, last); + /* + * We skip this check as it changes the runtime of this call from + * O(last-first) to O(container().size()). + */ + // for (auto it = this->container().begin(); it != end; ++it) + // { + // if (it->first == RecordComponent::SCALAR) + // { + // this->container().erase(it); + // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + // } + // } +} + +template +auto BaseRecord::insert(std::initializer_list ilist) -> void +{ + detail::verifyNonscalar(this); + this->container().insert(std::move(ilist)); + /* + * We skip this check as it changes the runtime of this call from + * O(last-first) to O(container().size()). + */ + // for (auto it = this->container().begin(); it != end; ++it) + // { + // if (it->first == RecordComponent::SCALAR) + // { + // this->container().erase(it); + // throw error::WrongAPIUsage(detail::NO_SCALAR_INSERT); + // } + // } +} + +template +auto BaseRecord::swap(BaseRecord &other) -> void +{ + detail::verifyNonscalar(this); + detail::verifyNonscalar(&other); + this->container().swap(other.container()); +} + +template +auto BaseRecord::contains(key_type const &key) const -> bool +{ + if (scalar()) + { + return key == RecordComponent::SCALAR; + } + else + { + return T_Container::contains(key); + } +} + +template +inline unit_representations::AsArray BaseRecord::unitDimension() const +{ + return this->getAttribute("unitDimension") + .template get>(); +} + +template +inline bool BaseRecord::scalar() const +{ + return this->datasetDefined(); +} + +template +inline void BaseRecord::readBase() +{ + using DT = Datatype; + Parameter aRead; + + aRead.name = "unitDimension"; + this->IOHandler()->enqueue(IOTask(this, aRead)); + this->IOHandler()->flush(internal::defaultFlushParams); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional>(); + val.has_value()) + this->setAttribute("unitDimension", val.value()); + else + throw std::runtime_error( + "Unexpected Attribute datatype for 'unitDimension'"); + + aRead.name = "timeOffset"; + this->IOHandler()->enqueue(IOTask(this, aRead)); + this->IOHandler()->flush(internal::defaultFlushParams); + if (*aRead.dtype == DT::FLOAT) + this->setAttribute( + "timeOffset", + Attribute(Attribute::from_any, *aRead.m_resource).get()); + else if (*aRead.dtype == DT::DOUBLE) + this->setAttribute( + "timeOffset", + Attribute(Attribute::from_any, *aRead.m_resource).get()); + // conversion cast if a backend reports an integer type + else if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional(); + val.has_value()) + this->setAttribute("timeOffset", val.value()); + else + throw std::runtime_error( + "Unexpected Attribute datatype for 'timeOffset'"); +} + +template +inline void BaseRecord::flush( + std::string const &name, internal::FlushParams const &flushParams) +{ + if (!this->written() && this->empty() && !this->datasetDefined()) + throw std::runtime_error( + "A Record can not be written without any contained " + "RecordComponents: " + + name); + + /* + * Defensive programming. Normally, this error should yield as soon as + * possible. + */ + if (scalar() && !T_Container::empty()) + { + throw error::WrongAPIUsage( + "A scalar component can not be contained at the same time as " + "one or more regular components."); + } + + this->flush_impl(name, flushParams); + if (flushParams.flushLevel != FlushLevel::SkeletonOnly) + { + this->setDirty(false); + } + // flush_impl must take care to correctly set the dirty() flag so this + // method doesn't do it +} + +template +void BaseRecord::eraseScalar() +{ + if (this->written()) + { + Parameter dDelete; + dDelete.name = "."; + this->IOHandler()->enqueue(IOTask(this, dDelete)); + this->IOHandler()->flush(internal::defaultFlushParams); + } + auto &data = T_RecordComponent::get(); + data.reset(); + this->writable().abstractFilePosition.reset(); +} + +template +BaseRecord::~BaseRecord() = default; + +#define OPENPMD_INSTANTIATE(recordcomponenttype) \ + template class BaseRecord; + +OPENPMD_FORALL_RECORDCOMPONENT_TYPES(OPENPMD_INSTANTIATE) + +#undef OPENPMD_INSTANTIATE + +} // namespace openPMD diff --git a/src/backend/Container.cpp b/src/backend/Container.cpp new file mode 100644 index 0000000000..7d421ac1a9 --- /dev/null +++ b/src/backend/Container.cpp @@ -0,0 +1,56 @@ +/* Copyright 2017-2025 Fabian Koller and Franz Poeschel + * + * This file is part of openPMD-api. + * + * openPMD-api is free software: you can redistribute it and/or modify + * it under the terms of of either the GNU General Public License or + * the GNU Lesser General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * openPMD-api is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License and the GNU Lesser General Public License + * for more details. + * + * You should have received a copy of the GNU General Public License + * and the GNU Lesser General Public License along with openPMD-api. + * If not, see . + */ + +#include "openPMD/backend/ContainerImpl.tpp" + +#include "openPMD/Iteration.hpp" +#include "openPMD/Mesh.hpp" +#include "openPMD/ParticlePatches.hpp" +#include "openPMD/ParticleSpecies.hpp" +#include "openPMD/backend/Container.hpp" +#include "openPMD/backend/PatchRecordComponent.hpp" + +namespace openPMD +{ +#define OPENPMD_COMMA , +#define OPENPMD_INSTANTIATE(type) template class Container; + +OPENPMD_INSTANTIATE(Mesh) +OPENPMD_INSTANTIATE(MeshRecordComponent) +OPENPMD_INSTANTIATE(ParticlePatches) +OPENPMD_INSTANTIATE(ParticleSpecies) +OPENPMD_INSTANTIATE(PatchRecord) +OPENPMD_INSTANTIATE(PatchRecordComponent) +OPENPMD_INSTANTIATE(Record) +OPENPMD_INSTANTIATE(RecordComponent) +OPENPMD_INSTANTIATE(Iteration OPENPMD_COMMA Iteration::IterationIndex_t) +#undef OPENPMD_INSTANTIATE +#undef OPENPMD_COMMA + +namespace internal +{ + template class EraseStaleEntries; + template class EraseStaleEntries; + template class EraseStaleEntries>; + template class EraseStaleEntries>; +} // namespace internal + +} // namespace openPMD diff --git a/src/backend/MeshRecordComponent.cpp b/src/backend/MeshRecordComponent.cpp index ed50080757..827102ad32 100644 --- a/src/backend/MeshRecordComponent.cpp +++ b/src/backend/MeshRecordComponent.cpp @@ -44,7 +44,7 @@ void MeshRecordComponent::read() aRead.name = "position"; IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - Attribute a = Attribute(*aRead.resource); + Attribute a = Attribute(Attribute::from_any, *aRead.m_resource); if (*aRead.dtype == DT::VEC_FLOAT || *aRead.dtype == DT::FLOAT) setPosition(a.get >()); else if (*aRead.dtype == DT::VEC_DOUBLE || *aRead.dtype == DT::DOUBLE) @@ -62,7 +62,9 @@ void MeshRecordComponent::read() {}, "Unexpected Attribute datatype for 'position' (expected a vector " "of any floating point type, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); readBase(/* require_unit_si = */ true); } diff --git a/src/backend/PatchRecord.cpp b/src/backend/PatchRecord.cpp index 5d2b38d50f..2a6581e56d 100644 --- a/src/backend/PatchRecord.cpp +++ b/src/backend/PatchRecord.cpp @@ -65,8 +65,8 @@ void PatchRecord::read() IOHandler()->enqueue(IOTask(this, aRead)); IOHandler()->flush(internal::defaultFlushParams); - if (auto val = - Attribute(*aRead.resource).getOptional >(); + if (auto val = Attribute(Attribute::from_any, *aRead.m_resource) + .getOptional >(); val.has_value()) this->setAttribute("unitDimension", val.value()); else @@ -76,7 +76,9 @@ void PatchRecord::read() {}, "Unexpected Attribute datatype for 'unitDimension' (expected an " "array of seven floating point numbers, found " + - datatypeToString(Attribute(*aRead.resource).dtype) + ")"); + datatypeToString( + Attribute(Attribute::from_any, *aRead.m_resource).dtype) + + ")"); Parameter dList; IOHandler()->enqueue(IOTask(this, dList)); diff --git a/src/binding/python/Attributable.cpp b/src/binding/python/Attributable.cpp index 4107616556..07f0118f3c 100644 --- a/src/binding/python/Attributable.cpp +++ b/src/binding/python/Attributable.cpp @@ -22,6 +22,7 @@ #include "openPMD/DatatypeHelpers.hpp" #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attribute.hpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/binding/python/Common.hpp" #include "openPMD/binding/python/Numpy.hpp" @@ -617,7 +618,7 @@ void init_Attributable(py::module &m) "get_attribute", [](Attributable &attr, std::string const &key) { auto v = attr.getAttribute(key); - return v.getResource(); + return v.getVariant(); // TODO instead of returning lists, return all arrays (ndim > 0) // as numpy arrays? }) diff --git a/test/AuxiliaryTest.cpp b/test/AuxiliaryTest.cpp index e863dd1527..012bb25139 100644 --- a/test/AuxiliaryTest.cpp +++ b/test/AuxiliaryTest.cpp @@ -12,6 +12,8 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/ContainerImpl.tpp" +#include "openPMD/backend/Variant_internal.hpp" #include "openPMD/backend/Writable.hpp" #include "openPMD/config.hpp" @@ -174,7 +176,8 @@ struct structure : public TestHelper [[nodiscard]] std::string text() const { - return std::get(getAttribute("text").getResource()); + return std::get( + getAttribute("text").getVariant()); } structure &setText(std::string newText) { @@ -313,9 +316,9 @@ struct AttributedWidget : public TestHelper AttributedWidget() : TestHelper() {} - Attribute::resource get(std::string const &key) + attribute_types get(std::string const &key) { - return getAttribute(key).getResource(); + return getAttribute(key).getVariant(); } }; } // namespace openPMD::test @@ -363,15 +366,18 @@ struct Dotty : public TestHelper [[nodiscard]] int att1() const { - return std::get(getAttribute("att1").getResource()); + return std::get( + getAttribute("att1").getVariant()); } [[nodiscard]] double att2() const { - return std::get(getAttribute("att2").getResource()); + return std::get( + getAttribute("att2").getVariant()); } [[nodiscard]] std::string att3() const { - return std::get(getAttribute("att3").getResource()); + return std::get( + getAttribute("att3").getVariant()); } Dotty &setAtt1(int i) { diff --git a/test/CoreTest.cpp b/test/CoreTest.cpp index f87be576bb..70dc53e72d 100644 --- a/test/CoreTest.cpp +++ b/test/CoreTest.cpp @@ -1454,17 +1454,25 @@ TEST_CASE("DoConvert_single_value_to_vector", "[core]") { std::vector vector{0, 1, 2, 3, 4, 5, 6}; std::array arraydouble{{0, 1, 2, 3, 4, 5, 6}}; - std::array arrayint{{0, 1, 2, 3, 4, 5, 6}}; Attribute attr{vector}; - - // the following conversions should be possible REQUIRE(attr.get>() == arraydouble); + + /* + * The following test no longer works since we pulled the definition for + * Attribute::get() from public headers into object files. These are + * instantiated for a selected subset of datatypes (those used in the + * Datatype enum), hence this test will now result in a link-time error. + */ +#if 0 + std::array arrayint{{0, 1, 2, 3, 4, 5, 6}}; + the following conversions should be possible REQUIRE(attr.get>() == arrayint); REQUIRE_THROWS_WITH( (attr.get>()), Catch::Equals( "getCast: no vector to array conversion possible " "(wrong requested array size).")); +#endif REQUIRE( attr.get>() == std::vector{0, 1, 2, 3, 4, 5, 6});