diff --git a/CMakeLists.txt b/CMakeLists.txt index f2dcc98f82..8290227be6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -426,6 +426,11 @@ set(CORE_SOURCE src/backend/PatchRecord.cpp src/backend/PatchRecordComponent.cpp src/backend/Writable.cpp + src/backend/scientific_defaults/ScientificDefaults.cpp + src/backend/scientific_defaults/ScientificDefaults_auxiliary.cpp + src/backend/scientific_defaults/ProcessParsedAttribute.cpp + src/backend/scientific_defaults/AttributeReader.cpp + src/backend/scientific_defaults/ConfigAttribute.cpp src/auxiliary/OneDimensionalBlockSlicer.cpp src/helper/list_series.cpp src/snapshots/ContainerImpls.cpp diff --git a/include/openPMD/IO/AbstractIOHandler.hpp b/include/openPMD/IO/AbstractIOHandler.hpp index 9b7735b5ba..b54b661c78 100644 --- a/include/openPMD/IO/AbstractIOHandler.hpp +++ b/include/openPMD/IO/AbstractIOHandler.hpp @@ -26,6 +26,7 @@ #include "openPMD/IterationEncoding.hpp" #include "openPMD/config.hpp" #include "openPMD/version.hpp" +#include #if openPMD_HAVE_MPI #include @@ -81,6 +82,66 @@ enum class FlushLevel CreateOrOpenFiles }; +std::ostream &operator<<(std::ostream &, FlushLevel); + +namespace flush_level +{ + inline constexpr auto global_flushpoint(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + // same as global_flushpoint for now, but we will soon introduce + // immediate_flush + inline constexpr auto write_datasets(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + return true; + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto write_attributes(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + return true; + case FlushLevel::SkeletonOnly: + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } + inline constexpr auto flush_hierarchy(FlushLevel fl) + { + switch (fl) + { + case FlushLevel::UserFlush: + case FlushLevel::InternalFlush: + case FlushLevel::SkeletonOnly: + return true; + case FlushLevel::CreateOrOpenFiles: + return false; + } + return false; // unreachable + } +} // namespace flush_level + enum class OpenpmdStandard { v_1_0_0, @@ -121,6 +182,7 @@ namespace internal * To be used for reading */ FlushParams const defaultFlushParams{}; + FlushParams const publicFlush{FlushLevel::UserFlush}; struct ParsedFlushParams; diff --git a/include/openPMD/IO/AbstractIOHandlerImpl.hpp b/include/openPMD/IO/AbstractIOHandlerImpl.hpp index d45ce1bdcc..fdb8af3599 100644 --- a/include/openPMD/IO/AbstractIOHandlerImpl.hpp +++ b/include/openPMD/IO/AbstractIOHandlerImpl.hpp @@ -39,7 +39,7 @@ class AbstractIOHandlerImpl virtual ~AbstractIOHandlerImpl() = default; - std::future flush(); + std::future flush(FlushLevel); /** * Close the file corresponding with the writable and release file handles. diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp index 25e0d6ad54..a8efab571e 100644 --- a/include/openPMD/IO/IOTask.hpp +++ b/include/openPMD/IO/IOTask.hpp @@ -35,6 +35,7 @@ #include #include #include +#include #include #include #include @@ -89,6 +90,8 @@ OPENPMDAPI_EXPORT_ENUM_CLASS(Operation){ }; // note: if you change the enum members here, please update // docs/source/dev/design.rst +std::ostream &operator<<(std::ostream &os, Operation op); + namespace internal { /* diff --git a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp index 6df0c60ced..3e0758aee2 100644 --- a/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp +++ b/include/openPMD/IO/JSON/JSONIOHandlerImpl.hpp @@ -241,7 +241,7 @@ class JSONIOHandlerImpl : public AbstractIOHandlerImpl void touch(Writable *, Parameter const &) override; - std::future flush(); + std::future flush(internal::ParsedFlushParams ¶ms); private: #if openPMD_HAVE_MPI diff --git a/include/openPMD/Iteration.hpp b/include/openPMD/Iteration.hpp index e2a4e6b1a5..e281686855 100644 --- a/include/openPMD/Iteration.hpp +++ b/include/openPMD/Iteration.hpp @@ -27,6 +27,8 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -145,13 +147,28 @@ namespace internal std::optional m_deferredParseAccess{}; }; } // namespace internal + +class Meshes : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; + +class Particles : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; + /** @brief Logical compilation of data from one snapshot (e.g. a single * simulation cycle). * * @see * https://github.com/openPMD/openPMD-standard/blob/latest/STANDARD.md#required-attributes-for-the-basepath */ -class Iteration : public Attributable +class Iteration + : public Attributable + , internal::ScientificDefaults { template friend class Container; @@ -164,6 +181,8 @@ class Iteration : public Attributable friend class StatefulSnapshotsContainer; template friend struct traits::GenerationPolicy; + friend class internal::ScientificDefaults; + friend class Attributable; public: Iteration(Iteration const &) = default; @@ -279,8 +298,10 @@ class Iteration : public Attributable [[deprecated("This attribute is no longer set by the openPMD-api.")]] bool closedByWriter() const; - Container meshes{}; - Container particles{}; // particleSpecies? + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + + Meshes meshes{}; + Particles particles{}; virtual ~Iteration() = default; @@ -442,7 +463,7 @@ class Iteration : public Attributable * * @param w The Writable representing the parent. */ - virtual void linkHierarchy(Writable &w); + void linkHierarchy(Writable &w) override; /** * @brief Access an iteration in read mode that has potentially not been @@ -450,6 +471,9 @@ class Iteration : public Attributable * */ void runDeferredParseAccess(); + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Iteration namespace traits diff --git a/include/openPMD/Mesh.hpp b/include/openPMD/Mesh.hpp index d0bf81ddef..06885d4558 100644 --- a/include/openPMD/Mesh.hpp +++ b/include/openPMD/Mesh.hpp @@ -23,7 +23,9 @@ #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/Container.hpp" #include "openPMD/backend/MeshRecordComponent.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -41,6 +43,8 @@ class Mesh : public BaseRecord { friend class Container; friend class Iteration; + friend class internal::ScientificDefaults; + friend class Attributable; public: Mesh(Mesh const &) = default; @@ -322,12 +326,18 @@ class Mesh : public BaseRecord typename = std::enable_if_t::value>> Mesh &setTimeOffset(T timeOffset); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: Mesh(); void flush_impl(std::string const &, internal::FlushParams const &) override; void read(); + auto retrieveDimensionality() const -> uint64_t; + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Mesh template diff --git a/include/openPMD/ParticlePatches.hpp b/include/openPMD/ParticlePatches.hpp index f3c4c0b943..ffe539d5f1 100644 --- a/include/openPMD/ParticlePatches.hpp +++ b/include/openPMD/ParticlePatches.hpp @@ -38,6 +38,8 @@ class ParticlePatches : public Container size_t numPatches() const; ~ParticlePatches() override = default; + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: ParticlePatches() = default; void read(); diff --git a/include/openPMD/ParticleSpecies.hpp b/include/openPMD/ParticleSpecies.hpp index 7309afddef..9171a2d85e 100644 --- a/include/openPMD/ParticleSpecies.hpp +++ b/include/openPMD/ParticleSpecies.hpp @@ -24,23 +24,31 @@ #include "openPMD/Record.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include namespace openPMD { -class ParticleSpecies : public Container +class ParticleSpecies + : public Container + , internal::ScientificDefaults { friend class Container; friend class Container; friend class Iteration; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; + friend class Attributable; public: ParticlePatches particlePatches; + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: ParticleSpecies(); @@ -53,6 +61,9 @@ class ParticleSpecies : public Container { return m_containerData; } + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; namespace traits diff --git a/include/openPMD/Record.hpp b/include/openPMD/Record.hpp index 2bb070365f..e7ffff5ac2 100644 --- a/include/openPMD/Record.hpp +++ b/include/openPMD/Record.hpp @@ -23,6 +23,7 @@ #include "openPMD/RecordComponent.hpp" #include "openPMD/UnitDimension.hpp" #include "openPMD/backend/BaseRecord.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" #include #include @@ -34,6 +35,7 @@ class Record : public BaseRecord friend class Container; friend class Iteration; friend class ParticleSpecies; + friend class internal::ScientificDefaults; public: Record(Record const &) = default; @@ -48,6 +50,8 @@ class Record : public BaseRecord template Record &setTimeOffset(T); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + private: Record(); @@ -55,6 +59,9 @@ class Record : public BaseRecord flush_impl(std::string const &, internal::FlushParams const &) override; [[nodiscard]] internal::HomogenizeExtents read(); + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // Record template diff --git a/include/openPMD/RecordComponent.hpp b/include/openPMD/RecordComponent.hpp index d06b4213f4..d01a183679 100644 --- a/include/openPMD/RecordComponent.hpp +++ b/include/openPMD/RecordComponent.hpp @@ -27,6 +27,8 @@ #include "openPMD/auxiliary/UniquePtr.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/BaseRecordComponent.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" +#include "openPMD/backend/scientific_defaults/ScientificDefaults.hpp" // comment to prevent this include from being moved by clang-format #include "openPMD/DatatypeMacros.hpp" @@ -110,7 +112,9 @@ namespace internal template class BaseRecord; -class RecordComponent : public BaseRecordComponent +class RecordComponent + : public BaseRecordComponent + , protected internal::ScientificDefaults { template friend class Container; @@ -128,6 +132,8 @@ class RecordComponent : public BaseRecordComponent friend class MeshRecordComponent; template friend T &internal::makeOwning(T &self, Series); + friend class internal::ScientificDefaults; + friend class Attributable; public: enum class Allocation @@ -482,11 +488,13 @@ class RecordComponent : public BaseRecordComponent auto visit(Args &&...args) -> decltype(Visitor::template call( std::declval(), std::forward(args)...)); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + static constexpr char const *const SCALAR = "\vScalar"; protected: void flush(std::string const &, internal::FlushParams const &); - void read(bool require_unit_si); + void read(); private: /** @@ -534,12 +542,15 @@ OPENPMD_protected BaseRecordComponent::setData(m_recordComponentData); } - void readBase(bool require_unit_si); + void readBase(); template void verifyChunk(Offset const &, Extent const &) const; void verifyChunk(Datatype, Offset const &, Extent const &) const; + +protected: + void scientificDefaults_impl(bool write, OpenpmdStandard) override; }; // RecordComponent namespace internal diff --git a/include/openPMD/RecordComponent.tpp b/include/openPMD/RecordComponent.tpp index b796ab1a93..523f4f1e41 100644 --- a/include/openPMD/RecordComponent.tpp +++ b/include/openPMD/RecordComponent.tpp @@ -90,6 +90,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) { size *= ext; } + /* * Flushing the skeleton does not create datasets, * so we might need to do it now. @@ -129,7 +130,7 @@ RecordComponent::storeChunk(Offset o, Extent e, F &&createBuffer) * actual data yet. */ seriesFlush_impl( - {FlushLevel::SkeletonOnly}); + {FlushLevel::SkeletonOnly}, /*flush_io_handler=*/false); Parameter dCreate(rc.m_dataset.value()); dCreate.name = Attributable::get().m_writable.ownKeyWithinParent; IOHandler()->enqueue(IOTask(this, dCreate)); diff --git a/include/openPMD/Series.hpp b/include/openPMD/Series.hpp index 603e540c2b..2cb82d3600 100644 --- a/include/openPMD/Series.hpp +++ b/include/openPMD/Series.hpp @@ -31,6 +31,7 @@ #include "openPMD/auxiliary/Variant.hpp" #include "openPMD/backend/Attributable.hpp" #include "openPMD/backend/Container.hpp" +#include "openPMD/backend/HierarchyVisitor.hpp" #include "openPMD/backend/ParsePreference.hpp" #include "openPMD/config.hpp" #include "openPMD/snapshots/Snapshots.hpp" @@ -64,7 +65,12 @@ namespace openPMD class ReadIterations; class StatefulIterator; class Series; -class Series; + +class Iterations : public Container +{ +public: + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; +}; namespace internal { @@ -100,8 +106,8 @@ namespace internal SeriesData &operator=(SeriesData &&) = delete; using IterationIndex_t = Iteration::IterationIndex_t; - using IterationsContainer_t = Container; - IterationsContainer_t iterations{}; + using IterationsContainer_t = Iterations; + Iterations iterations{}; /** * Series::readIterations() returns an iterator type that modifies the @@ -385,7 +391,7 @@ class Series : public Attributable * Type for a container of Iterations indexed by IterationIndex_t. */ using IterationsContainer_t = internal::SeriesData::IterationsContainer_t; - IterationsContainer_t iterations; + Iterations iterations; /** * @brief Is this a usable Series object? @@ -778,6 +784,8 @@ class Series : public Attributable */ void close(); + void visitHierarchy(HierarchyVisitor &v, bool recursive) override; + /** * This overrides Attributable::iterationFlush() which will fail on Series. */ @@ -898,9 +906,7 @@ OPENPMD_private iterations_iterator end, internal::FlushParams const &flushParams, bool flushIOHandler = true); - void flushMeshesPath(); - void flushParticlesPath(); - void flushRankTable(); + void flushRankTable(FlushLevel); /* Parameter `read_only_this_single_iteration` used for reopening an * Iteration after closing it. */ @@ -984,7 +990,7 @@ OPENPMD_private * * @param doFlush If true, flush the IO handler. */ - void flushStep(bool doFlush); + void flushStep(bool doFlush, FlushLevel l); /* * setIterationEncoding() should only be called by users of our public API, diff --git a/include/openPMD/auxiliary/TypeTraits.hpp b/include/openPMD/auxiliary/TypeTraits.hpp index faf2b29472..e373873a4f 100644 --- a/include/openPMD/auxiliary/TypeTraits.hpp +++ b/include/openPMD/auxiliary/TypeTraits.hpp @@ -124,6 +124,22 @@ namespace detail { constexpr static bool value = true; }; + + template + struct ScalarType + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; + template + struct ScalarType> + { + using type = T; + }; } // namespace detail template @@ -141,6 +157,12 @@ using IsPointer_t = typename detail::IsPointer::type; template inline constexpr bool IsChar_v = detail::IsChar::value; +template +using ScalarType_t = typename detail::ScalarType::type; + +template +using VectorType_t = std::vector>; + /** Emulate in the C++ concept ContiguousContainer * * Users can implement this trait for a type to signal it can be used as @@ -203,6 +225,34 @@ namespace detail // little trick to avoid trailing commas in the macro expansions below template using variant_tail_t = std::variant; + + template