Initialize Series attributes and datasets from template

Author: franzpoeschel

CMakeLists.txt

@@ -460,6 +460,7 @@ set(CORE_SOURCE
         src/auxiliary/Date.cpp
         src/auxiliary/Filesystem.cpp
         src/auxiliary/JSON.cpp
+        src/auxiliary/TemplateFile.cpp
         src/backend/Attributable.cpp
         src/backend/BaseRecordComponent.cpp
         src/backend/Container.cpp

include/openPMD/Iteration.hpp

@@ -115,6 +115,15 @@ namespace internal
          * alone.
          */
         std::optional<std::string> m_overrideFilebasedFilename{};
+
+        enum TernaryBool
+        {
+            Undefined,
+            True,
+            False
+        };
+        TernaryBool hasMeshes = TernaryBool::Undefined;
+        TernaryBool hasParticles = TernaryBool::Undefined;
     };
 } // namespace internal
 /** @brief  Logical compilation of data from one snapshot (e.g. a single
@@ -239,6 +248,9 @@ class Iteration : public Attributable
     Container<Mesh> meshes{};
     Container<ParticleSpecies> particles{}; // particleSpecies?
 
+    bool hasMeshes() const;
+    bool hasParticles() const;
+
     virtual ~Iteration() = default;
 
 private:

include/openPMD/auxiliary/TemplateFile.hpp

@@ -0,0 +1,10 @@
+#pragma once
+
+#include "openPMD/Series.hpp"
+
+namespace openPMD::auxiliary
+{
+// @todo replace uint64_t with proper type after merging #1285
+Series &initializeFromTemplate(
+    Series &initializeMe, Series const &fromTemplate, uint64_t iteration);
+} // namespace openPMD::auxiliary

src/Iteration.cpp

@@ -36,6 +36,50 @@ namespace openPMD
 using internal::CloseStatus;
 using internal::DeferredParseAccess;
 
+bool Iteration::hasMeshes() const
+{
+    /*
+     * Currently defined at the Series level, but might be defined at the
+     * Iteration level in next standard iterations.
+     * Hence an Iteration:: method.
+     */
+
+    switch (get().hasMeshes)
+    {
+    case internal::IterationData::TernaryBool::True:
+        return true;
+    case internal::IterationData::TernaryBool::False:
+        return false;
+    case internal::IterationData::TernaryBool::Undefined: {
+        Series s = retrieveSeries();
+        return !meshes.empty() || s.containsAttribute("meshesPath");
+    };
+    }
+    throw std::runtime_error("Unreachable!");
+}
+
+bool Iteration::hasParticles() const
+{
+    /*
+     * Currently defined at the Series level, but might be defined at the
+     * Iteration level in next standard iterations.
+     * Hence an Iteration:: method.
+     */
+
+    switch (get().hasParticles)
+    {
+    case internal::IterationData::TernaryBool::True:
+        return true;
+    case internal::IterationData::TernaryBool::False:
+        return false;
+    case internal::IterationData::TernaryBool::Undefined: {
+        Series s = retrieveSeries();
+        return !particles.empty() || s.containsAttribute("particlesPath");
+    };
+    }
+    throw std::runtime_error("Unreachable!");
+}
+
 Iteration::Iteration() : Attributable{nullptr}
 {
     Attributable::setData(m_iterationData);
@@ -318,7 +362,7 @@ void Iteration::flush(internal::FlushParams const &flushParams)
          * meshesPath and particlesPath are stored there */
         Series s = retrieveSeries();
 
-        if (!meshes.empty() || s.containsAttribute("meshesPath"))
+        if (hasMeshes())
         {
             if (!s.containsAttribute("meshesPath"))
             {
@@ -334,7 +378,7 @@ void Iteration::flush(internal::FlushParams const &flushParams)
             meshes.dirty() = false;
         }
 
-        if (!particles.empty() || s.containsAttribute("particlesPath"))
+        if (hasParticles())
         {
             if (!s.containsAttribute("particlesPath"))
             {
@@ -496,6 +540,12 @@ void Iteration::read_impl(std::string const &groupPath)
         hasMeshes = s.containsAttribute("meshesPath");
         hasParticles = s.containsAttribute("particlesPath");
     }
+    {
+        using TB = internal::IterationData::TernaryBool;
+        auto &data = get();
+        data.hasMeshes = hasMeshes ? TB::True : TB::False;
+        data.hasParticles = hasParticles ? TB::True : TB::False;
+    }
 
     if (hasMeshes)
     {

src/auxiliary/TemplateFile.cpp

@@ -0,0 +1,173 @@
+#include "openPMD/auxiliary/TemplateFile.hpp"
+#include "openPMD/DatatypeHelpers.hpp"
+
+#include <iostream>
+
+namespace openPMD::auxiliary
+{
+namespace
+{
+    // Some forward declarations
+    template <typename T>
+    void initializeFromTemplate(
+        Container<T> &initializeMe, Container<T> const &fromTemplate);
+
+    struct SetAttribute
+    {
+        template <typename T>
+        static void
+        call(Attributable &object, std::string const &name, Attribute attr)
+        {
+            object.setAttribute(name, attr.get<T>());
+        }
+
+        template <unsigned n>
+        static void call(Attributable &, std::string const &name, Attribute)
+        {
+            std::cerr << "Unknown datatype for template attribute '" << name
+                      << "'. Will skip it." << std::endl;
+        }
+    };
+
+    void copyAttributes(
+        Attributable &target,
+        Attributable const &source,
+        std::vector<std::string> ignore = {})
+    {
+        auto shouldBeIgnored = [&ignore](std::string const &attrName) {
+            // `ignore` is empty by default and normally has only a handful of
+            // entries otherwise.
+            // So just use linear search.
+            for (auto const &ignored : ignore)
+            {
+                if (attrName == ignored)
+                {
+                    return true;
+                }
+            }
+            return false;
+        };
+
+        for (auto const &attrName : source.attributes())
+        {
+            if (shouldBeIgnored(attrName))
+            {
+                continue;
+            }
+            auto attr = source.getAttribute(attrName);
+            auto dtype = attr.dtype;
+            switchType<SetAttribute>(dtype, target, attrName, std::move(attr));
+        }
+    }
+
+    void initializeFromTemplate(
+        BaseRecordComponent &initializeMe,
+        BaseRecordComponent const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate);
+    }
+
+    void initializeFromTemplate(
+        RecordComponent &initializeMe, RecordComponent const &fromTemplate)
+    {
+        if (fromTemplate.getDatatype() != Datatype::UNDEFINED)
+        {
+            initializeMe.resetDataset(
+                Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()});
+        }
+        initializeFromTemplate(
+            static_cast<BaseRecordComponent &>(initializeMe),
+            static_cast<BaseRecordComponent const &>(fromTemplate));
+    }
+
+    void initializeFromTemplate(
+        PatchRecordComponent &initializeMe,
+        PatchRecordComponent const &fromTemplate)
+    {
+        if (fromTemplate.getDatatype() != Datatype::UNDEFINED)
+        {
+            initializeMe.resetDataset(
+                Dataset{fromTemplate.getDatatype(), fromTemplate.getExtent()});
+        }
+        initializeFromTemplate(
+            static_cast<BaseRecordComponent &>(initializeMe),
+            static_cast<BaseRecordComponent const &>(fromTemplate));
+    }
+
+    void initializeFromTemplate(
+        ParticleSpecies &initializeMe, ParticleSpecies const &fromTemplate)
+    {
+        if (!fromTemplate.particlePatches.empty())
+        {
+            initializeFromTemplate(
+                static_cast<Container<PatchRecord> &>(
+                    initializeMe.particlePatches),
+                static_cast<Container<PatchRecord> const &>(
+                    fromTemplate.particlePatches));
+        }
+        initializeFromTemplate(
+            static_cast<Container<Record> &>(initializeMe),
+            static_cast<Container<Record> const &>(fromTemplate));
+    }
+
+    template <typename T>
+    void initializeFromTemplate(
+        Container<T> &initializeMe, Container<T> const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate);
+        for (auto const &pair : fromTemplate)
+        {
+            initializeFromTemplate(initializeMe[pair.first], pair.second);
+        }
+    }
+
+    void initializeFromTemplate(
+        Iteration &initializeMe, Iteration const &fromTemplate)
+    {
+        copyAttributes(initializeMe, fromTemplate, {"snapshot"});
+        if (fromTemplate.hasMeshes())
+        {
+            initializeFromTemplate(initializeMe.meshes, fromTemplate.meshes);
+        }
+        if (fromTemplate.hasParticles())
+        {
+            initializeFromTemplate(
+                initializeMe.particles, fromTemplate.particles);
+        }
+    }
+} // namespace
+
+Series &initializeFromTemplate(
+    Series &initializeMe, Series const &fromTemplate, uint64_t iteration)
+{
+    if (!initializeMe.containsAttribute("from_template"))
+    {
+        copyAttributes(
+            initializeMe,
+            fromTemplate,
+            {"basePath", "iterationEncoding", "iterationFormat", "openPMD"});
+        initializeMe.setAttribute("from_template", fromTemplate.name());
+    }
+
+    uint64_t sourceIteration = iteration;
+    if (!fromTemplate.iterations.contains(sourceIteration))
+    {
+        if (fromTemplate.iterations.empty())
+        {
+            std::cerr << "Template file has no iterations, will only fill in "
+                         "global attributes."
+                      << std::endl;
+            return initializeMe;
+        }
+        else
+        {
+            sourceIteration = fromTemplate.iterations.begin()->first;
+        }
+    }
+
+    initializeFromTemplate(
+        initializeMe.iterations[iteration],
+        fromTemplate.iterations.at(sourceIteration));
+    return initializeMe;
+}
+} // namespace openPMD::auxiliary

test/SerialIOTest.cpp

@@ -10,6 +10,7 @@
 #include "openPMD/auxiliary/Environment.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
+#include "openPMD/auxiliary/TemplateFile.hpp"
 #include "openPMD/openPMD.hpp"
 
 #include <catch2/catch.hpp>
@@ -1461,6 +1462,12 @@ inline void dtype_test(
 
     if (activateTemplateMode.has_value())
     {
+        Series out(
+            "../samples/dtype_test_from_template." + backend,
+            Access::CREATE,
+            activateTemplateMode.value());
+        auxiliary::initializeFromTemplate(out, s, 1000);
+        out.flush();
         return;
     }
     // same implementation types (not necessary aliases) detection