Read dataset-specific configuration also in ADIOS2::openDataset

franzpoeschel · franzpoeschel · commit da3a45f00d3f · 2025-02-11T12:43:58.000+01:00
diff --git a/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp b/include/openPMD/IO/ADIOS/ADIOS2IOHandler.hpp
@@ -338,6 +338,10 @@ class ADIOS2IOHandlerImpl
     // use m_config
     std::optional<std::vector<ParameterizedOperator>> getOperators();
 
+    template <typename Parameter>
+    std::vector<ParameterizedOperator> getDatasetOperators(
+        Parameter const &, Writable *, std::string const &varName);
+
     std::string fileSuffix(bool verbose = true) const;
 
     /*
@@ -560,7 +564,9 @@ namespace detail
             ADIOS2IOHandlerImpl *impl,
             InvalidatableFile const &,
             std::string const &varName,
-            Parameter<Operation::OPEN_DATASET> &parameters);
+            Parameter<Operation::OPEN_DATASET> &parameters,
+            std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator> const
+                &operators);
 
         static constexpr char const *errorMsg = "ADIOS2: openDataset()";
     };
diff --git a/include/openPMD/IO/IOTask.hpp b/include/openPMD/IO/IOTask.hpp
@@ -102,6 +102,18 @@ struct OPENPMDAPI_EXPORT AbstractParameter
 
     virtual std::unique_ptr<AbstractParameter> to_heap() && = 0;
 
+    /** Warn about unused JSON paramters
+     *
+     * Template parameter so we don't have to include the JSON lib here.
+     * This function is useful for the createDataset() methods in,
+     * IOHandlerImpl's, so putting that here is the simplest way to make it
+     * available for them. */
+    template <typename TracingJSON>
+    static void warnUnusedParameters(
+        TracingJSON &,
+        std::string const &currentBackendName,
+        std::string const &warningMessage);
+
 protected:
     // avoid object slicing
     // by allow only child classes to use these things for defining their own
@@ -363,18 +375,6 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
     std::string options = "{}";
     std::optional<size_t> joinedDimension;
 
-    /** Warn about unused JSON paramters
-     *
-     * Template parameter so we don't have to include the JSON lib here.
-     * This function is useful for the createDataset() methods in,
-     * IOHandlerImpl's, so putting that here is the simplest way to make it
-     * available for them. */
-    template <typename TracingJSON>
-    static void warnUnusedParameters(
-        TracingJSON &,
-        std::string const &currentBackendName,
-        std::string const &warningMessage);
-
     template <typename TracingJSON>
     TracingJSON compileJSONConfig(
         Writable const *writable,
@@ -417,6 +417,12 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::OPEN_DATASET>
             new Parameter<Operation::OPEN_DATASET>(std::move(*this)));
     }
 
+    template <typename TracingJSON>
+    static TracingJSON compileJSONConfig(
+        Writable const *writable,
+        json::JsonMatcher &,
+        std::string const &backendName);
+
     std::string name = "";
     std::shared_ptr<Datatype> dtype = std::make_shared<Datatype>();
     std::shared_ptr<Extent> extent = std::make_shared<Extent>();
diff --git a/include/openPMD/backend/Writable.hpp b/include/openPMD/backend/Writable.hpp
@@ -104,6 +104,7 @@ class Writable final
     friend class Span;
     friend void debug::printDirty(Series const &);
     friend struct Parameter<Operation::CREATE_DATASET>;
+    friend struct Parameter<Operation::OPEN_DATASET>;
 
 private:
     Writable(internal::AttributableData *);
diff --git a/src/IO/ADIOS/ADIOS2IOHandler.cpp b/src/IO/ADIOS/ADIOS2IOHandler.cpp
@@ -371,6 +371,35 @@ ADIOS2IOHandlerImpl::getOperators()
     return getOperators(m_config);
 }
 
+template <typename Parameter>
+auto ADIOS2IOHandlerImpl::getDatasetOperators(
+    Parameter const &parameters, Writable *writable, std::string const &varName)
+    -> std::vector<ParameterizedOperator>
+{
+    std::vector<ParameterizedOperator> operators;
+    json::TracingJSON options =
+        parameters.template compileJSONConfig<json::ParsedConfig>(
+            writable, *m_handler->jsonMatcher, "adios2");
+    if (options.json().contains("adios2"))
+    {
+        json::TracingJSON datasetConfig(options["adios2"]);
+        auto datasetOperators = getOperators(datasetConfig);
+
+        operators = datasetOperators ? std::move(datasetOperators.value())
+                                     : defaultOperators;
+    }
+    else
+    {
+        operators = defaultOperators;
+    }
+    parameters.warnUnusedParameters(
+        options,
+        "adios2",
+        "Warning: parts of the backend configuration for ADIOS2 dataset '" +
+            varName + "' remain unused:\n");
+    return operators;
+}
+
 using AcceptedEndingsForEngine = std::map<std::string, std::string>;
 
 std::string ADIOS2IOHandlerImpl::fileSuffix(bool verbose) const
@@ -793,27 +822,8 @@ void ADIOS2IOHandlerImpl::createDataset(
         filePos->gd = GroupOrDataset::DATASET;
         auto const varName = nameOfVariable(writable);
 
-        std::vector<ParameterizedOperator> operators;
-        json::TracingJSON options =
-            parameters.compileJSONConfig<json::ParsedConfig>(
-                writable, *m_handler->jsonMatcher, "adios2");
-        if (options.json().contains("adios2"))
-        {
-            json::TracingJSON datasetConfig(options["adios2"]);
-            auto datasetOperators = getOperators(datasetConfig);
-
-            operators = datasetOperators ? std::move(datasetOperators.value())
-                                         : defaultOperators;
-        }
-        else
-        {
-            operators = defaultOperators;
-        }
-        parameters.warnUnusedParameters(
-            options,
-            "adios2",
-            "Warning: parts of the backend configuration for ADIOS2 dataset '" +
-                varName + "' remain unused:\n");
+        std::vector<ParameterizedOperator> operators =
+            getDatasetOperators(parameters, writable, varName);
 
         // cast from openPMD::Extent to adios2::Dims
         adios2::Dims shape(parameters.extent.begin(), parameters.extent.end());
@@ -1018,8 +1028,18 @@ void ADIOS2IOHandlerImpl::openDataset(
     auto &fileData = getFileData(file, IfFileNotOpen::ThrowError);
     *parameters.dtype =
         detail::fromADIOS2Type(fileData.m_IO.VariableType(varName));
+
+    /*
+     * Technically, the only reason to set read-time operators is for specifying
+     * decompression threads. This needs not happen at a per-dataset level.
+     * However, users may apply the same JSON/TOML config for writing and
+     * reading, so the dataset-specific configuration should still be explored
+     * here.
+     */
+    std::vector<ParameterizedOperator> operators =
+        getDatasetOperators(parameters, writable, varName);
     switchAdios2VariableType<detail::DatasetOpener>(
-        *parameters.dtype, this, file, varName, parameters);
+        *parameters.dtype, this, file, varName, parameters, operators);
     writable->written = true;
 }
 
@@ -2072,7 +2092,9 @@ namespace detail
         ADIOS2IOHandlerImpl *impl,
         InvalidatableFile const &file,
         const std::string &varName,
-        Parameter<Operation::OPEN_DATASET> &parameters)
+        Parameter<Operation::OPEN_DATASET> &parameters,
+        std::vector<ADIOS2IOHandlerImpl::ParameterizedOperator> const
+            &operators)
     {
         auto &fileData = impl->getFileData(
             file, ADIOS2IOHandlerImpl::IfFileNotOpen::ThrowError);
@@ -2086,7 +2108,7 @@ namespace detail
         }
 
         // Operators in reading needed e.g. for setting decompression threads
-        for (auto const &operation : impl->defaultOperators)
+        for (auto const &operation : operators)
         {
             if (operation.op)
             {
diff --git a/src/IO/IOTask.cpp b/src/IO/IOTask.cpp
@@ -24,6 +24,7 @@
 #include "openPMD/backend/Attributable.hpp"
 
 #include <iostream> // std::cerr
+#include <utility>
 
 namespace openPMD
 {
@@ -33,8 +34,7 @@ Writable *getWritable(Attributable *a)
 }
 
 template <>
-void Parameter<Operation::CREATE_DATASET>::warnUnusedParameters<
-    json::TracingJSON>(
+void AbstractParameter::warnUnusedParameters<json::TracingJSON>(
     json::TracingJSON &config,
     std::string const &currentBackendName,
     std::string const &warningMessage)
@@ -75,24 +75,55 @@ void Parameter<Operation::CREATE_DATASET>::warnUnusedParameters<
     }
 }
 
+namespace
+{
+    template <typename Functor>
+    json::ParsedConfig doCompileJSONConfig(
+        Attributable const &attri,
+        json::JsonMatcher &jsonMatcher,
+        std::string const &backendName,
+        Functor &&transformResult)
+    {
+        auto path = attri.myPath().openPMDPath();
+        auto base_config = jsonMatcher.get(path, backendName);
+        json::ParsedConfig res{
+            std::move(base_config.config), base_config.originallySpecifiedAs};
+        std::forward<Functor>(transformResult)(res);
+        return res;
+    }
+} // namespace
+
 template <>
 json::ParsedConfig Parameter<Operation::CREATE_DATASET>::compileJSONConfig(
     Writable const *writable,
     json::JsonMatcher &jsonMatcher,
     std::string const &backendName) const
 {
     auto attri = writable->attributable->asInternalCopyOf<Attributable>();
-    auto path = attri.myPath().openPMDPath();
-    auto base_config = jsonMatcher.get(path, backendName);
-    auto manual_config =
-        json::parseOptions(options, /* considerFiles = */ false);
-    json::merge_internal(
-        base_config.config, manual_config.config, /* do_prune = */ true);
-    return json::ParsedConfig{
-        std::move(base_config.config),
-        (options.empty() || options == "{}")
-            ? base_config.originallySpecifiedAs
-            : manual_config.originallySpecifiedAs};
+    return doCompileJSONConfig(
+        attri, jsonMatcher, backendName, [&](json::ParsedConfig &base_config) {
+            auto manual_config =
+                json::parseOptions(options, /* considerFiles = */ false);
+            json::merge_internal(
+                base_config.config,
+                manual_config.config,
+                /* do_prune = */ true);
+            base_config.originallySpecifiedAs =
+                (options.empty() || options == "{}")
+                ? base_config.originallySpecifiedAs
+                : manual_config.originallySpecifiedAs;
+        });
+}
+
+template <>
+json::ParsedConfig Parameter<Operation::OPEN_DATASET>::compileJSONConfig(
+    Writable const *writable,
+    json::JsonMatcher &jsonMatcher,
+    std::string const &backendName)
+{
+    auto attri = writable->attributable->asInternalCopyOf<Attributable>();
+    return doCompileJSONConfig(
+        attri, jsonMatcher, backendName, [](auto const &) {});
 }
 
 namespace internal
