First attempt: Dataset-specific configuration

Author: franzpoeschel

examples/13_write_dynamic_configuration.cpp

@@ -34,14 +34,11 @@ int main()
 # be passed by adding an at-sign `@` in front of the path
 # The format will then be recognized by filename extension, i.e. .json or .toml
 
-backend = "adios2"
+backend = "hdf5"
 iteration_encoding = "group_based"
 # The following is only relevant in read mode
 defer_iteration_parsing = true
 
-[adios1.dataset]
-transform = "blosc:compressor=zlib,shuffle=bit,lvl=5;nometa"
-
 [adios2.engine]
 type = "bp4"
 
@@ -60,13 +57,18 @@ parameters.clevel = 5
 # type = "some other parameter"
 # # ...
 
-[hdf5.dataset]
-chunks = "auto"
+[[hdf5.dataset]]
+cfg.chunks = "auto"
+
+[[hdf5.dataset]]
+select = "particles/e/.*"
+cfg.chunks = [10]
+cfg.chornks = []
 )END";
 
     // open file for writing
     Series series =
-        Series("../samples/dynamicConfig.bp", Access::CREATE, defaults);
+        Series("../samples/dynamicConfig.h5", Access::CREATE, defaults);
 
     Datatype datatype = determineDatatype<position_t>();
     constexpr unsigned long length = 10ul;
@@ -103,11 +105,6 @@ chunks = "auto"
         std::string const differentCompressionSettings = R"END(
 {
   "resizable": true,
-  "adios1": {
-    "dataset": {
-      "transform": "blosc:compressor=zlib,shuffle=bit,lvl=1;nometa"
-    }
-  },
   "adios2": {
     "dataset": {
       "operators": [

include/openPMD/IO/IOTask.hpp

@@ -42,6 +42,10 @@ namespace openPMD
 {
 class Attributable;
 class Writable;
+namespace json
+{
+    class JsonMatcher;
+}
 
 Writable *getWritable(Attributable *);
 
@@ -355,6 +359,10 @@ struct OPENPMDAPI_EXPORT Parameter<Operation::CREATE_DATASET>
         TracingJSON &,
         std::string const &currentBackendName,
         std::string const &warningMessage);
+
+    template <typename TracingJSON>
+    TracingJSON
+    compileJSONConfig(Writable const *writable, json::JsonMatcher &) const;
 };
 
 template <>

include/openPMD/auxiliary/JSONMatcher.hpp

@@ -32,12 +32,7 @@ struct Pattern
     std::regex pattern;
     nlohmann::json config;
 
-    Pattern(std::string const &pattern_in, nlohmann::json config_in)
-        // we construct the patterns once and use them often, so let's ask for
-        // some optimization
-        : pattern{pattern_in, std::regex_constants::egrep | std::regex_constants::optimize}
-        , config{std::move(config_in)}
-    {}
+    Pattern(std::string const &pattern_in, nlohmann::json config_in);
 };
 
 /**

include/openPMD/backend/Attributable.hpp

@@ -157,6 +157,7 @@ class Attributable
     friend class WriteIterations;
     friend class internal::RecordComponentData;
     friend void debug::printDirty(Series const &);
+    friend class internal::AttributableData;
 
 protected:
     // tag for internal constructor

include/openPMD/backend/Writable.hpp

@@ -101,6 +101,7 @@ class Writable final
     template <typename>
     friend class Span;
     friend void debug::printDirty(Series const &);
+    friend struct Parameter<Operation::CREATE_DATASET>;
 
 private:
     Writable(internal::AttributableData *);

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -31,6 +31,7 @@
 #include "openPMD/auxiliary/Environment.hpp"
 #include "openPMD/auxiliary/Filesystem.hpp"
 #include "openPMD/auxiliary/JSONMatcher.hpp"
+#include "openPMD/auxiliary/JSON_internal.hpp"
 #include "openPMD/auxiliary/Mpi.hpp"
 #include "openPMD/auxiliary/StringManip.hpp"
 #include "openPMD/auxiliary/TypeTraits.hpp"
@@ -755,7 +756,8 @@ void ADIOS2IOHandlerImpl::createDataset(
 
         std::vector<ParameterizedOperator> operators;
         json::TracingJSON options =
-            json::parseOptions(parameters.options, /* considerFiles = */ false);
+            parameters.compileJSONConfig<json::ParsedConfig>(
+                writable, *m_handler->jsonMatcher);
         if (options.json().contains("adios2"))
         {
             json::TracingJSON datasetConfig(options["adios2"]);

src/IO/HDF5/HDF5IOHandler.cpp

@@ -496,8 +496,9 @@ void HDF5IOHandlerImpl::createDataset(
                     nlohmann::json::parse(mask_for_global_conf));
             }
             auto const &buffered_config = *m_buffered_dataset_config;
-            auto parsed_config = json::parseOptions(
-                parameters.options, /* considerFiles = */ false);
+            auto parsed_config =
+                parameters.compileJSONConfig<json::ParsedConfig>(
+                    writable, *m_handler->jsonMatcher);
             if (auto hdf5_config_it = parsed_config.config.find("hdf5");
                 hdf5_config_it != parsed_config.config.end())
             {

src/IO/IOTask.cpp

@@ -19,6 +19,7 @@
  * If not, see <http://www.gnu.org/licenses/>.
  */
 #include "openPMD/IO/IOTask.hpp"
+#include "openPMD/auxiliary/JSONMatcher.hpp"
 #include "openPMD/auxiliary/JSON_internal.hpp"
 #include "openPMD/backend/Attributable.hpp"
 
@@ -73,6 +74,23 @@ void Parameter<Operation::CREATE_DATASET>::warnUnusedParameters<
     }
 }
 
+template <>
+json::ParsedConfig Parameter<Operation::CREATE_DATASET>::compileJSONConfig(
+    Writable const *writable, json::JsonMatcher &jsonMatcher) const
+{
+    auto attri = writable->attributable->asInternalCopyOf<Attributable>();
+    auto path = attri.myPath().openPMDPath();
+    auto base_config = jsonMatcher.get(path);
+    auto manual_config =
+        json::parseOptions(options, /* considerFiles = */ false);
+    json::merge(base_config.config, manual_config.config);
+    return json::ParsedConfig{
+        std::move(base_config.config),
+        (options.empty() || options == "{}")
+            ? manual_config.originallySpecifiedAs
+            : base_config.originallySpecifiedAs};
+}
+
 namespace internal
 {
     std::string operationAsString(Operation op)

src/auxiliary/JSONMatcher.cpp

@@ -1,7 +1,9 @@
 #include "openPMD/auxiliary/JSONMatcher.hpp"
 #include "openPMD/Error.hpp"
 #include "openPMD/auxiliary/JSON_internal.hpp"
+
 #include <nlohmann/json.hpp>
+#include <sstream>
 
 namespace openPMD::json
 {
@@ -27,6 +29,19 @@ namespace
         nlohmann::json object) -> void;
 } // namespace
 
+Pattern::Pattern(std::string const &pattern_in, nlohmann::json config_in)
+    : config(std::move(config_in))
+{
+    // transform the regex such that the path to the Iteration is optional
+    std::stringstream build_pattern;
+    build_pattern << "(/data/[0-9]+/)?(" << pattern_in << ")";
+    // we construct the patterns once and use them often, so let's ask for
+    // some optimization
+    pattern = std::regex(
+        build_pattern.str(),
+        std::regex_constants::egrep | std::regex_constants::optimize);
+}
+
 void MatcherPerBackend::init(TracingJSON tracing_config)
 {
     auto &config = tracing_config.json();