Add compression example

Author: franzpoeschel

CMakeLists.txt

@@ -710,6 +710,7 @@ set(openPMD_EXAMPLE_NAMES
     12_span_write
     13_write_dynamic_configuration
     14_toml_template
+    15_compression
 )
 set(openPMD_PYTHON_EXAMPLE_NAMES
     2_read_serial

examples/15_compression.cpp

@@ -0,0 +1,156 @@
+#include <numeric>
+#include <openPMD/openPMD.hpp>
+
+#if __has_include(<blosc2_filter.h>)
+#include <blosc2_filter.h>
+#define OPENPMD_USE_BLOSC2_FILTER 1
+#else
+#define OPENPMD_USE_BLOSC2_FILTER 0
+#endif
+
+void init_blosc_for_hdf5()
+{
+#if OPENPMD_USE_BLOSC2_FILTER
+    /*
+     * This registers the Blosc2 plugin from
+     * https://github.com/Blosc/HDF5-Blosc2 as a demonstration on how to
+     * activate and configure dynamic HDF5 filter plugins through openPMD.
+     */
+
+    char *version, *date;
+    int r = register_blosc2(&version, &date);
+    if (r < 1)
+    {
+        throw std::runtime_error("Unable to register Blosc2 plugin with HDF5.");
+    }
+    else
+    {
+        std::cout << "Blosc2 plugin registered in version '" << version
+                  << "' and date '" << date << "'." << std::endl;
+    }
+#endif
+}
+
+void write(std::string const &filename, std::string const &config)
+{
+    using namespace openPMD;
+    Series series(
+        "../samples/compression/" + filename, Access::CREATE_LINEAR, config);
+
+    for (size_t i = 0; i < 10; ++i)
+    {
+        auto &current_iteration = series.snapshots()[i];
+
+        // First, write an E mesh.
+        auto &E = current_iteration.meshes["E"];
+        E.setAxisLabels({"x", "y"});
+        for (auto const &dim : {"x", "y", "z"})
+        {
+            auto &component = E[dim];
+            component.resetDataset({Datatype::FLOAT, {10, 10}});
+            auto buffer_view =
+                component.storeChunk<float>({0, 0}, {10, 10}).currentBuffer();
+            // Now fill the prepared buffer with some nonsense data.
+            std::iota(buffer_view.begin(), buffer_view.end(), i * 100);
+        }
+
+        // Now, write some e particles.
+        auto &e = current_iteration.particles["e"];
+        for (auto const &dim : {"x", "y", "z"})
+        {
+            // Do not bother with a positionOffset
+            auto &position_offset = e["positionOffset"][dim];
+            position_offset.makeConstant(0);
+
+            auto &position = e["position"][dim];
+            position.resetDataset({Datatype::FLOAT, {10}});
+            auto buffer_view =
+                position.storeChunk<float>({0}, {10}).currentBuffer();
+            // Now fill the prepared buffer with some nonsense data.
+            std::iota(buffer_view.begin(), buffer_view.end(), i * 10);
+        }
+    }
+}
+
+int main()
+{
+    init_blosc_for_hdf5();
+
+    // Backend specific configuration can be given in either JSON or TOML.
+    // We will stick with TOML in this example, since it allows inline comments
+    // and remains more legible for larger configurations.
+    // openPMD includes / will in future include separate tooling for converting
+    // between JSON and TOML (openpmd-convert-toml-json), which you may use to
+    // convert these configurations to JSON.
+
+    std::string const simple_adios2_config = R"(
+
+        # Backend can either be inferred from the filename ending, or specified
+        # explicitly. In the latter case, the filename ending can be given as
+        # a wildcard %E, openPMD will then pick a default ending.
+        backend = "adios2"
+
+        # ADIOS2 supports adding multiple operators to a variable, hence we
+        # specify a list of operators here (using TOML's double bracket syntax).
+        # How much sense this makes depends on the specific operators in use.
+
+        [[adios2.dataset.operators]]
+        type = "zlib"
+        parameters.clevel = 9  # The available parameters depend
+                               # on the operator.
+                               # Here, we specify zlib's compression level.
+    )";
+    write("adios2_with_zlib.%E", simple_adios2_config);
+
+    std::string const extended_adios2_config = R"(
+        backend = "adios2"
+
+        # The compression can also be specified per-dataset.
+        # For more details, also check:
+        # https://openpmd-api.readthedocs.io/en/latest/details/backendconfig.html#dataset-specific-configuration
+
+        # This example will demonstrate the use of pattern matching.
+        # adios2.dataset is now a list of dataset configurations. The specific
+        # configuration to be used for a dataset will be determined by matching
+        # the dataset name against the patterns specified by the 'select' keys.
+        [[adios2.dataset]]
+        # This uses egrep-type regular expressions.
+        select = "meshes/.*"
+        # Now, specify the operators list again. Let's use Blosc for this.
+        [[adios2.dataset.cfg.operators]]
+        type = "blosc"
+        parameters.doshuffle = "BLOSC_BITSHUFFLE"
+        parameters.clevel = 1
+
+        # Now, configure the particles.
+        [[adios2.dataset]]
+        # The match can either be against the path within the containing
+        # Iteration (e.g. 'meshes/E/x', as above) or (as in this example),
+        # against the full path (e.g. '/data/0/particles/e/position/x').
+        # the containing Iteration (e.g. 'meshes/E/x').
+        # In this example, completely deactivate compression specifically for
+        # particles in Iterations 2, 3 and 4. All other particle datasets will
+        # fall back to the default configuration specified below.
+        select = "/data/(2|3|4)/particles/.*"
+        cfg.operators = []
+
+        # Now, the default configuration.
+        # In general, the dataset configurations are matched top-down, going for
+        # the first matching configuration. So, a default configuration could
+        # theoretically be specified by emplacing a catch-all pattern
+        # (regex: ".*") as the last option.
+        # However, we also define an explicit shorthand for specifying default
+        # configurations: Just omit the 'select' key. This special syntax is
+        # understood as the default configuration no matter where in the list it
+        # is emplaced, and it allows the backends to initialize the default
+        # configuration globally, instead of applying it selectively to each
+        # dataset that matches a catch-all pattern.
+        [[adios2.dataset]]
+        [[adios2.dataset.cfg.operators]]
+        type = "zlib"
+        parameters.clevel = 2
+    )";
+    write(
+        "adios2_with_dataset_specific_configurations.%E",
+        extended_adios2_config);
+}

src/IO/ADIOS/ADIOS2IOHandler.cpp

@@ -2449,8 +2449,11 @@ namespace detail
             {
                 var.SetSelection({start, count});
             }
-            // don't add compression operators multiple times
-            return;
+            // Compression operators can be specified per-block in ADIOS2.
+            // While we don't support that, we do support changing the
+            // compression across multiple steps.
+            // Hence remove and reapply operators, they might have changed.
+            var.RemoveOperations();
         }
 
         if (!var)