Merge pull request #15 from scalableminds/storage-transformers

Storage transformers

Author: normanrz

src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java

@@ -55,6 +55,9 @@ public final class ArrayMetadata {
   @Nullable
   @JsonProperty("dimension_names")
   public String[] dimensionNames;
+  @Nullable
+  @JsonProperty("storage_transformers")
+  public Map<String, Object>[] storageTransformers;
 
   @JsonIgnore
   public CoreArrayMetadata coreArrayMetadata;
@@ -64,11 +67,12 @@ public ArrayMetadata(
       Object fillValue,
       @Nonnull Codec[] codecs,
       @Nullable String[] dimensionNames,
-      @Nullable Map<String, Object> attributes
+      @Nullable Map<String, Object> attributes,
+      @Nullable Map<String, Object>[] storageTransformers
   ) throws ZarrException {
     this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
         dimensionNames,
-        attributes
+        attributes, storageTransformers
     );
   }
 
@@ -83,7 +87,8 @@ public ArrayMetadata(
       @JsonProperty(value = "fill_value", required = true) Object fillValue,
       @Nonnull @JsonProperty(value = "codecs") Codec[] codecs,
       @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames,
-      @Nullable @JsonProperty(value = "attributes") Map<String, Object> attributes
+      @Nullable @JsonProperty(value = "attributes") Map<String, Object> attributes,
+      @Nullable @JsonProperty(value = "storage_transformers") Map<String, Object>[] storageTransformers
   ) throws ZarrException {
     if (zarrFormat != this.zarrFormat) {
       throw new ZarrException(
@@ -93,7 +98,10 @@ public ArrayMetadata(
       throw new ZarrException(
           "Expected node type '" + this.nodeType + "', got '" + nodeType + "'.");
     }
-
+    if (storageTransformers != null && storageTransformers.length > 0) {
+      throw new ZarrException(
+          "Storage transformers are not supported in this version of Zarr Java.");
+    }
     if (chunkGrid instanceof RegularChunkGrid) {
       int[] chunkShape = ((RegularChunkGrid) chunkGrid).configuration.chunkShape;
       if (shape.length != chunkShape.length) {
@@ -126,6 +134,7 @@ public ArrayMetadata(
     this.codecs = codecs;
     this.dimensionNames = dimensionNames;
     this.attributes = attributes;
+    this.storageTransformers = storageTransformers;
     this.coreArrayMetadata =
         new CoreArrayMetadata(shape, ((RegularChunkGrid) chunkGrid).configuration.chunkShape,
             dataType,

src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java

@@ -29,6 +29,7 @@ public class ArrayMetadataBuilder {
   Object fillValue = 0;
   Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)};
   Map<String, Object> attributes = new HashMap<>();
+  Map<String, Object>[] storageTransformers = new HashMap[]{};
   String[] dimensionNames = null;
 
   protected ArrayMetadataBuilder() {
@@ -44,6 +45,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad
     builder.codecs = arrayMetadata.codecs;
     builder.attributes = arrayMetadata.attributes;
     builder.dimensionNames = arrayMetadata.dimensionNames;
+    builder.storageTransformers = arrayMetadata.storageTransformers;
     return builder;
   }
 
@@ -125,6 +127,10 @@ public ArrayMetadataBuilder withAttributes(Map<String, Object> attributes) {
     this.attributes = attributes;
     return this;
   }
+  public ArrayMetadataBuilder withStorageTransformers(Map<String, Object>[] storageTransformers) {
+    this.storageTransformers = storageTransformers;
+    return this;
+  }
 
   public ArrayMetadata build() throws ZarrException {
     if (shape == null) {
@@ -140,7 +146,8 @@ public ArrayMetadata build() throws ZarrException {
 
     return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs,
         dimensionNames,
-        attributes
+        attributes,
+        storageTransformers
     );
   }
 }

src/test/java/dev/zarr/zarrjava/ZarrTest.java

@@ -3,9 +3,11 @@
 import com.amazonaws.auth.AWSStaticCredentialsProvider;
 import com.amazonaws.auth.AnonymousAWSCredentials;
 import com.amazonaws.services.s3.AmazonS3ClientBuilder;
+import com.fasterxml.jackson.databind.JsonMappingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.github.luben.zstd.Zstd;
 import com.github.luben.zstd.ZstdCompressCtx;
+import com.google.common.collect.Maps;
 import dev.zarr.zarrjava.store.*;
 import dev.zarr.zarrjava.utils.MultiArrayUtils;
 import dev.zarr.zarrjava.v3.*;
@@ -654,5 +656,32 @@ public void testParallel(boolean useParallel) throws IOException, ZarrException
         Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
             clearTestoutputFolder();
     }
+
+    @Test
+    public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException {
+        // non-empty storage transformers are currently not supported
+
+        Map<String, Object>[] storageTransformersEmpty = Array.open(
+            new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty")
+        ).metadata.storageTransformers;
+        assert storageTransformersEmpty.length == 0;
+
+        assertThrows(JsonMappingException.class, () -> Array.open(
+            new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists"))
+        );
+
+        ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder()
+                    .withShape(1)
+                    .withChunkShape(1)
+                    .withDataType(DataType.UINT8)
+                    .withStorageTransformers(new HashMap[]{new HashMap<String, Object>(){{
+                        put("some", "value");
+                    }}});
+
+        assertThrows(ZarrException.class, () -> Array.create(
+            new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"),
+            builderWithStorageTransformer.build()
+        ));
+    }
 }
 

testdata/storage_transformer/empty/zarr.json

@@ -0,0 +1,33 @@
+{
+  "zarr_format": 3,
+  "node_type": "array",
+  "shape": [
+    1
+  ],
+  "data_type": "uint8",
+  "chunk_grid": {
+    "name": "regular",
+    "configuration": {
+      "chunk_shape": [
+        1
+      ]
+    }
+  },
+  "chunk_key_encoding": {
+    "name": "default",
+    "configuration": {
+      "separator": "/"
+    }
+  },
+  "fill_value": 0,
+  "codecs": [
+    {
+      "name": "bytes",
+      "configuration": {
+        "endian": "little"
+      }
+    }
+  ],
+  "attributes": {},
+  "storage_transformers": []
+}

testdata/storage_transformer/exists/zarr.json

@@ -0,0 +1,40 @@
+{
+  "zarr_format": 3,
+  "node_type": "array",
+  "shape": [
+    1
+  ],
+  "data_type": "uint8",
+  "chunk_grid": {
+    "name": "regular",
+    "configuration": {
+      "chunk_shape": [
+        1
+      ]
+    }
+  },
+  "chunk_key_encoding": {
+    "name": "default",
+    "configuration": {
+      "separator": "/"
+    }
+  },
+  "fill_value": 0,
+  "codecs": [
+    {
+      "name": "bytes",
+      "configuration": {
+        "endian": "little"
+      }
+    }
+  ],
+  "attributes": {},
+  "storage_transformers": [
+    {
+      "name": "chunk-manifest-json",
+      "configuration": {
+        "manifest": "./manifest.json"
+      }
+    }
+  ]
+}