From dc35390216809b645f39c190b5f07e0b82e71b20 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 22 Oct 2025 17:36:52 +0200 Subject: [PATCH 01/14] add support for attributes in Array and ArrayMetadata classes --- .../dev/zarr/zarrjava/core/ArrayMetadata.java | 2 + .../dev/zarr/zarrjava/core/Attributes.java | 217 ++++++++++++++++++ .../java/dev/zarr/zarrjava/core/Node.java | 1 + src/main/java/dev/zarr/zarrjava/v2/Array.java | 23 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 35 ++- .../zarrjava/v2/ArrayMetadataBuilder.java | 13 +- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 10 + .../dev/zarr/zarrjava/ZarrPythonTests.java | 11 +- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 103 +++++++++ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 32 +++ .../python-scripts/zarr_python_read_v2.py | 2 +- 11 files changed, 435 insertions(+), 14 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/Attributes.java diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index aaa004ef..e19d193d 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -41,6 +41,8 @@ public int ndim() { public abstract Object parsedFillValue(); + public @Nonnull abstract Attributes attributes() throws ZarrException; + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) throws ZarrException { if (fillValue == null) { diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java new file mode 100644 index 00000000..b2903ed5 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -0,0 +1,217 @@ +package dev.zarr.zarrjava.core; + +import javax.annotation.Nonnull; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class Attributes extends HashMap { + + public Attributes() { + super(); + } + + public Attributes(Map attributes) { + super(attributes); + } + + public boolean getBoolean(String key) { + Object value = this.get(key); + if (value instanceof Boolean) { + return (Boolean) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a Boolean"); + } + + public int getInt(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).intValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Integer"); + } + + public double getDouble(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Double"); + } + + public float getFloat(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Float"); + } + + @Nonnull + public String getString(String key) { + Object value = this.get(key); + if (value instanceof String) { + return (String) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a String"); + } + + @Nonnull + public List getList(String key) { + Object value = this.get(key); + if (value instanceof List) { + return (List) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List"); + } + + public Attributes getAttributes(String key) { + Object value = this.get(key); + if (value instanceof Attributes) { + return (Attributes) value; + } + if (value instanceof Map) { + return new Attributes((Map) value); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Attributes object"); + } + + public T[] getArray(String key, Class clazz) throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException { + Object value = this.get(key); + if (value instanceof Object[] && ( ((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { + return (T[]) value; + } + if (value instanceof List) { + List list = (List) value; + T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (clazz.isInstance(elem)) { + array[i] = clazz.cast(elem); + } else { + java.lang.reflect.Constructor matched = null; + for (java.lang.reflect.Constructor c : clazz.getConstructors()) { + Class[] params = c.getParameterTypes(); + if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) { + matched = c; + break; + } + } + if (matched != null) { + array[i] = (T) matched.newInstance(elem); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName()); + } + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName()); +} + + public int[] getIntArray(String key) { + Object value = this.get(key); + if (value instanceof int[]) { + return (int[]) value; + } + if (value instanceof List) { + List list = (List) value; + int[] array = new int[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).intValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not an int array or List"); + } + + public long[] getLongArray(String key) { + Object value = this.get(key); + if (value instanceof long[]) { + return (long[]) value; + } + if (value instanceof List) { + List list = (List) value; + long[] array = new long[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).longValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a long array or List"); + } + + public double[] getDoubleArray(String key) { + Object value = this.get(key); + if (value instanceof double[]) { + return (double[]) value; + } + if (value instanceof List) { + List list = (List) value; + double[] array = new double[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).doubleValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a double array or List"); + } + + public float[] getFloatArray(String key) { + Object value = this.get(key); + if (value instanceof float[]) { + return (float[]) value; + } + if (value instanceof List) { + List list = (List) value; + float[] array = new float[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).floatValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a float array or List"); + } + + public boolean[] getBooleanArray(String key) { + Object value = this.get(key); + if (value instanceof boolean[]) { + return (boolean[]) value; + } + if (value instanceof List) { + List list = (List) value; + boolean[] array = new boolean[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Boolean) { + array[i] = (Boolean) elem; + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Boolean"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a boolean array or List"); + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 0b511936..fbac2ba3 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -13,6 +13,7 @@ public interface Node { String ZARR_JSON = "zarr.json"; String ZARRAY = ".zarray"; + String ZATTRS = ".zattrs"; String ZGROUP = ".zgroup"; /** diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 638d6484..184fae31 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -46,13 +47,19 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE * @throws ZarrException throws ZarrException if the Zarr array cannot be opened */ public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper mapper = makeObjectMapper(); + ArrayMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), + ArrayMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + Attributes.class + ); return new Array( storeHandle, - makeObjectMapper() - .readValue( - Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), - ArrayMetadata.class - ) + metadata ); } @@ -143,6 +150,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, } ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata)); + if (arrayMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectMapper.writeValueAsBytes(arrayMetadata.attributes)); + attrsHandle.set(attrsBytes); + } metadataHandle.set(metadataBytes); return new Array(storeHandle, arrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index dabd2855..b53eee3d 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; @@ -11,6 +12,7 @@ import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; +import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -39,6 +41,10 @@ public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { @Nullable public final Codec compressor; + @Nullable + @JsonIgnore + public Attributes attributes; + @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -54,6 +60,22 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator + ) throws ZarrException { + this(zarrFormat, shape, chunks, dataType, fillValue, order, filters, compressor, dimensionSeparator, null); + } + + + public ArrayMetadata( + int zarrFormat, + long[] shape, + int[] chunks, + DataType dataType, + @Nullable Object fillValue, + Order order, + @Nullable Codec[] filters, + @Nullable Codec compressor, + @Nullable Separator dimensionSeparator, + @Nullable Attributes attributes ) throws ZarrException { super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { @@ -78,6 +100,7 @@ public ArrayMetadata( } } this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); + this.attributes = attributes; } @@ -87,8 +110,6 @@ public int[] chunkShape() { return chunks; } - - @Override public DataType dataType() { return dataType; @@ -111,4 +132,14 @@ public ChunkKeyEncoding chunkKeyEncoding() { public Object parsedFillValue() { return parsedFillValue; } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } + + } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index ea29b165..2571de0e 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -2,6 +2,7 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.core.BloscCodec; @@ -16,6 +17,7 @@ public class ArrayMetadataBuilder { Object fillValue = null; Codec[] filters = null; Codec compressor = null; + Attributes attributes = null; protected ArrayMetadataBuilder() { @@ -119,6 +121,14 @@ public ArrayMetadataBuilder withZlibCompressor() { return withZlibCompressor(5); } + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } + return this; + } public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new IllegalStateException("Please call `withShape` first."); @@ -138,7 +148,8 @@ public ArrayMetadata build() throws ZarrException { order, filters, compressor, - dimensionSeparator + dimensionSeparator, + attributes ); } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 36253482..3536af37 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -143,6 +144,15 @@ public Object parsedFillValue() { return parsedFillValue; } + @Nonnull + @Override + public Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return new Attributes(attributes); //todo change attributes to Attributes type + } + public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index e4e763ac..9a53c061 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -2,6 +2,7 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.Group; @@ -314,7 +315,7 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, array.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); -// Assertions.assertEquals(42, array.metadata().attributes.get("answer")); + Assertions.assertEquals(42, array.metadata().attributes().get("answer")); assertIsTestdata(result, dt); } @@ -323,15 +324,15 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV2") public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws Exception { -// Map attributes = new HashMap<>(); -// attributes.put("test_key", "test_value"); + Attributes attributes = new Attributes(); + attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) .withDataType(dt) .withChunks(2, 4, 8) -// .withAttributes(attributes) + .withAttributes(attributes) .withFillValue(0); switch (compressor) { @@ -358,7 +359,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, readArray.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); -// Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Assertions.assertEquals("test_value", readArray.metadata().attributes().get("test_key")); assertIsTestdata(result, dt); //read in zarr_python diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 765c51a4..214353e2 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,5 +1,6 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.Array; @@ -14,10 +15,13 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; +import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @@ -255,4 +259,103 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } + + static void assertListEquals(List a, List b) { + Assertions.assertEquals(a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + Object aval = a.get(i); + Object bval = b.get(i); + if (aval instanceof List && bval instanceof List) { + assertListEquals((List) aval, (List) bval); + } else { + Assertions.assertEquals(aval, bval); + } + } + } + + @Test + public void testAttributes() throws IOException, ZarrException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); + ArrayList list = new ArrayList(){ + { + add(1); + add(2.0d); + add("string"); + } + }; + int[] intArray = new int[]{1,2,3}; + long[] longArray = new long[]{1,2,3}; + double[] doubleArray = new double[]{1.0,2.0,3.0}; + float[] floatArray = new float[]{1.0f,2.0f,3.0f}; + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 0.5f); + put("double", 3.14); + put("boolean", true); + put("list", list); + put("int_array", intArray); + put("long_array", longArray); + put("double_array", doubleArray); + put("float_array", floatArray); + put("nested", new Attributes() {{ + put("element", "value"); + }}); + put("array_of_attributes", new Attributes[] { + new Attributes() {{ put("a", 1); }}, + new Attributes() {{ put("b", 2); }} + }); + }}) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("stringvalue", array.metadata().attributes().getString("string")); + Assertions.assertEquals(42, array.metadata().attributes().getInt("int")); + Assertions.assertEquals(0.5, array.metadata().attributes().getFloat("float")); + Assertions.assertEquals(3.14, array.metadata().attributes().getDouble("double")); + Assertions.assertTrue(array.metadata().attributes().getBoolean("boolean")); + assertListEquals(list, array.metadata().attributes().getList("list")); + Assertions.assertArrayEquals(intArray, array.metadata().attributes().getIntArray("int_array")); + Assertions.assertArrayEquals(longArray, array.metadata().attributes().getLongArray("long_array")); + Assertions.assertArrayEquals(doubleArray, array.metadata().attributes().getDoubleArray("double_array")); + Assertions.assertArrayEquals(floatArray, array.metadata().attributes().getFloatArray("float_array")); + Assertions.assertEquals("value", array.metadata().attributes().getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ put("a", 1); }}, + new Attributes() {{ put("b", 2); }} + }, + array.metadata().attributes().getArray("array_of_attributes", Attributes.class) + ); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + Assertions.assertEquals("stringvalue", arrayOpened.metadata().attributes().getString("string")); + Assertions.assertEquals(42, arrayOpened.metadata().attributes().getInt("int")); + Assertions.assertEquals(0.5, arrayOpened.metadata().attributes().getFloat("float")); + Assertions.assertEquals(3.14, arrayOpened.metadata().attributes().getDouble("double")); + Assertions.assertTrue(arrayOpened.metadata().attributes().getBoolean("boolean")); + assertListEquals(list, arrayOpened.metadata().attributes().getList("list")); + Assertions.assertArrayEquals(intArray, arrayOpened.metadata().attributes().getIntArray("int_array")); + Assertions.assertArrayEquals(longArray, arrayOpened.metadata().attributes().getLongArray("long_array")); + Assertions.assertArrayEquals(doubleArray, arrayOpened.metadata().attributes().getDoubleArray("double_array")); + Assertions.assertArrayEquals(floatArray, arrayOpened.metadata().attributes().getFloatArray("float_array")); + Assertions.assertEquals("value", arrayOpened.metadata().attributes().getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ put("a", 1); }}, + new Attributes() {{ put("b", 2); }} + }, + arrayOpened.metadata().attributes().getArray("array_of_attributes", Attributes.class) + ); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 7f1c15c7..43a9c1af 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -569,6 +569,38 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new HashMap() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 3.14); + put("boolean", true); + put("array", new int[]{1, 2, 3}); + }}) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("stringvalue", array.metadata().attributes.get("string")); + Assertions.assertEquals(42, array.metadata().attributes.get("int")); + Assertions.assertEquals(3.14, array.metadata().attributes.get("float")); + Assertions.assertEquals(true, array.metadata().attributes.get("boolean")); + Assertions.assertArrayEquals(new int[]{1, 2, 3}, (int[]) array.metadata().attributes.get("array")); + + Array array2 = Array.open(storeHandle); + Assertions.assertEquals("stringvalue", array2.metadata().attributes.get("string")); + Assertions.assertEquals(42, array2.metadata().attributes.get("int")); + Assertions.assertEquals(3.14, array2.metadata().attributes.get("float")); + Assertions.assertEquals(true, array2.metadata().attributes.get("boolean")); + Assertions.assertArrayEquals(new int[]{1, 2, 3}, (int[])(array2.metadata().attributes.get("array"))); + } + + } diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5ff5e5a8..5d0b1c63 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -33,7 +33,7 @@ filters=filters, serializer=serializer, compressors=compressor, -# attributes={'test_key': 'test_value'}, + attributes={'test_key': 'test_value'}, ) assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" From 832ef4e0095dfaf4b679ee88c11d5eb09e89d971 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 22 Oct 2025 20:36:56 +0200 Subject: [PATCH 02/14] change v3 attributes class to Attributes --- .../dev/zarr/zarrjava/core/Attributes.java | 15 +- .../zarrjava/v2/ArrayMetadataBuilder.java | 7 +- src/main/java/dev/zarr/zarrjava/v3/Array.java | 10 +- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 6 +- .../zarrjava/v3/ArrayMetadataBuilder.java | 11 +- src/main/java/dev/zarr/zarrjava/v3/Group.java | 9 +- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 12 +- .../dev/zarr/zarrjava/ZarrAttributesTest.java | 139 ++++++++++++++++++ .../dev/zarr/zarrjava/ZarrPythonTests.java | 2 +- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 98 ------------ .../java/dev/zarr/zarrjava/ZarrV3Test.java | 40 +---- 11 files changed, 187 insertions(+), 162 deletions(-) create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java index b2903ed5..fd097c41 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Attributes.java +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -1,7 +1,8 @@ package dev.zarr.zarrjava.core; +import dev.zarr.zarrjava.ZarrException; + import javax.annotation.Nonnull; -import java.lang.reflect.InvocationTargetException; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -77,19 +78,21 @@ public Attributes getAttributes(String key) { throw new IllegalArgumentException("Value for key " + key + " is not an Attributes object"); } - public T[] getArray(String key, Class clazz) throws NoSuchMethodException, InvocationTargetException, InstantiationException, IllegalAccessException { + public T[] getArray(String key, Class clazz) throws ZarrException { Object value = this.get(key); if (value instanceof Object[] && ( ((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { return (T[]) value; } if (value instanceof List) { List list = (List) value; + @SuppressWarnings("unchecked") T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); for (int i = 0; i < list.size(); i++) { Object elem = list.get(i); if (clazz.isInstance(elem)) { array[i] = clazz.cast(elem); } else { + // Try to find a constructor that takes the element's class java.lang.reflect.Constructor matched = null; for (java.lang.reflect.Constructor c : clazz.getConstructors()) { Class[] params = c.getParameterTypes(); @@ -99,9 +102,13 @@ public T[] getArray(String key, Class clazz) throws NoSuchMethodException } } if (matched != null) { - array[i] = (T) matched.newInstance(elem); + try { + array[i] = (T) matched.newInstance(elem); + } catch (Exception e) { + throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e); + } } else { - throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName()); + throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName()); } } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 2571de0e..897e4e62 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -17,7 +17,7 @@ public class ArrayMetadataBuilder { Object fillValue = null; Codec[] filters = null; Codec compressor = null; - Attributes attributes = null; + Attributes attributes = new Attributes(); protected ArrayMetadataBuilder() { @@ -121,6 +121,11 @@ public ArrayMetadataBuilder withZlibCompressor() { return withZlibCompressor(5); } + public ArrayMetadataBuilder putAttribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + public ArrayMetadataBuilder withAttributes(Attributes attributes) { if (this.attributes == null) { this.attributes = attributes; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 4d1434f7..bf11c26c 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -218,7 +219,7 @@ public Array resize(long[] newShape) throws ZarrException, IOException { * @throws ZarrException throws ZarrException if the new metadata is invalid * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array setAttributes(Map newAttributes) throws ZarrException, IOException { + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) .withAttributes(newAttributes) @@ -234,11 +235,10 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized + * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array updateAttributes(Function, Map> attributeMapper) - throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(new HashMap(metadata.attributes) { + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(new Attributes(metadata.attributes) { })); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 3536af37..7a97dd05 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -40,7 +40,7 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { public final Codec[] codecs; @Nullable @JsonProperty("attributes") - public final Map attributes; + public final Attributes attributes; @Nullable @JsonProperty("dimension_names") public final String[] dimensionNames; @@ -56,7 +56,7 @@ public ArrayMetadata( Object fillValue, @Nonnull Codec[] codecs, @Nullable String[] dimensionNames, - @Nullable Map attributes, + @Nullable Attributes attributes, @Nullable Map[] storageTransformers ) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, @@ -76,7 +76,7 @@ public ArrayMetadata( @JsonProperty(value = "fill_value", required = true) Object fillValue, @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, - @Nullable @JsonProperty(value = "attributes") Map attributes, + @Nullable @JsonProperty(value = "attributes") Attributes attributes, @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { super(shape, fillValue, dataType); diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 212f4732..9def15f2 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -26,7 +27,7 @@ public class ArrayMetadataBuilder { Object fillValue = 0; Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); Map[] storageTransformers = new HashMap[]{}; String[] dimensionNames = null; @@ -121,8 +122,12 @@ public ArrayMetadataBuilder putAttribute(String key, Object value) { return this; } - public ArrayMetadataBuilder withAttributes(Map attributes) { - this.attributes = attributes; + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } return this; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 9c56f25c..c5e4cbc9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -53,7 +54,7 @@ public static Group create( public static Group create( @Nonnull StoreHandle storeHandle, - @Nonnull Map attributes + @Nonnull Attributes attributes ) throws IOException, ZarrException { return new Group(storeHandle, new GroupMetadata(attributes)); } @@ -93,7 +94,7 @@ public Group createGroup(String key, GroupMetadata groupMetadata) return Group.create(storeHandle.resolve(key), groupMetadata); } - public Group createGroup(String key, Map attributes) + public Group createGroup(String key, Attributes attributes) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); } @@ -121,12 +122,12 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { return new Group(storeHandle, newGroupMetadata); } - public Group setAttributes(Map newAttributes) throws ZarrException, IOException { + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); return writeMetadata(newGroupMetadata); } - public Group updateAttributes(Function, Map> attributeMapper) + public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index df32c046..5792213b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -3,8 +3,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import java.util.HashMap; -import java.util.Map; +import dev.zarr.zarrjava.core.Attributes; + import javax.annotation.Nullable; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -17,9 +17,9 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { public final String nodeType = "group"; @Nullable - public final Map attributes; + public final Attributes attributes; - public GroupMetadata(@Nullable Map attributes) throws ZarrException { + public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, attributes); } @@ -27,7 +27,7 @@ public GroupMetadata(@Nullable Map attributes) throws ZarrExcept public GroupMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "node_type", required = true) String nodeType, - @Nullable @JsonProperty(value = "attributes") Map attributes + @Nullable @JsonProperty(value = "attributes") Attributes attributes ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -41,6 +41,6 @@ public GroupMetadata( } public static GroupMetadata defaultValue() throws ZarrException { - return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new HashMap<>()); + return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java b/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java new file mode 100644 index 00000000..f2e5da25 --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java @@ -0,0 +1,139 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.core.Attributes; +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v2.Array; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v2.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +public class ZarrAttributesTest extends ZarrTest { + + ArrayList listAttribute = new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }; + int[] intArrayAttribute = new int[]{1, 2, 3}; + long[] longArrayAttribute = new long[]{1, 2, 3}; + double[] doubleArrayAttribute = new double[]{1.0, 2.0, 3.0}; + float[] floatArrayAttribute = new float[]{1.0f, 2.0f, 3.0f}; + Attributes testAttributes = new Attributes() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 0.5f); + put("double", 3.14); + put("boolean", true); + put("list", listAttribute); + put("int_array", intArrayAttribute); + put("long_array", longArrayAttribute); + put("double_array", doubleArrayAttribute); + put("float_array", floatArrayAttribute); + put("nested", new Attributes() {{ + put("element", "value"); + }}); + put("array_of_attributes", new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }); + }}; + + static void assertListEquals(List a, List b) { + Assertions.assertEquals(a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + Object aval = a.get(i); + Object bval = b.get(i); + if (aval instanceof List && bval instanceof List) { + assertListEquals((List) aval, (List) bval); + } else { + Assertions.assertEquals(aval, bval); + } + } + } + + void assertContainsTestAttributes(Attributes attributes) throws ZarrException { + Assertions.assertEquals("stringvalue", attributes.getString("string")); + Assertions.assertEquals(42, attributes.getInt("int")); + Assertions.assertEquals(0.5, attributes.getFloat("float")); + Assertions.assertEquals(3.14, attributes.getDouble("double")); + Assertions.assertTrue(attributes.getBoolean("boolean")); + assertListEquals(listAttribute, attributes.getList("list")); + Assertions.assertArrayEquals(intArrayAttribute, attributes.getIntArray("int_array")); + Assertions.assertArrayEquals(longArrayAttribute, attributes.getLongArray("long_array")); + Assertions.assertArrayEquals(doubleArrayAttribute, attributes.getDoubleArray("double_array")); + Assertions.assertArrayEquals(floatArrayAttribute, attributes.getFloatArray("float_array")); + Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ put("a", 1); }}, + new Attributes() {{ put("b", 2); }} + }, + attributes.getArray("array_of_attributes", Attributes.class) + ); + } + + @Test + public void testAttributesV2() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); + + ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(testAttributes) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + dev.zarr.zarrjava.v2.Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + } + + @Test + public void testAttributesV3() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); + + dev.zarr.zarrjava.v3.ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v3.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) + .withChunkShape(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(testAttributes) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + + .build(); + + dev.zarr.zarrjava.v3.Array array = dev.zarr.zarrjava.v3.Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + dev.zarr.zarrjava.v3.Array arrayOpened = dev.zarr.zarrjava.v3.Array.open(storeHandle); + assertContainsTestAttributes(arrayOpened.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 9a53c061..4a17d748 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -210,7 +210,7 @@ public void testReadV3(String codec, String codecParam, DataType dataType) throw @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV3") public void testWriteV3(String codec, String codecParam, DataType dataType) throws Exception { - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 214353e2..dc0f4fa8 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -260,102 +260,4 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } - static void assertListEquals(List a, List b) { - Assertions.assertEquals(a.size(), b.size()); - for (int i = 0; i < a.size(); i++) { - Object aval = a.get(i); - Object bval = b.get(i); - if (aval instanceof List && bval instanceof List) { - assertListEquals((List) aval, (List) bval); - } else { - Assertions.assertEquals(aval, bval); - } - } - } - - @Test - public void testAttributes() throws IOException, ZarrException, InvocationTargetException, NoSuchMethodException, InstantiationException, IllegalAccessException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); - ArrayList list = new ArrayList(){ - { - add(1); - add(2.0d); - add("string"); - } - }; - int[] intArray = new int[]{1,2,3}; - long[] longArray = new long[]{1,2,3}; - double[] doubleArray = new double[]{1.0,2.0,3.0}; - float[] floatArray = new float[]{1.0f,2.0f,3.0f}; - - ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) - .withAttributes(new Attributes() {{ - put("string", "stringvalue"); - put("int", 42); - put("float", 0.5f); - put("double", 3.14); - put("boolean", true); - put("list", list); - put("int_array", intArray); - put("long_array", longArray); - put("double_array", doubleArray); - put("float_array", floatArray); - put("nested", new Attributes() {{ - put("element", "value"); - }}); - put("array_of_attributes", new Attributes[] { - new Attributes() {{ put("a", 1); }}, - new Attributes() {{ put("b", 2); }} - }); - }}) - .withAttributes(new Attributes() {{ - put("another", "attribute"); - }}) - .build(); - - Array array = Array.create(storeHandle, arrayMetadata); - Assertions.assertEquals("stringvalue", array.metadata().attributes().getString("string")); - Assertions.assertEquals(42, array.metadata().attributes().getInt("int")); - Assertions.assertEquals(0.5, array.metadata().attributes().getFloat("float")); - Assertions.assertEquals(3.14, array.metadata().attributes().getDouble("double")); - Assertions.assertTrue(array.metadata().attributes().getBoolean("boolean")); - assertListEquals(list, array.metadata().attributes().getList("list")); - Assertions.assertArrayEquals(intArray, array.metadata().attributes().getIntArray("int_array")); - Assertions.assertArrayEquals(longArray, array.metadata().attributes().getLongArray("long_array")); - Assertions.assertArrayEquals(doubleArray, array.metadata().attributes().getDoubleArray("double_array")); - Assertions.assertArrayEquals(floatArray, array.metadata().attributes().getFloatArray("float_array")); - Assertions.assertEquals("value", array.metadata().attributes().getAttributes("nested").getString("element")); - Assertions.assertArrayEquals( - new Attributes[]{ - new Attributes() {{ put("a", 1); }}, - new Attributes() {{ put("b", 2); }} - }, - array.metadata().attributes().getArray("array_of_attributes", Attributes.class) - ); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); - - Array arrayOpened = Array.open(storeHandle); - Assertions.assertEquals("stringvalue", arrayOpened.metadata().attributes().getString("string")); - Assertions.assertEquals(42, arrayOpened.metadata().attributes().getInt("int")); - Assertions.assertEquals(0.5, arrayOpened.metadata().attributes().getFloat("float")); - Assertions.assertEquals(3.14, arrayOpened.metadata().attributes().getDouble("double")); - Assertions.assertTrue(arrayOpened.metadata().attributes().getBoolean("boolean")); - assertListEquals(list, arrayOpened.metadata().attributes().getList("list")); - Assertions.assertArrayEquals(intArray, arrayOpened.metadata().attributes().getIntArray("int_array")); - Assertions.assertArrayEquals(longArray, arrayOpened.metadata().attributes().getLongArray("long_array")); - Assertions.assertArrayEquals(doubleArray, arrayOpened.metadata().attributes().getDoubleArray("double_array")); - Assertions.assertArrayEquals(floatArray, arrayOpened.metadata().attributes().getFloatArray("float_array")); - Assertions.assertEquals("value", arrayOpened.metadata().attributes().getAttributes("nested").getString("element")); - Assertions.assertArrayEquals( - new Attributes[]{ - new Attributes() {{ put("a", 1); }}, - new Attributes() {{ put("b", 2); }} - }, - arrayOpened.metadata().attributes().getArray("array_of_attributes", Attributes.class) - ); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); - } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 43a9c1af..fac8750f 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,5 +1,6 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; @@ -512,7 +513,7 @@ public void testOpenOverloads() throws ZarrException, IOException { public void testGroup() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(fsStore.resolve("testgroup")); @@ -553,7 +554,7 @@ public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(storeHandle, new GroupMetadata(attributes)); @@ -568,39 +569,4 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } - - @Test - public void testAttributes() throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); - ArrayMetadata arrayMetadata = Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - .withAttributes(new HashMap() {{ - put("string", "stringvalue"); - put("int", 42); - put("float", 3.14); - put("boolean", true); - put("array", new int[]{1, 2, 3}); - }}) - .build(); - - Array array = Array.create(storeHandle, arrayMetadata); - Assertions.assertEquals("stringvalue", array.metadata().attributes.get("string")); - Assertions.assertEquals(42, array.metadata().attributes.get("int")); - Assertions.assertEquals(3.14, array.metadata().attributes.get("float")); - Assertions.assertEquals(true, array.metadata().attributes.get("boolean")); - Assertions.assertArrayEquals(new int[]{1, 2, 3}, (int[]) array.metadata().attributes.get("array")); - - Array array2 = Array.open(storeHandle); - Assertions.assertEquals("stringvalue", array2.metadata().attributes.get("string")); - Assertions.assertEquals(42, array2.metadata().attributes.get("int")); - Assertions.assertEquals(3.14, array2.metadata().attributes.get("float")); - Assertions.assertEquals(true, array2.metadata().attributes.get("boolean")); - Assertions.assertArrayEquals(new int[]{1, 2, 3}, (int[])(array2.metadata().attributes.get("array"))); - } - - - - } From 0ab59f6126d6fc77798b05bc9c7596b04cf3fd4d Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 23 Oct 2025 11:19:44 +0200 Subject: [PATCH 03/14] implement setAttributes and updateAttributes for v2 --- .../dev/zarr/zarrjava/core/Attributes.java | 10 ++ src/main/java/dev/zarr/zarrjava/v2/Array.java | 56 +++++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 9 +- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 2 +- .../dev/zarr/zarrjava/ZarrAttributesTest.java | 139 ------------------ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 82 ++++++++++- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 64 +++++++- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 62 +++++++- 8 files changed, 264 insertions(+), 160 deletions(-) delete mode 100644 src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java index fd097c41..02690565 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Attributes.java +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -17,6 +17,16 @@ public Attributes(Map attributes) { super(attributes); } + public Attributes add(String s, Object o){ + this.put(s, o); + return this; + } + + public Attributes delete(String s){ + this.remove(s); + return this; + } + public boolean getBoolean(String key) { Object value = this.get(key); if (value instanceof Boolean) { diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 184fae31..5deeadc3 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -177,6 +177,62 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } + private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { + return Array.create(storeHandle, newArrayMetadata, true); + } + + /** + * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or + * deleted. This method returns a new instance of the Zarr array class and the old instance + * becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape) throws ZarrException, IOException { + //TODO: test + if (newShape.length != metadata.ndim()) { + throw new IllegalArgumentException( + "'newShape' needs to have rank '" + metadata.ndim() + "'."); + } + + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withShape(newShape) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { + ArrayMetadata newArrayMetadata = + ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withAttributes(newAttributes) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Updates the attributes of the Zarr array. It provides a callback that gets the current + * attributes as input and needs to return the new set of attributes. The attributes in the + * callback may be mutated. This method overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param attributeMapper the callback that is used to construct the new attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + @Override public String toString() { return String.format("", storeHandle, diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index bf11c26c..69fc1409 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -183,11 +183,7 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat } private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); - return new Array(storeHandle, newArrayMetadata); + return Array.create(storeHandle, newArrayMetadata, true); } /** @@ -238,8 +234,7 @@ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOExc * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(new Attributes(metadata.attributes) { - })); + return setAttributes(attributeMapper.apply(metadata.attributes)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 7a97dd05..eb500326 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -150,7 +150,7 @@ public Attributes attributes() throws ZarrException { if (attributes == null) { throw new ZarrException("Array attributes have not been set."); } - return new Attributes(attributes); //todo change attributes to Attributes type + return attributes; } public static Optional getShardingIndexedCodec(Codec[] codecs) { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java b/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java deleted file mode 100644 index f2e5da25..00000000 --- a/src/test/java/dev/zarr/zarrjava/ZarrAttributesTest.java +++ /dev/null @@ -1,139 +0,0 @@ -package dev.zarr.zarrjava; - -import dev.zarr.zarrjava.core.Attributes; -import dev.zarr.zarrjava.store.FilesystemStore; -import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v2.Array; -import dev.zarr.zarrjava.v2.ArrayMetadata; -import dev.zarr.zarrjava.v2.DataType; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -public class ZarrAttributesTest extends ZarrTest { - - ArrayList listAttribute = new ArrayList() { - { - add(1); - add(2.0d); - add("string"); - } - }; - int[] intArrayAttribute = new int[]{1, 2, 3}; - long[] longArrayAttribute = new long[]{1, 2, 3}; - double[] doubleArrayAttribute = new double[]{1.0, 2.0, 3.0}; - float[] floatArrayAttribute = new float[]{1.0f, 2.0f, 3.0f}; - Attributes testAttributes = new Attributes() {{ - put("string", "stringvalue"); - put("int", 42); - put("float", 0.5f); - put("double", 3.14); - put("boolean", true); - put("list", listAttribute); - put("int_array", intArrayAttribute); - put("long_array", longArrayAttribute); - put("double_array", doubleArrayAttribute); - put("float_array", floatArrayAttribute); - put("nested", new Attributes() {{ - put("element", "value"); - }}); - put("array_of_attributes", new Attributes[]{ - new Attributes() {{ - put("a", 1); - }}, - new Attributes() {{ - put("b", 2); - }} - }); - }}; - - static void assertListEquals(List a, List b) { - Assertions.assertEquals(a.size(), b.size()); - for (int i = 0; i < a.size(); i++) { - Object aval = a.get(i); - Object bval = b.get(i); - if (aval instanceof List && bval instanceof List) { - assertListEquals((List) aval, (List) bval); - } else { - Assertions.assertEquals(aval, bval); - } - } - } - - void assertContainsTestAttributes(Attributes attributes) throws ZarrException { - Assertions.assertEquals("stringvalue", attributes.getString("string")); - Assertions.assertEquals(42, attributes.getInt("int")); - Assertions.assertEquals(0.5, attributes.getFloat("float")); - Assertions.assertEquals(3.14, attributes.getDouble("double")); - Assertions.assertTrue(attributes.getBoolean("boolean")); - assertListEquals(listAttribute, attributes.getList("list")); - Assertions.assertArrayEquals(intArrayAttribute, attributes.getIntArray("int_array")); - Assertions.assertArrayEquals(longArrayAttribute, attributes.getLongArray("long_array")); - Assertions.assertArrayEquals(doubleArrayAttribute, attributes.getDoubleArray("double_array")); - Assertions.assertArrayEquals(floatArrayAttribute, attributes.getFloatArray("float_array")); - Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); - Assertions.assertArrayEquals( - new Attributes[]{ - new Attributes() {{ put("a", 1); }}, - new Attributes() {{ put("b", 2); }} - }, - attributes.getArray("array_of_attributes", Attributes.class) - ); - } - - @Test - public void testAttributesV2() throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); - - ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunks(5, 5) - .putAttribute("specific", "attribute") - .withAttributes(testAttributes) - .withAttributes(new Attributes() {{ - put("another", "attribute"); - }}) - .build(); - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(storeHandle, arrayMetadata); - assertContainsTestAttributes(array.metadata().attributes()); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); - - dev.zarr.zarrjava.v2.Array arrayOpened = Array.open(storeHandle); - assertContainsTestAttributes(array.metadata().attributes()); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); - } - - @Test - public void testAttributesV3() throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); - - dev.zarr.zarrjava.v3.ArrayMetadata arrayMetadata = dev.zarr.zarrjava.v3.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dev.zarr.zarrjava.v3.DataType.UINT8) - .withChunkShape(5, 5) - .putAttribute("specific", "attribute") - .withAttributes(testAttributes) - .withAttributes(new Attributes() {{ - put("another", "attribute"); - }}) - - .build(); - - dev.zarr.zarrjava.v3.Array array = dev.zarr.zarrjava.v3.Array.create(storeHandle, arrayMetadata); - assertContainsTestAttributes(array.metadata().attributes()); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); - Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); - - dev.zarr.zarrjava.v3.Array arrayOpened = dev.zarr.zarrjava.v3.Array.open(storeHandle); - assertContainsTestAttributes(arrayOpened.metadata().attributes()); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); - Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); - } -} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b40d516d..f23a5d8c 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,13 +1,16 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.stream.Stream; public class ZarrTest { @@ -30,4 +33,81 @@ public static void clearTestoutputFolder() throws IOException { } Files.createDirectory(TESTOUTPUT); } + + protected void assertListEquals(List a, List b) { + Assertions.assertEquals(a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + Object aval = a.get(i); + Object bval = b.get(i); + if (aval instanceof List && bval instanceof List) { + assertListEquals((List) aval, (List) bval); + } else { + Assertions.assertEquals(aval, bval); + } + } + } + + protected Attributes defaultTestAttributes() { + return new Attributes() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 0.5f); + put("double", 3.14); + put("boolean", true); + put("list", new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }); + put("int_array", new int[]{1, 2, 3}); + put("long_array", new long[]{1, 2, 3}); + put("double_array", new double[]{1.0, 2.0, 3.0}); + put("float_array", new float[]{1.0f, 2.0f, 3.0f}); + put("nested", new Attributes() {{ + put("element", "value"); + }}); + put("array_of_attributes", new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }); + }}; + } + + protected void assertContainsTestAttributes(Attributes attributes) throws ZarrException { + Assertions.assertEquals("stringvalue", attributes.getString("string")); + Assertions.assertEquals(42, attributes.getInt("int")); + Assertions.assertEquals(0.5, attributes.getFloat("float")); + Assertions.assertEquals(3.14, attributes.getDouble("double")); + Assertions.assertTrue(attributes.getBoolean("boolean")); + assertListEquals(new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }, attributes.getList("list")); + Assertions.assertArrayEquals(new int[]{1, 2, 3}, attributes.getIntArray("int_array")); + Assertions.assertArrayEquals(new long[]{1, 2, 3}, attributes.getLongArray("long_array")); + Assertions.assertArrayEquals(new double[]{1, 2, 3}, attributes.getDoubleArray("double_array")); + Assertions.assertArrayEquals(new float[]{1, 2, 3}, attributes.getFloatArray("float_array")); + Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }, + attributes.getArray("array_of_attributes", Attributes.class) + ); + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index dc0f4fa8..160fa67a 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -3,11 +3,7 @@ import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v2.Array; -import dev.zarr.zarrjava.v2.ArrayMetadata; -import dev.zarr.zarrjava.v2.DataType; -import dev.zarr.zarrjava.v2.Group; -import dev.zarr.zarrjava.v2.Node; +import dev.zarr.zarrjava.v2.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -15,13 +11,10 @@ import org.junit.jupiter.params.provider.ValueSource; import java.io.IOException; -import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @@ -260,4 +253,59 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + } + + @Test + public void testSetAndUpdateAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + + // add attribute + array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index fac8750f..81a87089 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,8 +1,6 @@ package dev.zarr.zarrjava; import dev.zarr.zarrjava.core.Attributes; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import com.fasterxml.jackson.databind.JsonMappingException; import dev.zarr.zarrjava.store.*; @@ -10,8 +8,7 @@ import dev.zarr.zarrjava.v3.Node; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import dev.zarr.zarrjava.v3.codec.core.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -569,4 +566,61 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(arrayOpened.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + } + + + @Test + public void testSetAndUpdateAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + + // add attribute + array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } } From 526a16c039bbdd51c3adcf733e8f20a7362df0c4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 23 Oct 2025 17:36:54 +0200 Subject: [PATCH 04/14] add test for resize array --- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 31 ++++++++++++++++++- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 28 +++++++++++++++++ 2 files changed, 58 insertions(+), 1 deletion(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 160fa67a..f2bf3060 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -15,6 +15,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @@ -281,7 +282,7 @@ public void testAttributes() throws IOException, ZarrException { @Test public void testSetAndUpdateAttributes() throws IOException, ZarrException { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV2"); ArrayMetadata arrayMetadata = Array.metadataBuilder() .withShape(10, 10) @@ -308,4 +309,32 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { assertContainsTestAttributes(array.metadata().attributes()); } + + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 81a87089..11eb73e0 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -623,4 +623,32 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { assertContainsTestAttributes(array.metadata().attributes()); } + + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } } From baeffa904028e78ee93f067928f180e2984c147f Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 26 Oct 2025 17:08:37 +0100 Subject: [PATCH 05/14] fix array.setAttributes --- src/main/java/dev/zarr/zarrjava/core/Attributes.java | 9 ++++++++- src/main/java/dev/zarr/zarrjava/v2/Array.java | 3 +-- .../java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java | 7 +++++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 4 +--- .../java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java | 8 +++++++- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 2 ++ src/test/java/dev/zarr/zarrjava/ZarrV2Test.java | 3 +++ src/test/java/dev/zarr/zarrjava/ZarrV3Test.java | 3 +++ 8 files changed, 32 insertions(+), 7 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java index 02690565..5156eef0 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Attributes.java +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -6,6 +6,7 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.function.Function; public class Attributes extends HashMap { @@ -13,6 +14,12 @@ public Attributes() { super(); } + public Attributes (Function attributeMapper) { + super(); + attributeMapper.apply(this); + } + + public Attributes(Map attributes) { super(attributes); } @@ -90,7 +97,7 @@ public Attributes getAttributes(String key) { public T[] getArray(String key, Class clazz) throws ZarrException { Object value = this.get(key); - if (value instanceof Object[] && ( ((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { + if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { return (T[]) value; } if (value instanceof List) { diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 5deeadc3..237e67b5 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -191,7 +191,6 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array resize(long[] newShape) throws ZarrException, IOException { - //TODO: test if (newShape.length != metadata.ndim()) { throw new IllegalArgumentException( "'newShape' needs to have rank '" + metadata.ndim() + "'."); @@ -213,7 +212,7 @@ public Array resize(long[] newShape) throws ZarrException, IOException { */ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata) + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) .withAttributes(newAttributes) .build(); return writeMetadata(newArrayMetadata); diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 897e4e62..7f27567f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -24,6 +24,10 @@ protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.chunks = arrayMetadata.chunks; @@ -33,6 +37,9 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.fillValue = arrayMetadata.parsedFillValue; builder.filters = arrayMetadata.filters; builder.compressor = arrayMetadata.compressor; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 69fc1409..e30cd310 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -12,8 +12,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -217,7 +215,7 @@ public Array resize(long[] newShape) throws ZarrException, IOException { */ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata) + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) .withAttributes(newAttributes) .build(); return writeMetadata(newArrayMetadata); diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 9def15f2..9f124666 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -35,6 +35,10 @@ protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.dataType = arrayMetadata.dataType; @@ -42,9 +46,11 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; builder.fillValue = arrayMetadata.parsedFillValue; builder.codecs = arrayMetadata.codecs; - builder.attributes = arrayMetadata.attributes; builder.dimensionNames = arrayMetadata.dimensionNames; builder.storageTransformers = arrayMetadata.storageTransformers; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index f23a5d8c..8a6309d2 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -65,6 +65,7 @@ protected Attributes defaultTestAttributes() { put("long_array", new long[]{1, 2, 3}); put("double_array", new double[]{1.0, 2.0, 3.0}); put("float_array", new float[]{1.0f, 2.0f, 3.0f}); + put("boolean_array", new boolean[]{true, false, true}); put("nested", new Attributes() {{ put("element", "value"); }}); @@ -96,6 +97,7 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx Assertions.assertArrayEquals(new long[]{1, 2, 3}, attributes.getLongArray("long_array")); Assertions.assertArrayEquals(new double[]{1, 2, 3}, attributes.getDoubleArray("double_array")); Assertions.assertArrayEquals(new float[]{1, 2, 3}, attributes.getFloatArray("float_array")); + Assertions.assertArrayEquals(new boolean[]{true, false, true}, attributes.getBooleanArray("boolean_array")); Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); Assertions.assertArrayEquals( new Attributes[]{ diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index f2bf3060..932285a1 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -288,12 +288,15 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { .withShape(10, 10) .withDataType(DataType.UINT8) .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.add("some", "value"))) .build(); Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); array.setAttributes(defaultTestAttributes()); array = Array.open(storeHandle); assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); // add attribute array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 11eb73e0..613d7d8e 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -602,12 +602,15 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { .withShape(10, 10) .withDataType(DataType.UINT8) .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.add("some", "value"))) .build(); Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); array.setAttributes(defaultTestAttributes()); array = Array.open(storeHandle); assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); // add attribute array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); From 07b02c543f4972f228fe1fcb9e754e029f60a020 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 27 Oct 2025 14:55:35 +0100 Subject: [PATCH 06/14] rename attributes.add to set --- src/main/java/dev/zarr/zarrjava/core/Attributes.java | 2 +- src/test/java/dev/zarr/zarrjava/ZarrV2Test.java | 4 ++-- src/test/java/dev/zarr/zarrjava/ZarrV3Test.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java index 5156eef0..bc1cfe27 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Attributes.java +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -24,7 +24,7 @@ public Attributes(Map attributes) { super(attributes); } - public Attributes add(String s, Object o){ + public Attributes set(String s, Object o){ this.put(s, o); return this; } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 932285a1..7fd1852b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -288,7 +288,7 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { .withShape(10, 10) .withDataType(DataType.UINT8) .withChunks(5, 5) - .withAttributes(new Attributes(b -> b.add("some", "value"))) + .withAttributes(new Attributes(b -> b.set("some", "value"))) .build(); Array array = Array.create(storeHandle, arrayMetadata); @@ -299,7 +299,7 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { Assertions.assertNull(array.metadata().attributes().get("some")); // add attribute - array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); array = Array.open(storeHandle); Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 613d7d8e..10569e12 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -602,7 +602,7 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { .withShape(10, 10) .withDataType(DataType.UINT8) .withChunkShape(5, 5) - .withAttributes(new Attributes(b -> b.add("some", "value"))) + .withAttributes(new Attributes(b -> b.set("some", "value"))) .build(); Array array = Array.create(storeHandle, arrayMetadata); @@ -613,7 +613,7 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException { Assertions.assertNull(array.metadata().attributes().get("some")); // add attribute - array = array.updateAttributes(b -> b.add("new_attribute", "new_value")); + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); array = Array.open(storeHandle); Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); From f018f0aa0762c83320ab1f300372bb76807068c7 Mon Sep 17 00:00:00 2001 From: Dominik Lindner Date: Mon, 10 Nov 2025 13:21:11 +0000 Subject: [PATCH 07/14] Load v2 GroupMetadata attributes (#32) * Load v2 GroupMetadata attributes * Update src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java Co-authored-by: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> * Update src/main/java/dev/zarr/zarrjava/v3/Group.java Co-authored-by: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> --------- Co-authored-by: Hannes Spitz <44113112+brokkoli71@users.noreply.github.com> --- .../java/dev/zarr/zarrjava/core/Group.java | 2 ++ .../dev/zarr/zarrjava/core/GroupMetadata.java | 10 ++++++- src/main/java/dev/zarr/zarrjava/v2/Group.java | 26 ++++++++++++++++--- .../dev/zarr/zarrjava/v2/GroupMetadata.java | 19 ++++++++++++-- src/main/java/dev/zarr/zarrjava/v3/Group.java | 6 ++++- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 9 +++++++ 6 files changed, 64 insertions(+), 8 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index 5cff4774..b7cc83d8 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -83,4 +83,6 @@ public Node[] listAsArray() { return nodeStream.toArray(Node[]::new); } } + + public abstract GroupMetadata metadata(); } diff --git a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java index 695bfdff..e7bd6ebb 100644 --- a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java @@ -1,3 +1,11 @@ package dev.zarr.zarrjava.core; -public abstract class GroupMetadata {} +import javax.annotation.Nonnull; + +import dev.zarr.zarrjava.ZarrException; + +public abstract class GroupMetadata { + + public @Nonnull abstract Attributes attributes() throws ZarrException; + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 38c0b946..73d07a39 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -24,10 +24,17 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe } public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { - StoreHandle metadataHandle = storeHandle.resolve(ZGROUP); - ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, makeObjectMapper() - .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + ObjectMapper mapper = makeObjectMapper(); + GroupMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZGROUP).readNonNull()), + GroupMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + dev.zarr.zarrjava.core.Attributes.class + ); + return new Group(storeHandle, metadata); } public static Group open(Path path) throws IOException { @@ -44,6 +51,12 @@ public static Group create( ObjectMapper objectMapper = makeObjectMapper(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZGROUP).set(metadataBytes); + if (groupMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectMapper.writeValueAsBytes(groupMetadata.attributes)); + attrsHandle.set(attrsBytes); + } return new Group(storeHandle, groupMetadata); } @@ -87,4 +100,9 @@ public Array createArray(String key, Function", storeHandle); } + + @Override + public dev.zarr.zarrjava.core.GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java index e32f1065..b199f2c2 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -1,8 +1,13 @@ package dev.zarr.zarrjava.v2; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -10,18 +15,28 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @JsonProperty("zarr_format") public final int zarrFormat = ZARR_FORMAT; + @Nullable + @JsonIgnore + public Attributes attributes; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public GroupMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "attributes", required = false) @Nullable Attributes attributes ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); } + this.attributes = attributes; } public GroupMetadata() throws ZarrException { - this(ZARR_FORMAT); + this(ZARR_FORMAT, null); } + @Override + public @Nonnull Attributes attributes() throws ZarrException { + return attributes; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index c5e4cbc9..ced97264 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -10,7 +10,6 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Map; import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -136,4 +135,9 @@ public Group updateAttributes(Function attributeMapper) public String toString() { return String.format("", storeHandle); } + + @Override + public GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index 5792213b..d3f870c5 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -5,6 +5,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; +import javax.annotation.Nonnull; import javax.annotation.Nullable; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -43,4 +44,12 @@ public GroupMetadata( public static GroupMetadata defaultValue() throws ZarrException { return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } + return attributes; + } } From 09aebaa5b02aa4eb62c5c39d78f8374581b1e309 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 10 Nov 2025 15:33:09 +0100 Subject: [PATCH 08/14] add Group.create with Attributes --- src/main/java/dev/zarr/zarrjava/v2/Group.java | 15 ++++++++++++++- .../java/dev/zarr/zarrjava/v2/GroupMetadata.java | 7 +++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 73d07a39..2b91d34c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -64,14 +65,26 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, return create(storeHandle, new GroupMetadata()); } + public static Group create(@Nonnull StoreHandle storeHandle, Attributes attributes) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata(attributes)); + } + public static Group create(Path path) throws IOException, ZarrException { return create(new StoreHandle(new FilesystemStore(path))); } + public static Group create(Path path, Attributes attributes) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), attributes); + } + public static Group create(String path) throws IOException, ZarrException { return create(Paths.get(path)); } + public static Group create(String path, Attributes attributes) throws IOException, ZarrException { + return create(Paths.get(path), attributes); + } + @Nullable public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); @@ -102,7 +115,7 @@ public String toString() { } @Override - public dev.zarr.zarrjava.core.GroupMetadata metadata() { + public GroupMetadata metadata() { return metadata; } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java index b199f2c2..e0e6c00e 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -35,8 +35,15 @@ public GroupMetadata() throws ZarrException { this(ZARR_FORMAT, null); } + public GroupMetadata(Attributes attributes) throws ZarrException { + this(ZARR_FORMAT, attributes); + } + @Override public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } return attributes; } } From db96a07140493a9935b3996517b2052e80d9bbbe Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 10 Nov 2025 15:33:19 +0100 Subject: [PATCH 09/14] add tests for v2 group attributes and --- src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java | 3 ++- src/test/java/dev/zarr/zarrjava/ZarrV2Test.java | 4 ++++ src/test/python-scripts/zarr_python_group_v2.py | 2 ++ 3 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 4a17d748..c61b2c94 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -404,7 +404,7 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException public void testGroupReadWriteV2() throws Exception { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); - Group group = Group.create(storeHandle); + Group group = Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.INT32; dev.zarr.zarrjava.v2.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder .withShape(16, 16, 16) @@ -417,6 +417,7 @@ public void testGroupReadWriteV2() throws Exception { run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); Group group2 = Group.open(storeHandle2); + Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); Group subgroup = (Group) group2.get("group2"); Assertions.assertNotNull(subgroup); dev.zarr.zarrjava.v2.Array array2 = (dev.zarr.zarrjava.v2.Array) subgroup.get("array2"); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 7fd1852b..508c487b 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -252,6 +252,10 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + + Group.create(storeHandleString, new Attributes(b -> b.set("some", "value"))); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zattrs"))); } @Test diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group_v2.py index 5052ddf0..8070f4c2 100644 --- a/src/test/python-scripts/zarr_python_group_v2.py +++ b/src/test/python-scripts/zarr_python_group_v2.py @@ -11,12 +11,14 @@ expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=2) +assert g.attrs['attr'] == 'value' a = g['group']['array'] read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" store_path_write = Path(sys.argv[2]) g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=2) +g2.attrs['attr'] = 'value' a2 = g2.create_group('group2').create_array( name='array2', shape=(16, 16, 16), From 2890eabf2a5ec0bcfeb7966d2e60f060c7c8e43c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 10 Nov 2025 16:54:46 +0100 Subject: [PATCH 10/14] test and fix v3 Group Attributes --- src/main/java/dev/zarr/zarrjava/v3/Group.java | 2 +- .../dev/zarr/zarrjava/v3/GroupMetadata.java | 2 ++ .../dev/zarr/zarrjava/ZarrPythonTests.java | 29 ++++++++++++++++++- .../python-scripts/zarr_python_group_v2.py | 8 +++-- 4 files changed, 36 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index ced97264..a09266aa 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -55,7 +55,7 @@ public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull Attributes attributes ) throws IOException, ZarrException { - return new Group(storeHandle, new GroupMetadata(attributes)); + return create(storeHandle, new GroupMetadata(attributes)); } public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index d3f870c5..ebbaf568 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.Attributes; @@ -8,6 +9,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; +@JsonIgnoreProperties(ignoreUnknown = true) public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { static final String NODE_TYPE = "group"; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index c61b2c94..4f0cd8a9 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -414,7 +414,7 @@ public void testGroupReadWriteV2() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); + run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); Group group2 = Group.open(storeHandle2); Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); @@ -426,4 +426,31 @@ public void testGroupReadWriteV2() throws Exception { Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); assertIsTestdata(result, dataType); } + + @Test + public void testGroupReadWriteV3() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); + dev.zarr.zarrjava.v3.DataType dataType = DataType.INT32; + dev.zarr.zarrjava.v3.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder + .withShape(16, 16, 16) + .withDataType(dataType) + .withChunkShape(2, 4, 8) + ); + + array.write(testdata(dataType)); + + run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); + + dev.zarr.zarrjava.v3.Group group2 = dev.zarr.zarrjava.v3.Group.open(storeHandle2); + Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); + dev.zarr.zarrjava.v3.Group subgroup = (dev.zarr.zarrjava.v3.Group) group2.get("group2"); + Assertions.assertNotNull(subgroup); + dev.zarr.zarrjava.v3.Array array2 = (dev.zarr.zarrjava.v3.Array) subgroup.get("array2"); + Assertions.assertNotNull(array2); + ucar.ma2.Array result = array2.read(); + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + assertIsTestdata(result, dataType); + } } diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group_v2.py index 8070f4c2..3aa76c73 100644 --- a/src/test/python-scripts/zarr_python_group_v2.py +++ b/src/test/python-scripts/zarr_python_group_v2.py @@ -7,17 +7,19 @@ from zarr.storage import LocalStore store_path_read = Path(sys.argv[1]) +store_path_write = Path(sys.argv[2]) +zarr_format = int(sys.argv[3]) +assert zarr_format in (2,3), f"unexpected zarr format: {zarr_format}" expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=2) +g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=zarr_format) assert g.attrs['attr'] == 'value' a = g['group']['array'] read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" -store_path_write = Path(sys.argv[2]) -g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=2) +g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=zarr_format) g2.attrs['attr'] = 'value' a2 = g2.create_group('group2').create_array( name='array2', From bc1cfc2a33ea142013e40ecdf79a8c32b1a06866 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 10 Nov 2025 17:24:25 +0100 Subject: [PATCH 11/14] dont ignore all unknown group metadata but add consolidatedMetadata --- src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index ebbaf568..e7618736 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -9,7 +9,6 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; -@JsonIgnoreProperties(ignoreUnknown = true) public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { static final String NODE_TYPE = "group"; @@ -18,6 +17,8 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { public final int zarrFormat = ZARR_FORMAT; @JsonProperty("node_type") public final String nodeType = "group"; + @JsonProperty("consolidated_metadata") + public final String consolidatedMetadata = null; @Nullable public final Attributes attributes; From 13ccf1a4951ef729271c887068893717948db44a Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 10 Nov 2025 17:25:43 +0100 Subject: [PATCH 12/14] rename zarr_python_group_v2.py to zarr_python_group.py for consistency --- src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java | 4 ++-- .../{zarr_python_group_v2.py => zarr_python_group.py} | 0 2 files changed, 2 insertions(+), 2 deletions(-) rename src/test/python-scripts/{zarr_python_group_v2.py => zarr_python_group.py} (100%) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 4f0cd8a9..19792eaa 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -414,7 +414,7 @@ public void testGroupReadWriteV2() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); + run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); Group group2 = Group.open(storeHandle2); Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); @@ -441,7 +441,7 @@ public void testGroupReadWriteV3() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); + run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); dev.zarr.zarrjava.v3.Group group2 = dev.zarr.zarrjava.v3.Group.open(storeHandle2); Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group.py similarity index 100% rename from src/test/python-scripts/zarr_python_group_v2.py rename to src/test/python-scripts/zarr_python_group.py From f459a5287813d4abff9a2cc756fb5937177b0dc2 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Thu, 13 Nov 2025 09:50:45 +0100 Subject: [PATCH 13/14] add group attributes unit tests --- .../java/dev/zarr/zarrjava/ZarrV2Test.java | 18 ++++++++++++--- .../java/dev/zarr/zarrjava/ZarrV3Test.java | 22 +++++++++++++++---- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 508c487b..74556ecb 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -38,8 +38,6 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE Assertions.assertEquals(0, outArray.getByte(0)); } - - @ParameterizedTest @CsvSource({ "BOOL", "FLOAT64" @@ -48,7 +46,7 @@ public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrExc String arrayname = dt == DataType.BOOL ? "bool" : "double"; StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); Array array = Array.open(storeHandle); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); + array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); Assertions.assertEquals(dt, array.metadata().dataType); } @@ -344,4 +342,18 @@ public void testResizeArray() throws IOException, ZarrException { Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + + @Test + public void testGroupAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2"); + + Group group = Group.create(storeHandle, new Attributes() {{ + put("group_attr", "group_value"); + }}); + + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + + group = Group.open(storeHandle); + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 10569e12..202ff7f1 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -556,16 +556,17 @@ public void testCreateGroup() throws ZarrException, IOException { Group group = Group.create(storeHandle, new GroupMetadata(attributes)); Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); group = Group.create(storeHandlePath, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); group = Group.create(storeHandleString, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); } + @Test public void testAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); @@ -593,7 +594,6 @@ public void testAttributes() throws IOException, ZarrException { Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); } - @Test public void testSetAndUpdateAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV3"); @@ -654,4 +654,18 @@ public void testResizeArray() throws IOException, ZarrException { Arrays.fill(expectedData, 1); Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + + @Test + public void testGroupAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3"); + + Group group = Group.create(storeHandle, new Attributes() {{ + put("group_attr", "group_value"); + }}); + + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + + group = Group.open(storeHandle); + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + } } From 004838d354bc61aed65f79d4d47ff4883703cd83 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Fri, 21 Nov 2025 10:57:56 +0100 Subject: [PATCH 14/14] fix testoutput paths for testGroupReadWrite --- src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 05457f2b..1a6faade 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -402,8 +402,8 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException @Test public void testGroupReadWriteV2() throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); - StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV2", "write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV2", "read"); Group group = Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.INT32; dev.zarr.zarrjava.v2.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder @@ -429,8 +429,8 @@ public void testGroupReadWriteV2() throws Exception { @Test public void testGroupReadWriteV3() throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); - StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV3", "write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV3", "read"); dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); dev.zarr.zarrjava.v3.DataType dataType = DataType.INT32; dev.zarr.zarrjava.v3.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder