diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index 03b2e85f..154d6692 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -43,6 +43,8 @@ public int ndim() { public abstract Object parsedFillValue(); + public @Nonnull abstract Attributes attributes() throws ZarrException; + public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) throws ZarrException { if (fillValue == null) { diff --git a/src/main/java/dev/zarr/zarrjava/core/Attributes.java b/src/main/java/dev/zarr/zarrjava/core/Attributes.java new file mode 100644 index 00000000..bc1cfe27 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/Attributes.java @@ -0,0 +1,241 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.ZarrException; + +import javax.annotation.Nonnull; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.function.Function; + +public class Attributes extends HashMap { + + public Attributes() { + super(); + } + + public Attributes (Function attributeMapper) { + super(); + attributeMapper.apply(this); + } + + + public Attributes(Map attributes) { + super(attributes); + } + + public Attributes set(String s, Object o){ + this.put(s, o); + return this; + } + + public Attributes delete(String s){ + this.remove(s); + return this; + } + + public boolean getBoolean(String key) { + Object value = this.get(key); + if (value instanceof Boolean) { + return (Boolean) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a Boolean"); + } + + public int getInt(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).intValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Integer"); + } + + public double getDouble(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).doubleValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Double"); + } + + public float getFloat(String key) { + Object value = this.get(key); + if (value instanceof Number) { + return ((Number) value).floatValue(); + } + throw new IllegalArgumentException("Value for key " + key + " is not a Float"); + } + + @Nonnull + public String getString(String key) { + Object value = this.get(key); + if (value instanceof String) { + return (String) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a String"); + } + + @Nonnull + public List getList(String key) { + Object value = this.get(key); + if (value instanceof List) { + return (List) value; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List"); + } + + public Attributes getAttributes(String key) { + Object value = this.get(key); + if (value instanceof Attributes) { + return (Attributes) value; + } + if (value instanceof Map) { + return new Attributes((Map) value); + } + throw new IllegalArgumentException("Value for key " + key + " is not an Attributes object"); + } + + public T[] getArray(String key, Class clazz) throws ZarrException { + Object value = this.get(key); + if (value instanceof Object[] && (((Object[]) value).length == 0 || clazz.isInstance(((Object[]) value)[0]) )) { + return (T[]) value; + } + if (value instanceof List) { + List list = (List) value; + @SuppressWarnings("unchecked") + T[] array = (T[]) java.lang.reflect.Array.newInstance(clazz, list.size()); + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (clazz.isInstance(elem)) { + array[i] = clazz.cast(elem); + } else { + // Try to find a constructor that takes the element's class + java.lang.reflect.Constructor matched = null; + for (java.lang.reflect.Constructor c : clazz.getConstructors()) { + Class[] params = c.getParameterTypes(); + if (params.length == 1 && params[0].isAssignableFrom(elem.getClass())) { + matched = c; + break; + } + } + if (matched != null) { + try { + array[i] = (T) matched.newInstance(elem); + } catch (Exception e) { + throw new ZarrException("Failed to convert element at index " + i + " to type " + clazz.getName(), e); + } + } else { + throw new IllegalArgumentException("Element at index " + i + " is not of type " + clazz.getName() + " and no suitable constructor found for conversion of type " + elem.getClass().getName()); + } + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a List or array of type " + clazz.getName()); +} + + public int[] getIntArray(String key) { + Object value = this.get(key); + if (value instanceof int[]) { + return (int[]) value; + } + if (value instanceof List) { + List list = (List) value; + int[] array = new int[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).intValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not an int array or List"); + } + + public long[] getLongArray(String key) { + Object value = this.get(key); + if (value instanceof long[]) { + return (long[]) value; + } + if (value instanceof List) { + List list = (List) value; + long[] array = new long[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).longValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a long array or List"); + } + + public double[] getDoubleArray(String key) { + Object value = this.get(key); + if (value instanceof double[]) { + return (double[]) value; + } + if (value instanceof List) { + List list = (List) value; + double[] array = new double[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).doubleValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a double array or List"); + } + + public float[] getFloatArray(String key) { + Object value = this.get(key); + if (value instanceof float[]) { + return (float[]) value; + } + if (value instanceof List) { + List list = (List) value; + float[] array = new float[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Number) { + array[i] = ((Number) elem).floatValue(); + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Number"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a float array or List"); + } + + public boolean[] getBooleanArray(String key) { + Object value = this.get(key); + if (value instanceof boolean[]) { + return (boolean[]) value; + } + if (value instanceof List) { + List list = (List) value; + boolean[] array = new boolean[list.size()]; + for (int i = 0; i < list.size(); i++) { + Object elem = list.get(i); + if (elem instanceof Boolean) { + array[i] = (Boolean) elem; + } else { + throw new IllegalArgumentException("Element at index " + i + " is not a Boolean"); + } + } + return array; + } + throw new IllegalArgumentException("Value for key " + key + " is not a boolean array or List"); + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/core/Group.java b/src/main/java/dev/zarr/zarrjava/core/Group.java index 5cff4774..b7cc83d8 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Group.java +++ b/src/main/java/dev/zarr/zarrjava/core/Group.java @@ -83,4 +83,6 @@ public Node[] listAsArray() { return nodeStream.toArray(Node[]::new); } } + + public abstract GroupMetadata metadata(); } diff --git a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java index 695bfdff..e7bd6ebb 100644 --- a/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/GroupMetadata.java @@ -1,3 +1,11 @@ package dev.zarr.zarrjava.core; -public abstract class GroupMetadata {} +import javax.annotation.Nonnull; + +import dev.zarr.zarrjava.ZarrException; + +public abstract class GroupMetadata { + + public @Nonnull abstract Attributes attributes() throws ZarrException; + +} diff --git a/src/main/java/dev/zarr/zarrjava/core/Node.java b/src/main/java/dev/zarr/zarrjava/core/Node.java index 0b511936..fbac2ba3 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Node.java +++ b/src/main/java/dev/zarr/zarrjava/core/Node.java @@ -13,6 +13,7 @@ public interface Node { String ZARR_JSON = "zarr.json"; String ZARRAY = ".zarray"; + String ZATTRS = ".zattrs"; String ZGROUP = ".zgroup"; /** diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 1cbfb056..e66c20e7 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,7 +1,9 @@ package dev.zarr.zarrjava.v2; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -47,13 +49,19 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE * @throws ZarrException throws ZarrException if the Zarr array cannot be opened */ public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + ObjectMapper mapper = makeObjectMapper(); + ArrayMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), + ArrayMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + Attributes.class + ); return new Array( storeHandle, - makeObjectMapper() - .readValue( - Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), - ArrayMetadata.class - ) + metadata ); } @@ -144,6 +152,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, } ObjectWriter objectWriter = makeObjectWriter(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); + if (arrayMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectWriter.writeValueAsBytes(arrayMetadata.attributes)); + attrsHandle.set(attrsBytes); + } metadataHandle.set(metadataBytes); return new Array(storeHandle, arrayMetadata); } @@ -165,6 +179,61 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } + private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { + return Array.create(storeHandle, newArrayMetadata, true); + } + + /** + * Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or + * deleted. This method returns a new instance of the Zarr array class and the old instance + * becomes invalid. + * + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array resize(long[] newShape) throws ZarrException, IOException { + if (newShape.length != metadata.ndim()) { + throw new IllegalArgumentException( + "'newShape' needs to have rank '" + metadata.ndim() + "'."); + } + + ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata) + .withShape(newShape) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { + ArrayMetadata newArrayMetadata = + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) + .withAttributes(newAttributes) + .build(); + return writeMetadata(newArrayMetadata); + } + + /** + * Updates the attributes of the Zarr array. It provides a callback that gets the current + * attributes as input and needs to return the new set of attributes. The attributes in the + * callback may be mutated. This method overwrites and removes any existing attributes. This + * method returns a new instance of the Zarr array class and the old instance becomes invalid. + * + * @param attributeMapper the callback that is used to construct the new attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized + */ + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); + } + @Override public String toString() { return String.format("", storeHandle, diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index dabd2855..b53eee3d 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; @@ -11,6 +12,7 @@ import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; +import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -39,6 +41,10 @@ public class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { @Nullable public final Codec compressor; + @Nullable + @JsonIgnore + public Attributes attributes; + @JsonIgnore public CoreArrayMetadata coreArrayMetadata; @@ -54,6 +60,22 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator + ) throws ZarrException { + this(zarrFormat, shape, chunks, dataType, fillValue, order, filters, compressor, dimensionSeparator, null); + } + + + public ArrayMetadata( + int zarrFormat, + long[] shape, + int[] chunks, + DataType dataType, + @Nullable Object fillValue, + Order order, + @Nullable Codec[] filters, + @Nullable Codec compressor, + @Nullable Separator dimensionSeparator, + @Nullable Attributes attributes ) throws ZarrException { super(shape, fillValue, dataType); if (zarrFormat != this.zarrFormat) { @@ -78,6 +100,7 @@ public ArrayMetadata( } } this.compressor = compressor == null ? null : compressor.evolveFromCoreArrayMetadata(this.coreArrayMetadata); + this.attributes = attributes; } @@ -87,8 +110,6 @@ public int[] chunkShape() { return chunks; } - - @Override public DataType dataType() { return dataType; @@ -111,4 +132,14 @@ public ChunkKeyEncoding chunkKeyEncoding() { public Object parsedFillValue() { return parsedFillValue; } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } + + } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index ea29b165..7f27567f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -2,6 +2,7 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.core.BloscCodec; @@ -16,12 +17,17 @@ public class ArrayMetadataBuilder { Object fillValue = null; Codec[] filters = null; Codec compressor = null; + Attributes attributes = new Attributes(); protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.chunks = arrayMetadata.chunks; @@ -31,6 +37,9 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.fillValue = arrayMetadata.parsedFillValue; builder.filters = arrayMetadata.filters; builder.compressor = arrayMetadata.compressor; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } @@ -119,6 +128,19 @@ public ArrayMetadataBuilder withZlibCompressor() { return withZlibCompressor(5); } + public ArrayMetadataBuilder putAttribute(String key, Object value) { + this.attributes.put(key, value); + return this; + } + + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } + return this; + } public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new IllegalStateException("Please call `withShape` first."); @@ -138,7 +160,8 @@ public ArrayMetadata build() throws ZarrException { order, filters, compressor, - dimensionSeparator + dimensionSeparator, + attributes ); } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index c8982e13..701e7871 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -1,7 +1,9 @@ package dev.zarr.zarrjava.v2; +import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -25,10 +27,17 @@ protected Group(@Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMe } public static Group open(@Nonnull StoreHandle storeHandle) throws IOException { - StoreHandle metadataHandle = storeHandle.resolve(ZGROUP); - ByteBuffer metadataBytes = metadataHandle.readNonNull(); - return new Group(storeHandle, makeObjectMapper() - .readValue(Utils.toArray(metadataBytes), GroupMetadata.class)); + ObjectMapper mapper = makeObjectMapper(); + GroupMetadata metadata = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZGROUP).readNonNull()), + GroupMetadata.class + ); + if (storeHandle.resolve(ZATTRS).exists()) + metadata.attributes = mapper.readValue( + Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()), + dev.zarr.zarrjava.core.Attributes.class + ); + return new Group(storeHandle, metadata); } public static Group open(Path path) throws IOException { @@ -45,6 +54,12 @@ public static Group create( ObjectWriter objectWriter = makeObjectWriter(); ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZGROUP).set(metadataBytes); + if (groupMetadata.attributes != null) { + StoreHandle attrsHandle = storeHandle.resolve(ZATTRS); + ByteBuffer attrsBytes = ByteBuffer.wrap( + objectWriter.writeValueAsBytes(groupMetadata.attributes)); + attrsHandle.set(attrsBytes); + } return new Group(storeHandle, groupMetadata); } @@ -52,14 +67,26 @@ public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, return create(storeHandle, new GroupMetadata()); } + public static Group create(@Nonnull StoreHandle storeHandle, Attributes attributes) throws IOException, ZarrException { + return create(storeHandle, new GroupMetadata(attributes)); + } + public static Group create(Path path) throws IOException, ZarrException { return create(new StoreHandle(new FilesystemStore(path))); } + public static Group create(Path path, Attributes attributes) throws IOException, ZarrException { + return create(new StoreHandle(new FilesystemStore(path)), attributes); + } + public static Group create(String path) throws IOException, ZarrException { return create(Paths.get(path)); } + public static Group create(String path, Attributes attributes) throws IOException, ZarrException { + return create(Paths.get(path), attributes); + } + @Nullable public Node get(String key) throws ZarrException { StoreHandle keyHandle = storeHandle.resolve(key); @@ -88,4 +115,9 @@ public Array createArray(String key, Function", storeHandle); } + + @Override + public GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java index e32f1065..e0e6c00e 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/GroupMetadata.java @@ -1,8 +1,13 @@ package dev.zarr.zarrjava.v2; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; + import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -10,18 +15,35 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @JsonProperty("zarr_format") public final int zarrFormat = ZARR_FORMAT; + @Nullable + @JsonIgnore + public Attributes attributes; + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public GroupMetadata( - @JsonProperty(value = "zarr_format", required = true) int zarrFormat + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "attributes", required = false) @Nullable Attributes attributes ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); } + this.attributes = attributes; } public GroupMetadata() throws ZarrException { - this(ZARR_FORMAT); + this(ZARR_FORMAT, null); + } + + public GroupMetadata(Attributes attributes) throws ZarrException { + this(ZARR_FORMAT, attributes); } + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } + return attributes; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 19c82fbf..1536893c 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -11,8 +12,6 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -183,11 +182,7 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat } private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - ObjectWriter objectWriter = makeObjectWriter(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newArrayMetadata)); - storeHandle.resolve(ZARR_JSON) - .set(metadataBytes); - return new Array(storeHandle, newArrayMetadata); + return Array.create(storeHandle, newArrayMetadata, true); } /** @@ -219,9 +214,9 @@ public Array resize(long[] newShape) throws ZarrException, IOException { * @throws ZarrException throws ZarrException if the new metadata is invalid * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array setAttributes(Map newAttributes) throws ZarrException, IOException { + public Array setAttributes(Attributes newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = - ArrayMetadataBuilder.fromArrayMetadata(metadata) + ArrayMetadataBuilder.fromArrayMetadata(metadata, false) .withAttributes(newAttributes) .build(); return writeMetadata(newArrayMetadata); @@ -235,12 +230,10 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid - * @throws IOException throws IOException if the new metadata cannot be serialized + * @throws IOException throws IOException if the new metadata cannot be serialized */ - public Array updateAttributes(Function, Map> attributeMapper) - throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(new HashMap(metadata.attributes) { - })); + public Array updateAttributes(Function attributeMapper) throws ZarrException, IOException { + return setAttributes(attributeMapper.apply(metadata.attributes)); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index 36253482..eb500326 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -4,6 +4,7 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -39,7 +40,7 @@ public final class ArrayMetadata extends dev.zarr.zarrjava.core.ArrayMetadata { public final Codec[] codecs; @Nullable @JsonProperty("attributes") - public final Map attributes; + public final Attributes attributes; @Nullable @JsonProperty("dimension_names") public final String[] dimensionNames; @@ -55,7 +56,7 @@ public ArrayMetadata( Object fillValue, @Nonnull Codec[] codecs, @Nullable String[] dimensionNames, - @Nullable Map attributes, + @Nullable Attributes attributes, @Nullable Map[] storageTransformers ) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, @@ -75,7 +76,7 @@ public ArrayMetadata( @JsonProperty(value = "fill_value", required = true) Object fillValue, @Nonnull @JsonProperty(value = "codecs") Codec[] codecs, @Nullable @JsonProperty(value = "dimension_names") String[] dimensionNames, - @Nullable @JsonProperty(value = "attributes") Map attributes, + @Nullable @JsonProperty(value = "attributes") Attributes attributes, @Nullable @JsonProperty(value = "storage_transformers") Map[] storageTransformers ) throws ZarrException { super(shape, fillValue, dataType); @@ -143,6 +144,15 @@ public Object parsedFillValue() { return parsedFillValue; } + @Nonnull + @Override + public Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Array attributes have not been set."); + } + return attributes; + } + public static Optional getShardingIndexedCodec(Codec[] codecs) { return Arrays.stream(codecs).filter(codec -> codec instanceof ShardingIndexedCodec).findFirst(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 417d8ede..1da2015f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; @@ -26,7 +27,7 @@ public class ArrayMetadataBuilder { Object fillValue = 0; Codec[] codecs = new Codec[]{new BytesCodec(Endian.LITTLE)}; - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); Map[] storageTransformers = new HashMap[]{}; String[] dimensionNames = null; @@ -34,6 +35,10 @@ protected ArrayMetadataBuilder() { } protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + return fromArrayMetadata(arrayMetadata, true); + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata, boolean withAttributes) { ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.dataType = arrayMetadata.dataType; @@ -41,9 +46,11 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad builder.chunkKeyEncoding = arrayMetadata.chunkKeyEncoding; builder.fillValue = arrayMetadata.parsedFillValue; builder.codecs = arrayMetadata.codecs; - builder.attributes = arrayMetadata.attributes; builder.dimensionNames = arrayMetadata.dimensionNames; builder.storageTransformers = arrayMetadata.storageTransformers; + if (withAttributes) { + builder.attributes = arrayMetadata.attributes; + } return builder; } @@ -133,8 +140,12 @@ public ArrayMetadataBuilder putAttribute(String key, Object value) { return this; } - public ArrayMetadataBuilder withAttributes(Map attributes) { - this.attributes = attributes; + public ArrayMetadataBuilder withAttributes(Attributes attributes) { + if (this.attributes == null) { + this.attributes = attributes; + } else { + this.attributes.putAll(attributes); + } return this; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 7b025975..62a36c8e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; @@ -9,7 +10,6 @@ import java.nio.ByteBuffer; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.Map; import java.util.function.Function; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -54,9 +54,9 @@ public static Group create( public static Group create( @Nonnull StoreHandle storeHandle, - @Nonnull Map attributes + @Nonnull Attributes attributes ) throws IOException, ZarrException { - return new Group(storeHandle, new GroupMetadata(attributes)); + return create(storeHandle, new GroupMetadata(attributes)); } public static Group create(@Nonnull StoreHandle storeHandle) throws IOException, ZarrException { @@ -94,7 +94,7 @@ public Group createGroup(String key, GroupMetadata groupMetadata) return Group.create(storeHandle.resolve(key), groupMetadata); } - public Group createGroup(String key, Map attributes) + public Group createGroup(String key, Attributes attributes) throws IOException, ZarrException { return Group.create(storeHandle.resolve(key), new GroupMetadata(attributes)); } @@ -122,12 +122,12 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { return new Group(storeHandle, newGroupMetadata); } - public Group setAttributes(Map newAttributes) throws ZarrException, IOException { + public Group setAttributes(Attributes newAttributes) throws ZarrException, IOException { GroupMetadata newGroupMetadata = new GroupMetadata(newAttributes); return writeMetadata(newGroupMetadata); } - public Group updateAttributes(Function, Map> attributeMapper) + public Group updateAttributes(Function attributeMapper) throws ZarrException, IOException { return setAttributes(attributeMapper.apply(metadata.attributes)); } @@ -136,4 +136,9 @@ public Group updateAttributes(Function, Map> public String toString() { return String.format("", storeHandle); } + + @Override + public GroupMetadata metadata() { + return metadata; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java index df32c046..e7618736 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/GroupMetadata.java @@ -1,10 +1,12 @@ package dev.zarr.zarrjava.v3; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import java.util.HashMap; -import java.util.Map; +import dev.zarr.zarrjava.core.Attributes; + +import javax.annotation.Nonnull; import javax.annotation.Nullable; public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { @@ -15,11 +17,13 @@ public final class GroupMetadata extends dev.zarr.zarrjava.core.GroupMetadata { public final int zarrFormat = ZARR_FORMAT; @JsonProperty("node_type") public final String nodeType = "group"; + @JsonProperty("consolidated_metadata") + public final String consolidatedMetadata = null; @Nullable - public final Map attributes; + public final Attributes attributes; - public GroupMetadata(@Nullable Map attributes) throws ZarrException { + public GroupMetadata(@Nullable Attributes attributes) throws ZarrException { this(ZARR_FORMAT, NODE_TYPE, attributes); } @@ -27,7 +31,7 @@ public GroupMetadata(@Nullable Map attributes) throws ZarrExcept public GroupMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "node_type", required = true) String nodeType, - @Nullable @JsonProperty(value = "attributes") Map attributes + @Nullable @JsonProperty(value = "attributes") Attributes attributes ) throws ZarrException { if (zarrFormat != this.zarrFormat) { throw new ZarrException( @@ -41,6 +45,14 @@ public GroupMetadata( } public static GroupMetadata defaultValue() throws ZarrException { - return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new HashMap<>()); + return new GroupMetadata(ZARR_FORMAT, NODE_TYPE, new Attributes()); + } + + @Override + public @Nonnull Attributes attributes() throws ZarrException { + if (attributes == null) { + throw new ZarrException("Group attributes have not been set."); + } + return attributes; } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 1f2899f3..1a6faade 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -2,6 +2,7 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; +import dev.zarr.zarrjava.core.Attributes; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.v2.Group; @@ -209,7 +210,7 @@ public void testReadV3(String codec, String codecParam, DataType dataType) throw @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV3") public void testWriteV3(String codec, String codecParam, DataType dataType) throws Exception { - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testWriteV3", codec, codecParam, dataType.name()); @@ -314,7 +315,7 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, array.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); -// Assertions.assertEquals(42, array.metadata().attributes.get("answer")); + Assertions.assertEquals(42, array.metadata().attributes().get("answer")); assertIsTestdata(result, dt); } @@ -323,15 +324,15 @@ public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrj @ParameterizedTest @MethodSource("compressorAndDataTypeProviderV2") public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws Exception { -// Map attributes = new HashMap<>(); -// attributes.put("test_key", "test_value"); + Attributes attributes = new Attributes(); + attributes.put("test_key", "test_value"); StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecsWriteV2", compressor, compressorParam, dt.name()); dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) .withDataType(dt) .withChunks(2, 4, 8) -// .withAttributes(attributes) + .withAttributes(attributes) .withFillValue(0); switch (compressor) { @@ -358,7 +359,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); Assertions.assertEquals(dt, readArray.metadata().dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); -// Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Assertions.assertEquals("test_value", readArray.metadata().attributes().get("test_key")); assertIsTestdata(result, dt); //read in zarr_python @@ -401,9 +402,9 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException @Test public void testGroupReadWriteV2() throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("group_write"); - StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("group_read"); - Group group = Group.create(storeHandle); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV2", "write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV2", "read"); + Group group = Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.INT32; dev.zarr.zarrjava.v2.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder .withShape(16, 16, 16) @@ -413,9 +414,10 @@ public void testGroupReadWriteV2() throws Exception { array.write(testdata(dataType)); - run_python_script("zarr_python_group_v2.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString()); + run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 2); Group group2 = Group.open(storeHandle2); + Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); Group subgroup = (Group) group2.get("group2"); Assertions.assertNotNull(subgroup); dev.zarr.zarrjava.v2.Array array2 = (dev.zarr.zarrjava.v2.Array) subgroup.get("array2"); @@ -424,4 +426,31 @@ public void testGroupReadWriteV2() throws Exception { Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); assertIsTestdata(result, dataType); } + + @Test + public void testGroupReadWriteV3() throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV3", "write"); + StoreHandle storeHandle2 = new FilesystemStore(TESTOUTPUT).resolve("testGroupReadWriteV3", "read"); + dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(storeHandle, new Attributes(b -> b.set("attr", "value"))); + dev.zarr.zarrjava.v3.DataType dataType = DataType.INT32; + dev.zarr.zarrjava.v3.Array array = group.createGroup("group").createArray("array", arrayMetadataBuilder -> arrayMetadataBuilder + .withShape(16, 16, 16) + .withDataType(dataType) + .withChunkShape(2, 4, 8) + ); + + array.write(testdata(dataType)); + + run_python_script("zarr_python_group.py", storeHandle.toPath().toString(), storeHandle2.toPath().toString(), "" + 3); + + dev.zarr.zarrjava.v3.Group group2 = dev.zarr.zarrjava.v3.Group.open(storeHandle2); + Assertions.assertEquals("value", group2.metadata().attributes().get("attr")); + dev.zarr.zarrjava.v3.Group subgroup = (dev.zarr.zarrjava.v3.Group) group2.get("group2"); + Assertions.assertNotNull(subgroup); + dev.zarr.zarrjava.v3.Array array2 = (dev.zarr.zarrjava.v3.Array) subgroup.get("array2"); + Assertions.assertNotNull(array2); + ucar.ma2.Array result = array2.read(); + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + assertIsTestdata(result, dataType); + } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b40d516d..8a6309d2 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -1,13 +1,16 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; +import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; -import java.io.File; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.ArrayList; import java.util.Comparator; +import java.util.List; import java.util.stream.Stream; public class ZarrTest { @@ -30,4 +33,83 @@ public static void clearTestoutputFolder() throws IOException { } Files.createDirectory(TESTOUTPUT); } + + protected void assertListEquals(List a, List b) { + Assertions.assertEquals(a.size(), b.size()); + for (int i = 0; i < a.size(); i++) { + Object aval = a.get(i); + Object bval = b.get(i); + if (aval instanceof List && bval instanceof List) { + assertListEquals((List) aval, (List) bval); + } else { + Assertions.assertEquals(aval, bval); + } + } + } + + protected Attributes defaultTestAttributes() { + return new Attributes() {{ + put("string", "stringvalue"); + put("int", 42); + put("float", 0.5f); + put("double", 3.14); + put("boolean", true); + put("list", new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }); + put("int_array", new int[]{1, 2, 3}); + put("long_array", new long[]{1, 2, 3}); + put("double_array", new double[]{1.0, 2.0, 3.0}); + put("float_array", new float[]{1.0f, 2.0f, 3.0f}); + put("boolean_array", new boolean[]{true, false, true}); + put("nested", new Attributes() {{ + put("element", "value"); + }}); + put("array_of_attributes", new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }); + }}; + } + + protected void assertContainsTestAttributes(Attributes attributes) throws ZarrException { + Assertions.assertEquals("stringvalue", attributes.getString("string")); + Assertions.assertEquals(42, attributes.getInt("int")); + Assertions.assertEquals(0.5, attributes.getFloat("float")); + Assertions.assertEquals(3.14, attributes.getDouble("double")); + Assertions.assertTrue(attributes.getBoolean("boolean")); + assertListEquals(new ArrayList() { + { + add(1); + add(2.0d); + add("string"); + } + }, attributes.getList("list")); + Assertions.assertArrayEquals(new int[]{1, 2, 3}, attributes.getIntArray("int_array")); + Assertions.assertArrayEquals(new long[]{1, 2, 3}, attributes.getLongArray("long_array")); + Assertions.assertArrayEquals(new double[]{1, 2, 3}, attributes.getDoubleArray("double_array")); + Assertions.assertArrayEquals(new float[]{1, 2, 3}, attributes.getFloatArray("float_array")); + Assertions.assertArrayEquals(new boolean[]{true, false, true}, attributes.getBooleanArray("boolean_array")); + Assertions.assertEquals("value", attributes.getAttributes("nested").getString("element")); + Assertions.assertArrayEquals( + new Attributes[]{ + new Attributes() {{ + put("a", 1); + }}, + new Attributes() {{ + put("b", 2); + }} + }, + attributes.getArray("array_of_attributes", Attributes.class) + ); + } + } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index eb79cc83..f783646d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,5 +1,6 @@ package dev.zarr.zarrjava; +import dev.zarr.zarrjava.core.Attributes; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.store.FilesystemStore; @@ -18,6 +19,7 @@ import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -45,8 +47,6 @@ public void testCreateBlosc(String cname, String shuffle, int clevel) throws IOE Assertions.assertEquals(0, outArray.getByte(0)); } - - @ParameterizedTest @CsvSource({ "BOOL", "FLOAT64" @@ -55,7 +55,7 @@ public void testReadBloscDetectTypesize(DataType dt) throws IOException, ZarrExc String arrayname = dt == DataType.BOOL ? "bool" : "double"; StoreHandle storeHandle = new FilesystemStore(TESTDATA).resolve("v2_sample", arrayname); Array array = Array.open(storeHandle); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); + array.read(new long[]{0, 0, 0}, new int[]{3, 4, 5}); Assertions.assertEquals(dt, array.metadata().dataType); } @@ -259,6 +259,111 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + + Group.create(storeHandleString, new Attributes(b -> b.set("some", "value"))); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); + Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zattrs"))); + } + + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV2"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + } + + @Test + public void testSetAndUpdateAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testSetAttributesV2"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("some", "value"))) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); + + // add attribute + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } + + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testGroupAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2"); + + Group group = Group.create(storeHandle, new Attributes() {{ + put("group_attr", "group_value"); + }}); + + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + + group = Group.open(storeHandle); + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index cec859ac..846f770d 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -3,8 +3,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; +import dev.zarr.zarrjava.core.Attributes; import com.fasterxml.jackson.databind.JsonMappingException; import dev.zarr.zarrjava.store.*; @@ -12,8 +11,7 @@ import dev.zarr.zarrjava.v3.Node; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; +import dev.zarr.zarrjava.v3.codec.core.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; @@ -517,7 +515,7 @@ public void testOpenOverloads() throws ZarrException, IOException { public void testGroup() throws IOException, ZarrException { FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(fsStore.resolve("testgroup")); @@ -559,20 +557,47 @@ public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); Path storeHandlePath = TESTOUTPUT.resolve("testCreateGroupV3Path"); String storeHandleString = String.valueOf(TESTOUTPUT.resolve("testCreateGroupV3String")); - Map attributes = new HashMap<>(); + Attributes attributes = new Attributes(); attributes.put("hello", "world"); Group group = Group.create(storeHandle, new GroupMetadata(attributes)); Assertions.assertTrue(storeHandle.resolve("zarr.json").exists()); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); group = Group.create(storeHandlePath, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(storeHandlePath.resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); group = Group.create(storeHandleString, new GroupMetadata(attributes)); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); - Assertions.assertEquals("world", group.metadata.attributes.get("hello")); + Assertions.assertEquals("world", group.metadata.attributes().get("hello")); + } + + @Test + public void testAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testAttributesV3"); + + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .putAttribute("specific", "attribute") + .withAttributes(defaultTestAttributes()) + .withAttributes(new Attributes() {{ + put("another", "attribute"); + }}) + + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", array.metadata().attributes().getString("another")); + + Array arrayOpened = Array.open(storeHandle); + assertContainsTestAttributes(arrayOpened.metadata().attributes()); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("specific")); + Assertions.assertEquals("attribute", arrayOpened.metadata().attributes().getString("another")); } @Test @@ -602,6 +627,7 @@ static Stream> codecBuilders() { CodecBuilder::withCrc32c ); } + static Stream> chunkKeyEncodingsAndCodecs() { Stream> builders = Stream.of( ArrayMetadataBuilder::withDefaultChunkKeyEncoding, @@ -611,7 +637,6 @@ static Stream> chunkKeyEnco return Stream.concat(builders, codecBuilders().map(codecFunc -> b -> b.withCodecs(codecFunc))); } - @ParameterizedTest @MethodSource("chunkKeyEncodingsAndCodecs") public void testZarrJsonFormat(Function chunkKeyEncodingsAndCodecs) throws ZarrException, IOException { @@ -631,4 +656,79 @@ public void testZarrJsonFormat(Function b.set("some", "value"))) + .build(); + + Array array = Array.create(storeHandle, arrayMetadata); + Assertions.assertEquals("value", array.metadata().attributes().getString("some")); + array.setAttributes(defaultTestAttributes()); + array = Array.open(storeHandle); + assertContainsTestAttributes(array.metadata().attributes()); + Assertions.assertNull(array.metadata().attributes().get("some")); + + // add attribute + array = array.updateAttributes(b -> b.set("new_attribute", "new_value")); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + array = Array.open(storeHandle); + Assertions.assertEquals("new_value", array.metadata().attributes().getString("new_attribute")); + + // delete attribute + array = array.updateAttributes(b -> b.delete("new_value")); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + array = Array.open(storeHandle); + Assertions.assertNull(array.metadata().attributes().get("new_value")); + + assertContainsTestAttributes(array.metadata().attributes()); + } + + @Test + public void testResizeArray() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .withFillValue(1) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{20, 15}); + Assertions.assertArrayEquals(new int[]{20, 15}, array.read().getShape()); + + ucar.ma2.Array data = array.read(new long[]{0, 0}, new int[]{10, 10}); + Assertions.assertArrayEquals(testData, (int[]) data.get1DJavaArray(ma2DataType)); + + data = array.read(new long[]{10, 10}, new int[]{5, 5}); + int[] expectedData = new int[5 * 5]; + Arrays.fill(expectedData, 1); + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + + @Test + public void testGroupAttributes() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3"); + + Group group = Group.create(storeHandle, new Attributes() {{ + put("group_attr", "group_value"); + }}); + + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + + group = Group.open(storeHandle); + Assertions.assertEquals("group_value", group.metadata().attributes().getString("group_attr")); + } } diff --git a/src/test/python-scripts/zarr_python_group_v2.py b/src/test/python-scripts/zarr_python_group.py similarity index 78% rename from src/test/python-scripts/zarr_python_group_v2.py rename to src/test/python-scripts/zarr_python_group.py index 5052ddf0..3aa76c73 100644 --- a/src/test/python-scripts/zarr_python_group_v2.py +++ b/src/test/python-scripts/zarr_python_group.py @@ -7,16 +7,20 @@ from zarr.storage import LocalStore store_path_read = Path(sys.argv[1]) +store_path_write = Path(sys.argv[2]) +zarr_format = int(sys.argv[3]) +assert zarr_format in (2,3), f"unexpected zarr format: {zarr_format}" expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=2) +g = zarr.open_group(store=LocalStore(store_path_read), zarr_format=zarr_format) +assert g.attrs['attr'] == 'value' a = g['group']['array'] read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" -store_path_write = Path(sys.argv[2]) -g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=2) +g2 = zarr.create_group(store=LocalStore(store_path_write), zarr_format=zarr_format) +g2.attrs['attr'] = 'value' a2 = g2.create_group('group2').create_array( name='array2', shape=(16, 16, 16), diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5ff5e5a8..5d0b1c63 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -33,7 +33,7 @@ filters=filters, serializer=serializer, compressors=compressor, -# attributes={'test_key': 'test_value'}, + attributes={'test_key': 'test_value'}, ) assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}"