diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index aaa004ef..03b2e85f 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.core; import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.MultiArrayUtils; @@ -16,6 +17,7 @@ public abstract class ArrayMetadata { public final long[] shape; + @JsonInclude(JsonInclude.Include.ALWAYS) @JsonProperty("fill_value") public final Object fillValue; @JsonIgnore diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 4ccc8f91..9f865307 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.core.codec.core; import com.fasterxml.jackson.annotation.JsonValue; +import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import ucar.ma2.*; @@ -8,31 +9,36 @@ import java.nio.ByteOrder; public abstract class BytesCodec extends ArrayBytesCodec { - protected abstract ByteOrder getByteOrder(); + protected abstract ByteOrder getByteOrder() throws ZarrException; @Override -public Array decode(ByteBuffer chunkBytes) { - chunkBytes.order(getByteOrder()); - DataType dtype = arrayMetadata.dataType.getMA2DataType(); - int[] shape = arrayMetadata.chunkShape; - - // Array.factory does not support boolean arrays directly from ByteBuffer - if (dtype == DataType.BOOLEAN) { - int size = chunkBytes.remaining(); - boolean[] bools = new boolean[size]; - for (int i = 0; i < size; i++) { - bools[i] = chunkBytes.get(i) != 0; - } + public Array decode(ByteBuffer chunkBytes) throws ZarrException { + ByteOrder order = ByteOrder.BIG_ENDIAN; // Default for 1-byte types + if (arrayMetadata.dataType.getByteCount() > 1) + order = getByteOrder(); + chunkBytes.order(order); + DataType dtype = arrayMetadata.dataType.getMA2DataType(); + int[] shape = arrayMetadata.chunkShape; + + // Array.factory does not support boolean arrays directly from ByteBuffer + if (dtype == DataType.BOOLEAN) { + int size = chunkBytes.remaining(); + boolean[] bools = new boolean[size]; + for (int i = 0; i < size; i++) { + bools[i] = chunkBytes.get(i) != 0; + } - Index index = Index.factory(shape); - return Array.factory(DataType.BOOLEAN, index, bools); + Index index = Index.factory(shape); + return Array.factory(DataType.BOOLEAN, index, bools); + } + return Array.factory(dtype, shape, chunkBytes); } - return Array.factory(dtype, shape, chunkBytes); -} @Override - public ByteBuffer encode(Array chunkArray) { - ByteOrder order = getByteOrder(); + public ByteBuffer encode(Array chunkArray) throws ZarrException { + ByteOrder order = ByteOrder.BIG_ENDIAN; // Default for 1-byte types + if (arrayMetadata.dataType.getByteCount() > 1) + order = getByteOrder(); // Boolean if (chunkArray instanceof ArrayBoolean) { @@ -94,6 +100,10 @@ public ByteOrder getByteOrder() { throw new RuntimeException("Unreachable"); } } + + public static Endian nativeOrder() { + return ByteOrder.nativeOrder() == ByteOrder.LITTLE_ENDIAN ? LITTLE : BIG; + } } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 638d6484..1cbfb056 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.v2; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -18,6 +18,7 @@ import java.util.function.Function; import java.util.stream.Collectors; import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; +import static dev.zarr.zarrjava.v2.Node.makeObjectWriter; public class Array extends dev.zarr.zarrjava.core.Array implements Node { @@ -141,8 +142,8 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, "Trying to create a new array in " + storeHandle + ". But " + metadataHandle + " already exists."); } - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); metadataHandle.set(metadataBytes); return new Array(storeHandle, arrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Group.java b/src/main/java/dev/zarr/zarrjava/v2/Group.java index 38c0b946..c8982e13 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.v2; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -14,6 +14,7 @@ import java.nio.file.Paths; import java.util.function.Function; import static dev.zarr.zarrjava.v2.Node.makeObjectMapper; +import static dev.zarr.zarrjava.v2.Node.makeObjectWriter; public class Group extends dev.zarr.zarrjava.core.Group implements Node{ public GroupMetadata metadata; @@ -41,8 +42,8 @@ public static Group open(String path) throws IOException { public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZGROUP).set(metadataBytes); return new Group(storeHandle, groupMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/Node.java b/src/main/java/dev/zarr/zarrjava/v2/Node.java index 46232000..09b972a4 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Node.java @@ -1,6 +1,8 @@ package dev.zarr.zarrjava.v2; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; @@ -21,7 +23,11 @@ static ObjectMapper makeObjectMapper() { return objectMapper; } - /** + static ObjectWriter makeObjectWriter() { + return makeObjectMapper().writerWithDefaultPrettyPrinter(); + } + + /** * Opens an existing Zarr array or group at a specified storage location. * * @param storeHandle the storage location of the Zarr array diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 0bccc184..4e618d47 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v2.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParser; @@ -22,6 +23,7 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { + @JsonIgnore public final String id = "blosc"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index 773022a9..6f7d7973 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -1,8 +1,5 @@ package dev.zarr.zarrjava.v2.codec.core; -import com.fasterxml.jackson.annotation.JsonCreator; -import com.fasterxml.jackson.annotation.JsonProperty; -import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.v2.codec.Codec; @@ -14,9 +11,7 @@ public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec imp @Nonnull public final Endian endian; - @JsonCreator - public BytesCodec( - @JsonProperty(value = "endian", defaultValue = "little") Endian endian) { + public BytesCodec(Endian endian) { this.endian = endian; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 5093b586..3e9b5cf7 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v2.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.ArrayMetadata; @@ -16,6 +17,7 @@ public class ZlibCodec extends BytesBytesCodec implements Codec { + @JsonIgnore public final String id = "zlib"; public final int level; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 4d1434f7..19c82fbf 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.v3; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -17,6 +17,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; +import static dev.zarr.zarrjava.v3.Node.makeObjectWriter; public class Array extends dev.zarr.zarrjava.core.Array implements Node { @@ -136,8 +137,8 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, "Trying to create a new array in " + storeHandle + ". But " + metadataHandle + " already exists."); } - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(arrayMetadata)); metadataHandle.set(metadataBytes); return new Array(storeHandle, arrayMetadata); } @@ -182,8 +183,8 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat } private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newArrayMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newArrayMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); return new Array(storeHandle, newArrayMetadata); diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 212f4732..417d8ede 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -73,6 +73,12 @@ public ArrayMetadataBuilder withDefaultChunkKeyEncoding(Separator separator) { return this; } + public ArrayMetadataBuilder withDefaultChunkKeyEncoding() { + this.chunkKeyEncoding = new DefaultChunkKeyEncoding( + new DefaultChunkKeyEncoding.Configuration(Separator.SLASH)); + return this; + } + public ArrayMetadataBuilder withDefaultChunkKeyEncoding(String separator) { this.chunkKeyEncoding = new DefaultChunkKeyEncoding( @@ -85,6 +91,12 @@ public ArrayMetadataBuilder withV2ChunkKeyEncoding(Separator separator) { return this; } + public ArrayMetadataBuilder withV2ChunkKeyEncoding() { + this.chunkKeyEncoding = new V2ChunkKeyEncoding( + new V2ChunkKeyEncoding.Configuration(Separator.DOT)); + return this; + } + public ArrayMetadataBuilder withV2ChunkKeyEncoding(String separator) { this.chunkKeyEncoding = new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(Separator.valueOf(separator))); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Group.java b/src/main/java/dev/zarr/zarrjava/v3/Group.java index 9c56f25c..7b025975 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.v3; -import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; @@ -14,6 +14,7 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; +import static dev.zarr.zarrjava.v3.Node.makeObjectWriter; public class Group extends dev.zarr.zarrjava.core.Group implements Node { @@ -44,8 +45,8 @@ public static Group open(String path) throws IOException { public static Group create( @Nonnull StoreHandle storeHandle, @Nonnull GroupMetadata groupMetadata ) throws IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(groupMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(groupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); return new Group(storeHandle, groupMetadata); @@ -114,8 +115,8 @@ public Array createArray(String key, } private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException { - ObjectMapper objectMapper = makeObjectMapper(); - ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(newGroupMetadata)); + ObjectWriter objectWriter = makeObjectWriter(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectWriter.writeValueAsBytes(newGroupMetadata)); storeHandle.resolve(ZARR_JSON) .set(metadataBytes); return new Group(storeHandle, newGroupMetadata); diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index 235e37f3..00e17446 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -1,6 +1,8 @@ package dev.zarr.zarrjava.v3; +import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.FilesystemStore; @@ -20,9 +22,14 @@ static ObjectMapper makeObjectMapper() { ObjectMapper objectMapper = new ObjectMapper(); objectMapper.registerModule(new Jdk8Module()); objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + objectMapper.setDefaultPropertyInclusion(JsonInclude.Include.NON_NULL); return objectMapper; } + static ObjectWriter makeObjectWriter() { + return makeObjectMapper().writerWithDefaultPrettyPrinter(); + } + /** * Opens an existing Zarr array or group at a specified storage location. * diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java index 380a2a97..0e897696 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkgrid/RegularChunkGrid.java @@ -1,12 +1,14 @@ package dev.zarr.zarrjava.v3.chunkgrid; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import javax.annotation.Nonnull; public class RegularChunkGrid extends ChunkGrid { + @JsonIgnore public final String name = "regular"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java index 52b7ea6d..208280c0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.chunkkeyencoding; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; @@ -11,6 +12,7 @@ public class DefaultChunkKeyEncoding extends ChunkKeyEncoding { + @JsonIgnore public final String name = "default"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java index d72b429b..d62994c9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.chunkkeyencoding; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; @@ -11,6 +12,7 @@ public class V2ChunkKeyEncoding extends ChunkKeyEncoding { + @JsonIgnore public final String name = "v2"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index ad2f70c4..e281cde0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -67,7 +67,10 @@ public CodecBuilder withTranspose(int[] order) { } public CodecBuilder withBytes(Endian endian) { - codecs.add(new BytesCodec(new BytesCodec.Configuration(endian))); + if (dataType.getByteCount() <= 1) + codecs.add(new BytesCodec()); + else + codecs.add(new BytesCodec(endian)); return this; } @@ -75,6 +78,10 @@ public CodecBuilder withBytes(String endian) { return withBytes(BytesCodec.Endian.valueOf(endian)); } + public CodecBuilder withBytes() { + return withBytes(Endian.nativeOrder()); + } + public CodecBuilder withGzip(int clevel) { try { codecs.add(new GzipCodec(new GzipCodec.Configuration(clevel))); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index caa385fa..2daaec32 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParseException; @@ -23,6 +24,7 @@ public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec{ + @JsonIgnore public final String name = "blosc"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index 311e9f56..ed9380be 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; @@ -8,20 +9,26 @@ import java.nio.ByteOrder; import javax.annotation.Nonnull; +import javax.annotation.Nullable; public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { + @JsonIgnore public final String name = "bytes"; - @Nonnull + @Nullable public final Configuration configuration; @JsonCreator public BytesCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration + @JsonProperty(value = "configuration") Configuration configuration ) { this.configuration = configuration; } + public BytesCodec() { + this((Configuration) null); + } + public BytesCodec(Endian endian) { this(new BytesCodec.Configuration(endian)); } @@ -33,7 +40,10 @@ public long computeEncodedSize(long inputByteLength, } @Override - protected ByteOrder getByteOrder() { + protected ByteOrder getByteOrder() throws ZarrException { + if (configuration == null) { + throw new ZarrException("BytesCodec configuration is required to determine endianess."); + } return configuration.endian.getByteOrder(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 823106d7..61329218 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.CRC32C; @@ -12,6 +13,7 @@ public class Crc32cCodec extends BytesBytesCodec implements Codec { + @JsonIgnore public final String name = "crc32c"; @JsonCreator diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index c197fc43..cb03b268 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; @@ -17,6 +18,7 @@ public class GzipCodec extends BytesBytesCodec implements Codec { + @JsonIgnore public final String name = "gzip"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index dd30a930..6e598ab3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; @@ -24,6 +25,7 @@ public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode implements Codec { + @JsonIgnore public final String name = "sharding_indexed"; @Nonnull public final Configuration configuration; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index 5eebefeb..c0855322 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; @@ -16,6 +17,7 @@ public class TransposeCodec extends ArrayArrayCodec implements Codec{ + @JsonIgnore @Nonnull public final String name = "transpose"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 75d27f55..bc5304be 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v3.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; @@ -14,6 +15,7 @@ public class ZstdCodec extends BytesBytesCodec implements Codec { + @JsonIgnore public final String name = "zstd"; @Nonnull public final Configuration configuration; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index e4e763ac..1f2899f3 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -150,9 +150,9 @@ static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataT static Stream compressorAndDataTypeProviderV3() { Stream datatypeTests = Stream.of( -// DataType.BOOL, -// DataType.INT8, -// DataType.UINT8, // -> BUG: see https://github.com/zarr-developers/zarr-java/issues/27 + DataType.BOOL, + DataType.INT8, + DataType.UINT8, DataType.INT16, DataType.UINT16, DataType.INT32, @@ -203,7 +203,7 @@ public void testReadV3(String codec, String codecParam, DataType dataType) throw Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata().chunkShape()); Assertions.assertEquals(42, array.metadata().attributes.get("answer")); - assertIsTestdata(result, DataType.INT32); + assertIsTestdata(result, dataType); } @ParameterizedTest @@ -266,7 +266,7 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata().chunkShape()); Assertions.assertEquals("test_value", readArray.metadata().attributes.get("test_key")); - assertIsTestdata(result, DataType.INT32); + assertIsTestdata(result, dataType); //read in zarr_python run_python_script("zarr_python_read.py", codec, codecParam, dataType.name().toLowerCase(), storeHandle.toPath().toString()); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index 765c51a4..eb79cc83 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -1,23 +1,28 @@ package dev.zarr.zarrjava; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.store.FilesystemStore; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v2.Array; -import dev.zarr.zarrjava.v2.ArrayMetadata; -import dev.zarr.zarrjava.v2.DataType; -import dev.zarr.zarrjava.v2.Group; -import dev.zarr.zarrjava.v2.Node; +import dev.zarr.zarrjava.v2.*; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.CsvSource; +import org.junit.jupiter.params.provider.MethodSource; import org.junit.jupiter.params.provider.ValueSource; +import java.io.BufferedReader; import java.io.IOException; import java.nio.file.Files; import java.nio.file.NoSuchFileException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static dev.zarr.zarrjava.core.Node.ZARRAY; public class ZarrV2Test extends ZarrTest { @ParameterizedTest @@ -255,4 +260,32 @@ public void testCreateGroup() throws ZarrException, IOException { Group.create(storeHandleString); Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve(".zgroup"))); } + + + static Stream> compressorBuilder() { + return Stream.of( + ArrayMetadataBuilder::withBloscCompressor, + ArrayMetadataBuilder::withZlibCompressor, + b -> b + ); + } + + @ParameterizedTest + @MethodSource("compressorBuilder") + public void testZarrJsonFormat(Function compressorBuilder) throws ZarrException, IOException { + // regression test: ensure that 'id' keyword of codecs is only written once. + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZarrJsonFormatV2").resolve(String.valueOf(compressorBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(6, 6); + builder = compressorBuilder.apply(builder); + Array.create(storeHandle, builder.build()); + + try (BufferedReader reader = Files.newBufferedReader(storeHandle.resolve(ZARRAY).toPath())) { + String jsonInString = reader.lines().collect(Collectors.joining(System.lineSeparator())); + JsonNode JSON = new ObjectMapper().readTree(jsonInString); + Assertions.assertEquals(JSON.toPrettyString(), jsonInString); + } + } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 7f1c15c7..cec859ac 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -1,5 +1,8 @@ package dev.zarr.zarrjava; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; @@ -29,9 +32,11 @@ import java.util.HashMap; import java.util.Map; import java.util.function.Function; +import java.util.stream.Collectors; import java.util.stream.Stream; import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; +import static dev.zarr.zarrjava.core.Node.ZARR_JSON; import static org.junit.Assert.assertThrows; public class ZarrV3Test extends ZarrTest { @@ -548,6 +553,7 @@ public void testCreateArray() throws ZarrException, IOException { Assertions.assertTrue(Files.exists(Paths.get(storeHandleString).resolve("zarr.json"))); } + @Test public void testCreateGroup() throws ZarrException, IOException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCreateGroupV3"); @@ -569,6 +575,60 @@ public void testCreateGroup() throws ZarrException, IOException { Assertions.assertEquals("world", group.metadata.attributes.get("hello")); } + @Test + public void testCodecWithoutConfiguration() throws ZarrException, IOException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testCodecWithoutConfigurationV3"); + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withCodecs(CodecBuilder::withBytes) + .build() + ); + Assertions.assertTrue(storeHandle.resolve(ZARR_JSON).exists()); + Codec bytesCodec = array.metadata().codecs[0]; + Assertions.assertInstanceOf(BytesCodec.class, bytesCodec); + Assertions.assertNull(((BytesCodec) bytesCodec).configuration); + } + + static Stream> codecBuilders() { + return Stream.of( + CodecBuilder::withBlosc, + c -> c.withTranspose(new int[]{1, 0}), + CodecBuilder::withBytes, + CodecBuilder::withGzip, + CodecBuilder::withZstd, + c -> c.withSharding(new int[]{2, 2}), + CodecBuilder::withCrc32c + ); + } + static Stream> chunkKeyEncodingsAndCodecs() { + Stream> builders = Stream.of( + ArrayMetadataBuilder::withDefaultChunkKeyEncoding, + ArrayMetadataBuilder::withV2ChunkKeyEncoding + ); + + return Stream.concat(builders, codecBuilders().map(codecFunc -> b -> b.withCodecs(codecFunc))); + } + + + @ParameterizedTest + @MethodSource("chunkKeyEncodingsAndCodecs") + public void testZarrJsonFormat(Function chunkKeyEncodingsAndCodecs) throws ZarrException, IOException { + // regression test: ensure that 'name' keyword of named configurations (e.g. codecs) are only written once. + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZarrJsonFormatV3").resolve(String.valueOf(chunkKeyEncodingsAndCodecs.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(6, 6); + builder = chunkKeyEncodingsAndCodecs.apply(builder); + Array.create(storeHandle, builder.build()); + try (BufferedReader reader = Files.newBufferedReader(storeHandle.resolve(ZARR_JSON).toPath())) { + String jsonInString = reader.lines().collect(Collectors.joining(System.lineSeparator())); + JsonNode JSON = new ObjectMapper().readTree(jsonInString); + Assertions.assertEquals(JSON.toPrettyString(), jsonInString); + } + } }