From 23164e0e57f867c0690558e19f1072b824309fe0 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Sun, 7 Sep 2025 14:18:16 +0200 Subject: [PATCH 01/27] v2.Array create, arrayMetadata and codecPipeline --- src/main/java/dev/zarr/zarrjava/v2/Array.java | 60 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 71 +- .../zarrjava/v2/ArrayMetadataBuilder.java | 115 ++ .../java/dev/zarr/zarrjava/v2/DataType.java | 30 - .../java/dev/zarr/zarrjava/v2/DataTypeV2.java | 91 ++ .../java/dev/zarr/zarrjava/v2/Endianness.java | 13 + .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 2 +- .../zarrjava/v3/ArrayMetadataBuilder.java | 3 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 1295 +++++++++-------- 9 files changed, 991 insertions(+), 689 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java delete mode 100644 src/main/java/dev/zarr/zarrjava/v2/DataType.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index f9e590a2..12455aa7 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,34 +1,76 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; +import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.codec.CodecPipeline; +import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; + import java.io.IOException; +import java.nio.ByteBuffer; import java.util.Arrays; +import java.util.function.Function; import java.util.stream.Collectors; +import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; + public class Array { static final String ZARRAY = ".zarray"; public ArrayMetadata metadata; public StoreHandle storeHandle; + CodecPipeline codecPipeline; - Array(StoreHandle storeHandle) throws IOException { + protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { this.storeHandle = storeHandle; + this.metadata = arrayMetadata; + this.codecPipeline = new CodecPipeline(Utils.concatArrays( + metadata.filters, + new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, + metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} + ), metadata.coreArrayMetadata); + } - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.registerModule(new Jdk8Module()); - this.metadata = objectMapper.readValue( - Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), - ArrayMetadata.class + public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { + return new Array( + storeHandle, + makeObjectMapper() + .readValue( + Utils.toArray(storeHandle.resolve(ZARRAY).readNonNull()), + ArrayMetadata.class + ) ); } - public static Array open(StoreHandle storeHandle) throws IOException { - return new Array(storeHandle); + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) + throws IOException, ZarrException { + return Array.create(storeHandle, arrayMetadata, false); + } + + public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) + throws IOException, ZarrException { + StoreHandle metadataHandle = storeHandle.resolve(ZARRAY); + if (!existsOk && metadataHandle.exists()) { + throw new RuntimeException( + "Trying to create a new array in " + storeHandle + ". But " + metadataHandle + + " already exists."); + } + ObjectMapper objectMapper = makeObjectMapper(); + ByteBuffer metadataBytes = ByteBuffer.wrap(objectMapper.writeValueAsBytes(arrayMetadata)); + metadataHandle.set(metadataBytes); + return new Array(storeHandle, arrayMetadata); + } + + public static Array create(StoreHandle storeHandle, + Function arrayMetadataBuilderMapper, + boolean existsOk) throws IOException, ZarrException { + return create(storeHandle, + arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); } + @Override public String toString() { return String.format("", storeHandle, diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 24de9d44..ddd55966 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -1,20 +1,35 @@ package dev.zarr.zarrjava.v2; +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; -import java.util.Optional; +import dev.zarr.zarrjava.v3.codec.Codec; + +import javax.annotation.Nullable; + +import static dev.zarr.zarrjava.v3.ArrayMetadata.parseFillValue; + public class ArrayMetadata { + static final int ZARR_FORMAT = 2; @JsonProperty("zarr_format") - public final int zarrFormat = 2; + public final int zarrFormat = ZARR_FORMAT; public long[] shape; - public long[] chunks; + public int[] chunks; @JsonProperty("dtype") - public DataType dataType; + public DataTypeV2 dataTypeV2; + @JsonIgnore + public final DataType dataType; + @JsonIgnore + public final Endianness endianness; + @JsonProperty("order") public Order order; @JsonProperty("dimension_separator") @@ -22,7 +37,51 @@ public class ArrayMetadata { @JsonProperty("fill_value") public Object fillValue; + @JsonIgnore + public final Object parsedFillValue; + + public Codec[] filters; + @Nullable + public Codec compressor; + + @JsonIgnore + public dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata coreArrayMetadata; + + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ArrayMetadata( + @JsonProperty(value = "zarr_format", required = true) int zarrFormat, + @JsonProperty(value = "shape", required = true) long[] shape, + @JsonProperty(value = "chunks", required = true) int[] chunks, + @JsonProperty(value = "dtype", required = true) DataTypeV2 dataTypeV2, + @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo can be "null" + @JsonProperty(value = "order", required = true) Order order, + @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator, + @Nullable @JsonProperty(value = "filters") Codec[] filters, //todo can be "null" + @Nullable @JsonProperty(value = "compressor") Codec compressor //todo can be "null" + ) throws ZarrException { + if (zarrFormat != this.zarrFormat) { + throw new ZarrException( + "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); + } + + this.shape = shape; + this.chunks = chunks; + this.dataTypeV2 = dataTypeV2; + this.endianness = dataTypeV2.getEndianness(); + this.dataType = dataTypeV2.toV3(); + this.fillValue = fillValue; + this.parsedFillValue = parseFillValue(fillValue, this.dataType); + this.order = order; + this.dimensionSeparator = dimensionSeparator; + this.filters = filters; + this.compressor = compressor; + this.coreArrayMetadata = + new dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata(shape, chunks, + this.dataType, + parsedFillValue + ); + } + - public Optional filters; - public Optional compressor; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java new file mode 100644 index 00000000..bf618a78 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -0,0 +1,115 @@ +package dev.zarr.zarrjava.v2; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; + +import java.util.function.Function; + +public class ArrayMetadataBuilder { + long[] shape = null; + int[] chunks = null; + DataTypeV2 dataTypeV2 = null; + Order order = Order.C; + Separator dimensionSeparator = Separator.DOT; + Object fillValue = 0; + Codec[] filters = new Codec[]{}; + Codec compressor = null; + + + protected ArrayMetadataBuilder() { + } + + protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetadata) { + ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); + builder.shape = arrayMetadata.shape; + builder.chunks = arrayMetadata.chunks; + builder.dataTypeV2 = arrayMetadata.dataTypeV2; + builder.order = arrayMetadata.order; + builder.dimensionSeparator = arrayMetadata.dimensionSeparator; + builder.fillValue = arrayMetadata.parsedFillValue; + builder.filters = arrayMetadata.filters; + builder.compressor = arrayMetadata.compressor; + return builder; + } + + public ArrayMetadataBuilder withShape(long... shape) { + this.shape = shape; + return this; + } + + public ArrayMetadataBuilder withChunks(int... chunks) { + this.chunks = chunks; + return this; + } + + public ArrayMetadataBuilder withDataType(DataTypeV2 dataTypeV2) { + this.dataTypeV2 = dataTypeV2; + return this; + } + + public ArrayMetadataBuilder withDataType(DataType dataType) { + this.dataTypeV2 = DataTypeV2.fromDataType(dataType); + return this; + } + + public ArrayMetadataBuilder withOrder(Order order) { + this.order = order; + return this; + } + + public ArrayMetadataBuilder withDimensionSeparator(Separator dimensionSeparator) { + this.dimensionSeparator = dimensionSeparator; + return this; + } + + public ArrayMetadataBuilder withFillValue(Object fillValue) { + this.fillValue = fillValue; + return this; + } + + public ArrayMetadataBuilder withFilters(Codec... filters) { + this.filters = filters; + return this; + } + + public ArrayMetadataBuilder withFilters(Function codecBuilder) throws ZarrException { + if (dataTypeV2 == null) { + throw new IllegalStateException("Please call `withDataType` first."); + } + CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3()); + this.filters = codecBuilder.apply(nestedCodecBuilder) + .build(); + return this; + } + + public ArrayMetadataBuilder withCompressor(Codec compressor) { + this.compressor = compressor; + return this; + } + + public ArrayMetadata build() throws ZarrException { + if (shape == null) { + throw new IllegalStateException("Please call `withShape` first."); + } + if (chunks == null) { + throw new IllegalStateException("Please call `withChunks` first."); + } + if (dataTypeV2 == null) { + throw new IllegalStateException("Please call `withDataType` first."); + } + return new ArrayMetadata( + 2, + shape, + chunks, + dataTypeV2, + fillValue, + order, + dimensionSeparator, + filters, + compressor + ); + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java deleted file mode 100644 index 3d592e25..00000000 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ /dev/null @@ -1,30 +0,0 @@ -package dev.zarr.zarrjava.v2; - -import com.fasterxml.jackson.annotation.JsonValue; - -public enum DataType { - BOOL("b1", Endianness.UNSPECIFIED), - INT8("i1", Endianness.UNSPECIFIED), - INT16("i2", Endianness.LITTLE), - INT32("i4", Endianness.LITTLE), - INT64("i8", Endianness.LITTLE), - UINT8("u1", Endianness.UNSPECIFIED), - UINT16("u2", Endianness.LITTLE), - UINT32("u4", Endianness.LITTLE), - UINT64("u8", Endianness.LITTLE), - FLOAT32("f4", Endianness.LITTLE), - FLOAT64("f8", Endianness.LITTLE); - - private final String dtype; - private final Endianness endianness; - - DataType(String dtype, Endianness endianness) { - this.dtype = dtype; - this.endianness = endianness; - } - - @JsonValue - public String getValue() { - return String.format("%s%s", endianness.getValue(), dtype); - } -} diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java b/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java new file mode 100644 index 00000000..ae661bae --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java @@ -0,0 +1,91 @@ +package dev.zarr.zarrjava.v2; + +import com.fasterxml.jackson.annotation.JsonValue; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.DataType; + +public enum DataTypeV2 { + BOOL("b1", Endianness.UNSPECIFIED), + INT8("i1", Endianness.UNSPECIFIED), + INT16("i2", Endianness.LITTLE), + INT32("i4", Endianness.LITTLE), + INT64("i8", Endianness.LITTLE), + UINT8("u1", Endianness.UNSPECIFIED), + UINT16("u2", Endianness.LITTLE), + UINT32("u4", Endianness.LITTLE), + UINT64("u8", Endianness.LITTLE), + FLOAT32("f4", Endianness.LITTLE), + FLOAT64("f8", Endianness.LITTLE); + + private final String dtype; + private final Endianness endianness; + + DataTypeV2(String dtype, Endianness endianness) { + this.dtype = dtype; + this.endianness = endianness; + } + + public static DataTypeV2 fromDataType(DataType dataType) { + switch (dataType) { + case BOOL: + return BOOL; + case INT8: + return INT8; + case INT16: + return INT16; + case INT32: + return INT32; + case INT64: + return INT64; + case UINT8: + return UINT8; + case UINT16: + return UINT16; + case UINT32: + return UINT32; + case UINT64: + return UINT64; + case FLOAT32: + return FLOAT32; + case FLOAT64: + return FLOAT64; + default: + throw new IllegalArgumentException("Unsupported DataType: " + dataType); + } + } + + public Endianness getEndianness() { + return endianness; + } + + public DataType toV3() throws ZarrException { + if (this.dtype.equals(BOOL.dtype)) + return DataType.BOOL; + if (this.dtype.equals(INT8.dtype)) + return DataType.INT8; + if (this.dtype.equals(INT16.dtype)) + return DataType.INT16; + if (this.dtype.equals(INT32.dtype)) + return DataType.INT32; + if (this.dtype.equals(INT64.dtype)) + return DataType.INT64; + if (this.dtype.equals(UINT8.dtype)) + return DataType.UINT8; + if (this.dtype.equals(UINT16.dtype)) + return DataType.UINT16; + if (this.dtype.equals(UINT32.dtype)) + return DataType.UINT32; + if (this.dtype.equals(UINT64.dtype)) + return DataType.UINT64; + if (this.dtype.equals(FLOAT32.dtype)) + return DataType.FLOAT32; + if (this.dtype.equals(FLOAT64.dtype)) + return DataType.FLOAT64; + throw new ZarrException("Unknown DataTypeV2: " + this.dtype); + } + + @JsonValue + public String getValue() { + return String.format("%s%s", endianness.getValue(), dtype); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java index cc7a6b4d..9c01c673 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java @@ -1,6 +1,7 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.annotation.JsonValue; +import dev.zarr.zarrjava.v3.codec.core.BytesCodec; public enum Endianness { LITTLE("<"), @@ -17,4 +18,16 @@ public enum Endianness { public String getValue() { return value; } + + public BytesCodec.Endian toEndian() { + switch (this) { + case LITTLE: + return BytesCodec.Endian.LITTLE; + case BIG: + return BytesCodec.Endian.BIG; + case UNSPECIFIED: + default: + return BytesCodec.Endian.LITTLE; + } + } } \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index f23a9fca..f8f5b2bd 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -286,7 +286,7 @@ public static final class CoreArrayMetadata { public final Object parsedFillValue; public CoreArrayMetadata(long[] shape, int[] chunkShape, DataType dataType, - Object parsedFillValue) { + Object parsedFillValue) { this.shape = shape; this.chunkShape = chunkShape; this.dataType = dataType; diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index 3d07ad49..c16a9e54 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -127,6 +127,7 @@ public ArrayMetadataBuilder withAttributes(Map attributes) { this.attributes = attributes; return this; } + public ArrayMetadataBuilder withStorageTransformers(Map[] storageTransformers) { this.storageTransformers = storageTransformers; return this; @@ -142,8 +143,6 @@ public ArrayMetadata build() throws ZarrException { if (chunkGrid == null) { throw new ZarrException("Chunk grid needs to be provided. Please call `.withChunkShape`."); } - - return new ArrayMetadata(shape, dataType, chunkGrid, chunkKeyEncoding, fillValue, codecs, dimensionNames, attributes, diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b9489a10..b1b1d390 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -7,13 +7,13 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; -import com.google.common.collect.Maps; import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -40,648 +40,661 @@ public class ZarrTest { - final static Path TESTDATA = Paths.get("testdata"); - final static Path TESTOUTPUT = Paths.get("testoutput"); - final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); - - public static String pythonPath() { - if (System.getProperty("os.name").startsWith("Windows")) { - return "venv_zarrita\\Scripts\\python.exe"; - } - return "venv_zarrita/bin/python"; - } - - @BeforeAll - public static void clearTestoutputFolder() throws IOException { - if (Files.exists(TESTOUTPUT)) { - try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } - } - Files.createDirectory(TESTOUTPUT); - } - - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - - public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { - String command = pythonPath(); - ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py").toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); - } - - int exitCode = process.waitFor(); - assert exitCode == 0; - - Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); - ucar.ma2.Array result = array.read(); - - //for expected values see zarrita_write.py - Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); - Assertions.assertEquals(42, array.metadata.attributes.get("answer")); - - int[] expectedData = new int[16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); - } - - @CsvSource({"0,true", "0,false", "5, true", "10, false"}) - @ParameterizedTest - public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { - //compress using ZstdCompressCtx - int number = 123456; - byte[] src = ByteBuffer.allocate(4).putInt(number).array(); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(clevel); - ctx.setChecksum(checksumFlag); - compressed = ctx.compress(src); - } - //decompress with Zstd.decompress - long originalSize = Zstd.decompressedSize(compressed); - byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); - Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); - - //write compressed to file - String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); - try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { - fos.write(compressed); - } - - //decompress in python - Process process = new ProcessBuilder( - pythonPath(), - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) - ).start(); - int exitCode = process.waitFor(); - assert exitCode == 0; - } - - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - Map attributes = new HashMap<>(); - attributes.put("test_key", "test_value"); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withAttributes(attributes); - - switch (codec) { - case "blosc": - String cname = codecParam.split("_")[0]; - String shuffle = codecParam.split("_")[1]; - int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); - builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); - break; - case "gzip": - builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); - break; - case "zstd": - int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); - boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); - builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); - break; - case "bytes": - builder = builder.withCodecs(c -> c.withBytes(codecParam)); - break; - case "transpose": - builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); - break; - case "sharding": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); - break; - case "sharding_nested": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); - break; - case "crc32c": - builder = builder.withCodecs(CodecBuilder::withCrc32c); - break; - default: - throw new IllegalArgumentException("Invalid Codec: " + codec); - } - - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); - Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - - //read in zarrita - String command = pythonPath(); - - ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_read.py").toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); - } - - int exitCode = process.waitFor(); - assert exitCode == 0; - } - - static Stream> invalidCodecBuilder() { - return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) - ); - } - - @ParameterizedTest - @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(new long[]{4, 4}) - .withDataType(DataType.UINT32) - .withChunkShape(new int[]{2, 2}) - .withCodecs(codecBuilder); - - assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); - } - - @Test - public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(32, 32, 32) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - static Stream invalidChunkSizes() { - return Stream.of( - new int[]{1}, - new int[]{1, 1, 1} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidChunkDimensions(int[] chunkSize) { - long[] shape = new long[]{4, 4}; - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32) - .withChunkShape(chunkSize); - - assertThrows(ZarrException.class, builder::build); - } - - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - - @ParameterizedTest - @MethodSource("invalidShardSizes") - public void testCheckShardingBounds(int[] shardSize) throws Exception { - long[] shape = new long[]{10, 10}; - int[] innerChunkSize = new int[]{2, 2}; - - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); - - if (false) { - int[] nestedChunkSize = new int[]{4, 4}; - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); - } - builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); - assertThrows(ZarrException.class, builder::build); - } - - @ParameterizedTest - @CsvSource({"0,true", "0,false", "5, true", "5, false"}) - public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(clevel, checksum)); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testTransposeCodec() throws ZarrException { - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); - - assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); - assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); - } - - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { - int[] shapeInt = new int[]{2, 3, 3}; - long[] shapeLong = new long[]{2, 3, 3}; - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); - - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); - assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = Node.makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testHttpStore() throws IOException, ZarrException { - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data/"); - System.out.println( - dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); - System.out.println(Array.open(httpStore.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - } - - @Test - public void testV3Access() throws IOException, ZarrException { - Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata - ); - writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); - } - - @ParameterizedTest - @ValueSource(strings = {"start", "end"}) - public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); - } - - @Test - public void testV3Codecs() throws IOException, ZarrException { - int[] readShape = new int[]{1, 1, 1024, 1024}; - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); - } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); - } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); - } - } - - @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { + final static Path TESTDATA = Paths.get("testdata"); + final static Path TESTOUTPUT = Paths.get("testoutput"); + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + public static String pythonPath() { + if (System.getProperty("os.name").startsWith("Windows")) { + return "venv_zarrita\\Scripts\\python.exe"; + } + return "venv_zarrita/bin/python"; + } + + @BeforeAll + public static void clearTestoutputFolder() throws IOException { + if (Files.exists(TESTOUTPUT)) { + try (Stream walk = Files.walk(TESTOUTPUT)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + Files.createDirectory(TESTOUTPUT); + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + + public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { + String command = pythonPath(); + ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py") + .toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + + Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); + ucar.ma2.Array result = array.read(); + + //for expected values see zarrita_write.py + Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); + Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); + Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + + int[] expectedData = new int[16 * 16]; + Arrays.setAll(expectedData, p -> p); + Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @CsvSource({"0,true", "0,false", "5, true", "10, false"}) + @ParameterizedTest + public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { + //compress using ZstdCompressCtx + int number = 123456; + byte[] src = ByteBuffer.allocate(4).putInt(number).array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(clevel); + ctx.setChecksum(checksumFlag); + compressed = ctx.compress(src); + } + //decompress with Zstd.decompress + long originalSize = Zstd.decompressedSize(compressed); + byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); + Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); + + //write compressed to file + String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); + try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { + fos.write(compressed); + } + + //decompress in python + Process process = new ProcessBuilder( + pythonPath(), + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) + ).start(); + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + Map attributes = new HashMap<>(); + attributes.put("test_key", "test_value"); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withAttributes(attributes); + + switch (codec) { + case "blosc": + String cname = codecParam.split("_")[0]; + String shuffle = codecParam.split("_")[1]; + int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); + builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); + break; + case "gzip": + builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); + break; + case "zstd": + int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); + boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); + builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); + break; + case "bytes": + builder = builder.withCodecs(c -> c.withBytes(codecParam)); + break; + case "transpose": + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); + break; + case "sharding": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); + break; + case "sharding_nested": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); + break; + case "crc32c": + builder = builder.withCodecs(CodecBuilder::withCrc32c); + break; + default: + throw new IllegalArgumentException("Invalid Codec: " + codec); + } + + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + + //read in zarrita + String command = pythonPath(); + + ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_read.py") + .toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + static Stream> invalidCodecBuilder() { + return Stream.of( + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + @ParameterizedTest + @MethodSource("invalidCodecBuilder") + public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(new long[]{4, 4}) + .withDataType(DataType.UINT32) + .withChunkShape(new int[]{2, 2}) + .withCodecs(codecBuilder); + + assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); + } + + @Test + public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(32, 32, 32) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + static Stream invalidChunkSizes() { + return Stream.of( + new int[]{1}, + new int[]{1, 1, 1} + ); + } + + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidChunkDimensions(int[] chunkSize) { + long[] shape = new long[]{4, 4}; + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32) + .withChunkShape(chunkSize); + + assertThrows(ZarrException.class, builder::build); + } + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension + ); + } + + @ParameterizedTest + @MethodSource("invalidShardSizes") + public void testCheckShardingBounds(int[] shardSize) throws Exception { + long[] shape = new long[]{10, 10}; + int[] innerChunkSize = new int[]{2, 2}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); + + if (false) { + int[] nestedChunkSize = new int[]{4, 4}; + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + } + builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); + assertThrows(ZarrException.class, builder::build); + } + + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } + + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + } + + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } + + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { + int[] shapeInt = new int[]{2, 3, 3}; + long[] shapeLong = new long[]{2, 3, 3}; + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + shapeLong, + shapeInt, + DataType.UINT32, + null)); + + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); + assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); + } + + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = Node.makeObjectMapper(); + + GroupMetadata group = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + System.out.println(group); + System.out.println(objectMapper.writeValueAsString(group)); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + System.out.println(arrayMetadata); + System.out.println(objectMapper.writeValueAsString(arrayMetadata)); + + System.out.println( + Array.open(fsStore.resolve("l4_sample", "color", "1"))); + System.out.println( + Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); + System.out.println( + Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() + .toArray(Node[]::new))); + } + + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() + .withRegion("eu-west-1") + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .build(), "static.webknossos.org", "data"); + System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); + } + + @Test + public void testHttpStore() throws IOException, ZarrException { + HttpStore httpStore = new HttpStore("https://static.webknossos.org/data/"); + dev.zarr.zarrjava.v2.Array v2Array = dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1")); + Array v3Array = Array.open(httpStore.resolve("zarr_v3", "l4_sample", "color", "1")); + + System.out.println(v2Array.metadata.fillValue); + System.out.println(v2Array.metadata.dataType); + System.out.println(v2Array.metadata.chunks[0]); + System.out.println(v2Array.metadata.filters); + System.out.println(v2Array.metadata.compressor); + System.out.println(v2Array.metadata.order); + System.out.println(v2Array.metadata.dimensionSeparator); + System.out.println(v3Array); + for (Codec codec : ((ShardingIndexedCodec) v3Array.metadata.codecs[0]).configuration.codecs) { + System.out.println(codec); + } + } + + @Test + public void testV3ShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + } + + @Test + public void testV3Access() throws IOException, ZarrException { + Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) + .withShape(1, 64, 64, 64) + .read(); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata + ); + writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); + } + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); + } + + @Test + public void testV3Codecs() throws IOException, ZarrException { + int[] readShape = new int[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } + } + + @Test + public void testV3ArrayMetadataBuilder() throws ZarrException { + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); + } + + @Test + public void testV3FillValue() throws ZarrException { + Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); + Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); + Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); + assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); + } + + @Test + public void testV3Group() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", attributes); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); + } + + @Test + public void testV2() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(""); + HttpStore httpStore = new HttpStore("https://static.webknossos.org/data"); + + System.out.println(dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); + } + + + @Test + public void testReadme1() throws IOException, ZarrException { + Group hierarchy = Group.open( + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") + ); + Group color = (Group) hierarchy.get("color"); + Array array = (Array) color.get("1"); + ucar.ma2.Array outArray = array.read( + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape + ); + } + + @Test + public void testReadme2() throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) - .build(); - } - - @Test - public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); - } - - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", attributes); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); - } - - @Test - public void testV2() throws IOException { - FilesystemStore fsStore = new FilesystemStore(""); - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data"); - - System.out.println(dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); - } - - - @Test - public void testReadme1() throws IOException, ZarrException { - Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") - ); - Group color = (Group) hierarchy.get("color"); - Array array = (Array) color.get("1"); - ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape - ); - } - - @Test - public void testReadme2() throws IOException, ZarrException { - Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() - ); - ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); - array.write( - new long[]{0, 0, 0, 0}, // offset - data - ); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); - assert MultiArrayUtils.allValuesEqual(data, output); - - } - - @ParameterizedTest - @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) - public void testReadL4Sample(String mag) throws IOException, ZarrException { - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); - StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); - - Array httpArray = Array.open(httpStoreHandle); - Array localArray = Array.open(localStoreHandle); - System.out.println(httpArray); - System.out.println(localArray); - - ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData1, localData1); - - //offset to where l4_sample contains non-zero values - long[] offset = new long[4]; - long[] originalOffset = new long[]{0, 3073, 3073, 513}; - long[] originalShape = new long[]{1, 4096, 4096, 2048}; - long[] arrayShape = httpArray.metadata.shape; - for (int i = 0; i < 4; i++) { - offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); - } - - ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData2, localData2); - } - - @ParameterizedTest - @ValueSource(booleans = {false,true}) - public void testParallel(boolean useParallel) throws IOException, ZarrException { - int[] testData = new int[512 * 512 * 512]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(512, 512, 512) + .withShape(1, 4096, 4096, 1536) .withDataType(DataType.UINT32) - .withChunkShape(100, 100, 100) + .withChunkShape(1, 1024, 1024, 1024) .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(useParallel); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - clearTestoutputFolder(); - } - - @Test - public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { - // non-empty storage transformers are currently not supported - - Map[] storageTransformersEmpty = Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") - ).metadata.storageTransformers; - assert storageTransformersEmpty.length == 0; - - assertThrows(JsonMappingException.class, () -> Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) - ); - - ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap(){{ - put("some", "value"); - }}}); - - assertThrows(ZarrException.class, () -> Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - builderWithStorageTransformer.build() - )); - } + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() + ); + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); + array.write( + new long[]{0, 0, 0, 0}, // offset + data + ); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); + assert MultiArrayUtils.allValuesEqual(data, output); + + } + + @ParameterizedTest + @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) + public void testReadL4Sample(String mag) throws IOException, ZarrException { + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); + StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); + + Array httpArray = Array.open(httpStoreHandle); + Array localArray = Array.open(localStoreHandle); + System.out.println(httpArray); + System.out.println(localArray); + + ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData1, localData1); + + //offset to where l4_sample contains non-zero values + long[] offset = new long[4]; + long[] originalOffset = new long[]{0, 3073, 3073, 513}; + long[] originalShape = new long[]{1, 4096, 4096, 2048}; + long[] arrayShape = httpArray.metadata.shape; + for (int i = 0; i < 4; i++) { + offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); + } + + ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData2, localData2); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testParallel(boolean useParallel) throws IOException, ZarrException { + int[] testData = new int[512 * 512 * 512]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(512, 512, 512) + .withDataType(DataType.UINT32) + .withChunkShape(100, 100, 100) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(useParallel); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + clearTestoutputFolder(); + } + + @Test + public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { + // non-empty storage transformers are currently not supported + + Map[] storageTransformersEmpty = Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + ).metadata.storageTransformers; + assert storageTransformersEmpty.length == 0; + + assertThrows(JsonMappingException.class, () -> Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) + ); + + ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) + .withDataType(DataType.UINT8) + .withStorageTransformers(new HashMap[]{new HashMap() {{ + put("some", "value"); + }}}); + + assertThrows(ZarrException.class, () -> Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() + )); + } } From e07b7fe0b3126fe3d6b49a6b3718234e6feb77dc Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Sep 2025 15:04:25 +0200 Subject: [PATCH 02/27] v2 and v3 Array implements Array Interface --- .../dev/zarr/zarrjava/interfaces/Array.java | 337 ++++++++++++++++++ .../zarrjava/interfaces/ArrayMetadata.java | 21 ++ .../zarr/zarrjava/interfaces/DataType.java | 5 + src/main/java/dev/zarr/zarrjava/v2/Array.java | 34 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 40 ++- .../zarrjava/v2/ArrayMetadataBuilder.java | 13 +- .../java/dev/zarr/zarrjava/v2/DataTypeV2.java | 33 +- src/main/java/dev/zarr/zarrjava/v3/Array.java | 308 +--------------- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 22 +- .../java/dev/zarr/zarrjava/v3/DataType.java | 2 +- src/main/java/dev/zarr/zarrjava/v3/Node.java | 4 + .../zarr/zarrjava/v3/codec/CodecBuilder.java | 7 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 22 ++ 13 files changed, 540 insertions(+), 308 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/interfaces/Array.java create mode 100644 src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java create mode 100644 src/main/java/dev/zarr/zarrjava/interfaces/DataType.java diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java new file mode 100644 index 00000000..d600b05f --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java @@ -0,0 +1,337 @@ +package dev.zarr.zarrjava.interfaces; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.IndexingUtils; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.codec.CodecPipeline; +import ucar.ma2.InvalidRangeException; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.nio.ByteBuffer; +import java.util.Arrays; +import java.util.stream.Stream; + +public interface Array { + + ArrayMetadata metadata(); + + /** + * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array + * needs be large enough for the write. + * + * @param offset + * @param array + * @param parallel + */ + default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { + ArrayMetadata metadata = metadata(); + if (offset.length != metadata.ndim()) { + throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); + } + if (array.getRank() != metadata.ndim()) { + throw new IllegalArgumentException("'array' needs to have rank '" + metadata.ndim() + "'."); + } + + int[] shape = array.getShape(); + + final int[] chunkShape = metadata.chunkShape(); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + if (parallel) { + chunkStream = chunkStream.parallel(); + } + chunkStream.forEach( + chunkCoords -> { + try { + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + shape + ); + + ucar.ma2.Array chunkArray; + if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape, + chunkShape + )) { + chunkArray = array.sectionNoReduce(chunkProjection.outOffset, + chunkProjection.shape, + null + ); + } else { + chunkArray = readChunk(chunkCoords); + MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray, + chunkProjection.chunkOffset, chunkProjection.shape + ); + } + writeChunk(chunkCoords, chunkArray); + } catch (ZarrException | InvalidRangeException e) { + throw new RuntimeException(e); + } + }); + + } + + /** + * Writes one chunk into the Zarr array as specified by the chunk coordinates. The shape of the + * Zarr array needs be large enough for the write. + * + * @param chunkCoords + * @param chunkArray + * @throws ZarrException + */ + default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { + ArrayMetadata metadata = metadata(); + String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); + StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + + if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue())) { + chunkHandle.delete(); + } else { + ByteBuffer chunkBytes = codecPipeline().encode(chunkArray); + chunkHandle.set(chunkBytes); + } + } + + /** + * Reads one chunk of the Zarr array as specified by the chunk coordinates into an + * ucar.ma2.Array. + * + * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided + * by the chunk shape. + * @throws ZarrException + */ + @Nonnull + default ucar.ma2.Array readChunk(long[] chunkCoords) + throws ZarrException { + ArrayMetadata metadata = metadata(); + if (!chunkIsInArray(chunkCoords)) { + throw new ZarrException("Attempting to read data outside of the array's domain."); + } + + final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); + final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + + ByteBuffer chunkBytes = chunkHandle.read(); + if (chunkBytes == null) { + return metadata.allocateFillValueChunk(); + } + + return codecPipeline().decode(chunkBytes); + } + + + /** + * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of + * the Zarr array needs be large enough for the write. + * Utilizes no parallelism. + * + * @param array + */ + default void write(ucar.ma2.Array array) { + write(new long[metadata().ndim()], array); + } + + /** + * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array + * needs be large enough for the write. + * Utilizes no parallelism. + * + * @param offset + * @param array + */ + default void write(long[] offset, ucar.ma2.Array array) { + write(offset, array, false); + } + + /** + * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of + * the Zarr array needs be large enough for the write. + * + * @param array + * @param parallel + */ + default void write(ucar.ma2.Array array, boolean parallel) { + write(new long[metadata().ndim()], array, parallel); + } + + /** + * Reads the entire Zarr array into an ucar.ma2.Array. + * Utilizes no parallelism. + * + * @throws ZarrException + */ + @Nonnull + default ucar.ma2.Array read() throws ZarrException { + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape())); + } + + /** + * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. + * Utilizes no parallelism. + * + * @param offset + * @param shape + * @throws ZarrException + */ + @Nonnull + default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { + return read(offset, shape, false); + } + + /** + * Reads the entire Zarr array into an ucar.ma2.Array. + * + * @param parallel + * @throws ZarrException + */ + @Nonnull + default ucar.ma2.Array read(final boolean parallel) throws ZarrException { + return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape()), parallel); + } + + default boolean chunkIsInArray(long[] chunkCoords) { + final int[] chunkShape = metadata().chunkShape(); + for (int dimIdx = 0; dimIdx < metadata().ndim(); dimIdx++) { + if (chunkCoords[dimIdx] < 0 + || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata().shape()[dimIdx]) { + return false; + } + } + return true; + } + + + StoreHandle storeHandle(); + + CodecPipeline codecPipeline(); + + /** + * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. + * + * @param offset + * @param shape + * @param parallel + * @throws ZarrException + */ + @Nonnull + default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { + ArrayMetadata metadata = metadata(); + CodecPipeline codecPipeline = codecPipeline(); + if (offset.length != metadata.ndim()) { + throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); + } + if (shape.length != metadata.ndim()) { + throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'."); + } + for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { + if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape()[dimIdx]) { + throw new ZarrException("Requested data is outside of the array's domain."); + } + } + + final int[] chunkShape = metadata.chunkShape(); + if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape)) { + return readChunk(IndexingUtils.computeSingleChunkCoords(offset, chunkShape)); + } + + final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(), + shape); + Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape(), chunkShape, offset, shape)); + if (parallel) { + chunkStream = chunkStream.parallel(); + } + chunkStream.forEach( + chunkCoords -> { + try { + final IndexingUtils.ChunkProjection chunkProjection = + IndexingUtils.computeProjection(chunkCoords, metadata.shape(), chunkShape, offset, + shape + ); + + if (chunkIsInArray(chunkCoords)) { + MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(), + chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset, + chunkProjection.shape + ); + } + + final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); + final StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + if (!chunkHandle.exists()) { + return; + } + if (codecPipeline.supportsPartialDecode()) { + final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, + Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); + MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, + chunkProjection.outOffset, chunkProjection.shape + ); + } else { + MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset, + outputArray, chunkProjection.outOffset, chunkProjection.shape + ); + } + + } catch (ZarrException e) { + throw new RuntimeException(e); + } + }); + return outputArray; + } + + default ArrayAccessor access() { + return new ArrayAccessor(this); + } + + final class ArrayAccessor { + @Nullable + long[] offset; + @Nullable + int[] shape; + @Nonnull + Array array; + + public ArrayAccessor(@Nonnull Array array) { + this.array = array; + } + + @Nonnull + public ArrayAccessor withOffset(@Nonnull long... offset) { + this.offset = offset; + return this; + } + + + @Nonnull + public ArrayAccessor withShape(@Nonnull int... shape) { + this.shape = shape; + return this; + } + + @Nonnull + public ArrayAccessor withShape(@Nonnull long... shape) { + this.shape = Utils.toIntArray(shape); + return this; + } + + @Nonnull + public ucar.ma2.Array read() throws ZarrException { + if (offset == null) { + throw new ZarrException("`offset` needs to be set."); + } + if (shape == null) { + throw new ZarrException("`shape` needs to be set."); + } + return array.read(offset, shape); + } + + public void write(@Nonnull ucar.ma2.Array content) throws ZarrException { + if (offset == null) { + throw new ZarrException("`offset` needs to be set."); + } + array.write(offset, content); + } + + } +} \ No newline at end of file diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java new file mode 100644 index 00000000..7e79e0dd --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java @@ -0,0 +1,21 @@ +package dev.zarr.zarrjava.interfaces; + +import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; +import ucar.ma2.Array; + +public interface ArrayMetadata { + int ndim(); + + int[] chunkShape(); + + long[] shape(); + + DataType dataType(); + + Array allocateFillValueChunk(); + + ChunkKeyEncoding chunkKeyEncoding(); + + Object parsedFillValue(); + +} diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/DataType.java b/src/main/java/dev/zarr/zarrjava/interfaces/DataType.java new file mode 100644 index 00000000..04112047 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/interfaces/DataType.java @@ -0,0 +1,5 @@ +package dev.zarr.zarrjava.interfaces; + +public interface DataType { + ucar.ma2.DataType getMA2DataType(); +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 12455aa7..d65d335f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -3,20 +3,25 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.utils.IndexingUtils; +import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecPipeline; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import ucar.ma2.InvalidRangeException; +import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; +import java.util.stream.Stream; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; -public class Array { +public class Array implements dev.zarr.zarrjava.interfaces.Array { static final String ZARRAY = ".zarray"; public ArrayMetadata metadata; @@ -70,6 +75,17 @@ public static Array create(StoreHandle storeHandle, arrayMetadataBuilderMapper.apply(new ArrayMetadataBuilder()).build(), existsOk); } + @Nonnull + public static ArrayMetadataBuilder metadataBuilder() { + return new ArrayMetadataBuilder(); + } + + @Nonnull + public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadata) { + return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); + } + + @Override public String toString() { @@ -80,4 +96,20 @@ public String toString() { metadata.dataType ); } + + @Override + public ArrayMetadata metadata() { + return metadata; + } + + @Override + public StoreHandle storeHandle() { + return storeHandle; + } + + @Override + public CodecPipeline codecPipeline() { + return codecPipeline; + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index ddd55966..ed41e31b 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -5,15 +5,17 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v3.codec.Codec; +import ucar.ma2.Array; import javax.annotation.Nullable; import static dev.zarr.zarrjava.v3.ArrayMetadata.parseFillValue; -public class ArrayMetadata { +public class ArrayMetadata implements dev.zarr.zarrjava.interfaces.ArrayMetadata { static final int ZARR_FORMAT = 2; @JsonProperty("zarr_format") @@ -60,6 +62,7 @@ public ArrayMetadata( @Nullable @JsonProperty(value = "filters") Codec[] filters, //todo can be "null" @Nullable @JsonProperty(value = "compressor") Codec compressor //todo can be "null" ) throws ZarrException { + super(); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); @@ -84,4 +87,39 @@ public ArrayMetadata( } + public int ndim() { + return shape.length; + } + + @Override + public int[] chunkShape() { + return chunks; + } + + @Override + public long[] shape() { + return shape; + } + + @Override + public DataType dataType() { + return dataType; + } + + @Override + public Array allocateFillValueChunk() { + //TODO + return null; + } + + @Override + public ChunkKeyEncoding chunkKeyEncoding() { + //TODO + return null; + } + + @Override + public Object parsedFillValue() { + return parsedFillValue; + } } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index bf618a78..bcbdf46c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -81,7 +81,7 @@ public ArrayMetadataBuilder withFilters(Function cod } CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3()); this.filters = codecBuilder.apply(nestedCodecBuilder) - .build(); + .build(false); return this; } @@ -90,6 +90,17 @@ public ArrayMetadataBuilder withCompressor(Codec compressor) { return this; } + public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { + try { + this.compressor = new CodecBuilder(dataTypeV2.toV3()) + .withBlosc(cname, shuffle, clevel) + .build(false)[0]; + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new IllegalStateException("Please call `withShape` first."); diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java b/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java index ae661bae..734d4863 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java @@ -4,7 +4,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.DataType; -public enum DataTypeV2 { +public enum DataTypeV2 implements dev.zarr.zarrjava.interfaces.DataType{ BOOL("b1", Endianness.UNSPECIFIED), INT8("i1", Endianness.UNSPECIFIED), INT16("i2", Endianness.LITTLE), @@ -88,4 +88,35 @@ public DataType toV3() throws ZarrException { public String getValue() { return String.format("%s%s", endianness.getValue(), dtype); } + + @Override + public ucar.ma2.DataType getMA2DataType() { + switch (this) { + case BOOL: + return ucar.ma2.DataType.BOOLEAN; + case INT8: + return ucar.ma2.DataType.BYTE; + case INT16: + return ucar.ma2.DataType.SHORT; + case INT32: + return ucar.ma2.DataType.INT; + case INT64: + return ucar.ma2.DataType.LONG; + case UINT8: + return ucar.ma2.DataType.UBYTE; + case UINT16: + return ucar.ma2.DataType.USHORT; + case UINT32: + return ucar.ma2.DataType.UINT; + case UINT64: + return ucar.ma2.DataType.ULONG; + case FLOAT32: + return ucar.ma2.DataType.FLOAT; + case FLOAT64: + return ucar.ma2.DataType.DOUBLE; + default: + throw new RuntimeException("Unreachable"); + } + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 0d240be2..973aa7e4 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecPipeline; @@ -14,12 +13,9 @@ import java.util.Map; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import ucar.ma2.InvalidRangeException; -public class Array extends Node { +public class Array extends Node implements dev.zarr.zarrjava.interfaces.Array { public ArrayMetadata metadata; CodecPipeline codecPipeline; @@ -118,261 +114,20 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } - /** - * Reads the entire Zarr array into an ucar.ma2.Array. - * Utilizes no parallelism. - * - * @throws ZarrException - */ - @Nonnull - public ucar.ma2.Array read() throws ZarrException { - return read(new long[metadata.ndim()], Utils.toIntArray(metadata.shape)); - } - - /** - * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. - * Utilizes no parallelism. - * - * @param offset - * @param shape - * @throws ZarrException - */ - @Nonnull - public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { - return read(offset, shape, false); - } - - /** - * Reads the entire Zarr array into an ucar.ma2.Array. - * - * @param parallel - * @throws ZarrException - */ - @Nonnull - public ucar.ma2.Array read(final boolean parallel) throws ZarrException { - return read(new long[metadata.ndim()], Utils.toIntArray(metadata.shape), parallel); - } - - /** - * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. - * - * @param offset - * @param shape - * @param parallel - * @throws ZarrException - */ - @Nonnull - public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { - if (offset.length != metadata.ndim()) { - throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); - } - if (shape.length != metadata.ndim()) { - throw new IllegalArgumentException("'shape' needs to have rank '" + metadata.ndim() + "'."); - } - for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { - if (offset[dimIdx] < 0 || offset[dimIdx] + shape[dimIdx] > metadata.shape[dimIdx]) { - throw new ZarrException("Requested data is outside of the array's domain."); - } - } - - final int[] chunkShape = metadata.chunkShape(); - if (IndexingUtils.isSingleFullChunk(offset, shape, chunkShape)) { - return readChunk(IndexingUtils.computeSingleChunkCoords(offset, chunkShape)); - } - - final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType.getMA2DataType(), - shape); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); - if (parallel) { - chunkStream = chunkStream.parallel(); - } - chunkStream.forEach( - chunkCoords -> { - try { - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape - ); - - if (chunkIsInArray(chunkCoords)) { - MultiArrayUtils.copyRegion(metadata.allocateFillValueChunk(), - chunkProjection.chunkOffset, outputArray, chunkProjection.outOffset, - chunkProjection.shape - ); - } - - final String[] chunkKeys = metadata.chunkKeyEncoding.encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - if (!chunkHandle.exists()) { - return; - } - if (codecPipeline.supportsPartialDecode()) { - final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle, - Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape); - MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray, - chunkProjection.outOffset, chunkProjection.shape - ); - } else { - MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset, - outputArray, chunkProjection.outOffset, chunkProjection.shape - ); - } - - } catch (ZarrException e) { - throw new RuntimeException(e); - } - }); - return outputArray; - } - - boolean chunkIsInArray(long[] chunkCoords) { - final int[] chunkShape = metadata.chunkShape(); - for (int dimIdx = 0; dimIdx < metadata.ndim(); dimIdx++) { - if (chunkCoords[dimIdx] < 0 - || chunkCoords[dimIdx] * chunkShape[dimIdx] >= metadata.shape[dimIdx]) { - return false; - } - } - return true; - } - - /** - * Reads one chunk of the Zarr array as specified by the chunk coordinates into an - * ucar.ma2.Array. - * - * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided - * by the chunk shape. - * @throws ZarrException - */ - @Nonnull - public ucar.ma2.Array readChunk(long[] chunkCoords) - throws ZarrException { - if (!chunkIsInArray(chunkCoords)) { - throw new ZarrException("Attempting to read data outside of the array's domain."); - } - - final String[] chunkKeys = metadata.chunkKeyEncoding.encodeChunkKey(chunkCoords); - final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - - ByteBuffer chunkBytes = chunkHandle.read(); - if (chunkBytes == null) { - return metadata.allocateFillValueChunk(); - } - - return codecPipeline.decode(chunkBytes); - } - - /** - * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of - * the Zarr array needs be large enough for the write. - * Utilizes no parallelism. - * - * @param array - */ - public void write(ucar.ma2.Array array) { - write(new long[metadata.ndim()], array); - } - - /** - * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array - * needs be large enough for the write. - * Utilizes no parallelism. - * - * @param offset - * @param array - */ - public void write(long[] offset, ucar.ma2.Array array) { - write(offset, array, false); - } - - /** - * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of - * the Zarr array needs be large enough for the write. - * - * @param array - * @param parallel - */ - public void write(ucar.ma2.Array array, boolean parallel) { - write(new long[metadata.ndim()], array, parallel); + @Override + public CodecPipeline codecPipeline() { + return codecPipeline; } - /** - * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array - * needs be large enough for the write. - * - * @param offset - * @param array - * @param parallel - */ - public void write(long[] offset, ucar.ma2.Array array, boolean parallel) { - if (offset.length != metadata.ndim()) { - throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'."); - } - if (array.getRank() != metadata.ndim()) { - throw new IllegalArgumentException("'array' needs to have rank '" + metadata.ndim() + "'."); - } - - int[] shape = array.getShape(); - final int[] chunkShape = metadata.chunkShape(); - Stream chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape)); - if(parallel) { - chunkStream = chunkStream.parallel(); - } - chunkStream.forEach( - chunkCoords -> { - try { - final IndexingUtils.ChunkProjection chunkProjection = - IndexingUtils.computeProjection(chunkCoords, metadata.shape, chunkShape, offset, - shape - ); - - ucar.ma2.Array chunkArray; - if (IndexingUtils.isFullChunk(chunkProjection.chunkOffset, chunkProjection.shape, - chunkShape - )) { - chunkArray = array.sectionNoReduce(chunkProjection.outOffset, - chunkProjection.shape, - null - ); - } else { - chunkArray = readChunk(chunkCoords); - MultiArrayUtils.copyRegion(array, chunkProjection.outOffset, chunkArray, - chunkProjection.chunkOffset, chunkProjection.shape - ); - } - writeChunk(chunkCoords, chunkArray); - } catch (ZarrException | InvalidRangeException e) { - throw new RuntimeException(e); - } - }); + @Override + public ArrayMetadata metadata() { + return metadata; } - /** - * Writes one chunk into the Zarr array as specified by the chunk coordinates. The shape of the - * Zarr array needs be large enough for the write. - * - * @param chunkCoords - * @param chunkArray - * @throws ZarrException - */ - public void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { - String[] chunkKeys = metadata.chunkKeyEncoding.encodeChunkKey(chunkCoords); - StoreHandle chunkHandle = storeHandle.resolve(chunkKeys); - - if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue)) { - chunkHandle.delete(); - } else { - ByteBuffer chunkBytes = codecPipeline.encode(chunkArray); - chunkHandle.set(chunkBytes); - } - } - public ArrayAccessor access() { - return new ArrayAccessor(this); - } private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException, IOException { ObjectMapper objectMapper = makeObjectMapper(); @@ -445,54 +200,5 @@ public String toString() { ); } - public static final class ArrayAccessor { - - @Nullable - long[] offset; - @Nullable - int[] shape; - @Nonnull - Array array; - - private ArrayAccessor(@Nonnull Array array) { - this.array = array; - } - - @Nonnull - public ArrayAccessor withOffset(@Nonnull long... offset) { - this.offset = offset; - return this; - } - - - @Nonnull - public ArrayAccessor withShape(@Nonnull int... shape) { - this.shape = shape; - return this; - } - @Nonnull - public ArrayAccessor withShape(@Nonnull long... shape) { - this.shape = Utils.toIntArray(shape); - return this; - } - - @Nonnull - public ucar.ma2.Array read() throws ZarrException { - if (offset == null) { - throw new ZarrException("`offset` needs to be set."); - } - if (shape == null) { - throw new ZarrException("`shape` needs to be set."); - } - return array.read(offset, shape); - } - - public void write(@Nonnull ucar.ma2.Array content) throws ZarrException { - if (offset == null) { - throw new ZarrException("`offset` needs to be set."); - } - array.write(offset, content); - } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index f8f5b2bd..fb7cf87a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -20,7 +20,7 @@ import javax.annotation.Nullable; -public final class ArrayMetadata { +public final class ArrayMetadata implements dev.zarr.zarrjava.interfaces.ArrayMetadata { static final String NODE_TYPE = "array"; static final int ZARR_FORMAT = 3; @@ -258,6 +258,16 @@ public ucar.ma2.Array allocateFillValueChunk() { return coreArrayMetadata.allocateFillValueChunk(); } + @Override + public ChunkKeyEncoding chunkKeyEncoding() { + return chunkKeyEncoding; + } + + @Override + public Object parsedFillValue() { + return parsedFillValue; + } + public int ndim() { return shape.length; } @@ -270,6 +280,16 @@ public int[] chunkShape() { return ((RegularChunkGrid) this.chunkGrid).configuration.chunkShape; } + @Override + public long[] shape() { + return shape; + } + + @Override + public DataType dataType() { + return dataType; + } + public int chunkSize() { return coreArrayMetadata.chunkSize(); } diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index d76fdc29..46384f56 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonValue; -public enum DataType { +public enum DataType implements dev.zarr.zarrjava.interfaces.DataType { BOOL("bool", 1), INT8("int8", 1), INT16("int16", 2), diff --git a/src/main/java/dev/zarr/zarrjava/v3/Node.java b/src/main/java/dev/zarr/zarrjava/v3/Node.java index e2164ccf..4362999d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Node.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Node.java @@ -24,4 +24,8 @@ public static ObjectMapper makeObjectMapper() { return objectMapper; } + public StoreHandle storeHandle() { + return storeHandle; + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 3776a433..7d9a3b01 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -160,8 +160,13 @@ private void autoInsertBytesCodec() { } public Codec[] build() { - autoInsertBytesCodec(); + return build(true); + } + public Codec[] build(boolean includeBytesCodec) { + if (includeBytesCodec) { + autoInsertBytesCodec(); + } return codecs.toArray(new Codec[0]); } } diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index b1b1d390..96713f77 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -12,6 +12,7 @@ import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; @@ -696,5 +697,26 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I builderWithStorageTransformer.build() )); } + + // TODO test withBloscCompressor + @ParameterizedTest + @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) + public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException { + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withFillValue(1) + .withBloscCompressor(cname, shuffle, clevel) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(outArray.getSize(), 8 * 8); + Assertions.assertEquals(outArray.getByte(0), 0); + } } From e287318646e84727566f5ccfb78dbb8f513b5cd9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Sep 2025 16:10:55 +0200 Subject: [PATCH 03/27] documentation for Array methods and improve parameter descriptions --- .../dev/zarr/zarrjava/interfaces/Array.java | 47 +++++++-------- src/main/java/dev/zarr/zarrjava/v2/Array.java | 34 +++++++++-- src/main/java/dev/zarr/zarrjava/v3/Array.java | 57 ++++++++++--------- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 1 - 4 files changed, 81 insertions(+), 58 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java index d600b05f..4dcbdea1 100644 --- a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java +++ b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java @@ -22,9 +22,9 @@ public interface Array { * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array * needs be large enough for the write. * - * @param offset - * @param array - * @param parallel + * @param offset the offset where to write the data + * @param array the data to write + * @param parallel utilizes parallelism if true */ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { ArrayMetadata metadata = metadata(); @@ -74,11 +74,12 @@ default void write(long[] offset, ucar.ma2.Array array, boolean parallel) { /** * Writes one chunk into the Zarr array as specified by the chunk coordinates. The shape of the - * Zarr array needs be large enough for the write. + * Zarr array needs to be large enough to write. * - * @param chunkCoords - * @param chunkArray - * @throws ZarrException + * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided + * by the chunk shape. + * @param chunkArray The data to write into the chunk + * @throws ZarrException throws ZarrException if the write fails */ default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws ZarrException { ArrayMetadata metadata = metadata(); @@ -99,7 +100,7 @@ default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Za * * @param chunkCoords The coordinates of the chunk as computed by the offset of the chunk divided * by the chunk shape. - * @throws ZarrException + * @throws ZarrException throws ZarrException if the requested chunk is outside the array's domain or if the read fails */ @Nonnull default ucar.ma2.Array readChunk(long[] chunkCoords) @@ -126,7 +127,7 @@ default ucar.ma2.Array readChunk(long[] chunkCoords) * the Zarr array needs be large enough for the write. * Utilizes no parallelism. * - * @param array + * @param array the data to write */ default void write(ucar.ma2.Array array) { write(new long[metadata().ndim()], array); @@ -137,8 +138,8 @@ default void write(ucar.ma2.Array array) { * needs be large enough for the write. * Utilizes no parallelism. * - * @param offset - * @param array + * @param offset the offset where to write the data + * @param array the data to write */ default void write(long[] offset, ucar.ma2.Array array) { write(offset, array, false); @@ -148,8 +149,8 @@ default void write(long[] offset, ucar.ma2.Array array) { * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of * the Zarr array needs be large enough for the write. * - * @param array - * @param parallel + * @param array the data to write + * @param parallel utilizes parallelism if true */ default void write(ucar.ma2.Array array, boolean parallel) { write(new long[metadata().ndim()], array, parallel); @@ -159,7 +160,7 @@ default void write(ucar.ma2.Array array, boolean parallel) { * Reads the entire Zarr array into an ucar.ma2.Array. * Utilizes no parallelism. * - * @throws ZarrException + * @throws ZarrException throws ZarrException if the read fails */ @Nonnull default ucar.ma2.Array read() throws ZarrException { @@ -170,9 +171,9 @@ default ucar.ma2.Array read() throws ZarrException { * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. * Utilizes no parallelism. * - * @param offset - * @param shape - * @throws ZarrException + * @param offset the offset where to start reading + * @param shape the shape of the data to read + * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException { @@ -182,8 +183,8 @@ default ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrE /** * Reads the entire Zarr array into an ucar.ma2.Array. * - * @param parallel - * @throws ZarrException + * @param parallel utilizes parallelism if true + * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull default ucar.ma2.Array read(final boolean parallel) throws ZarrException { @@ -209,10 +210,10 @@ default boolean chunkIsInArray(long[] chunkCoords) { /** * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. * - * @param offset - * @param shape - * @param parallel - * @throws ZarrException + * @param offset the offset where to start reading + * @param shape the shape of the data to read + * @param parallel utilizes parallelism if true + * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails */ @Nonnull default ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException { diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index d65d335f..8cf4d77f 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -3,13 +3,10 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.utils.IndexingUtils; -import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecPipeline; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import ucar.ma2.InvalidRangeException; import javax.annotation.Nonnull; import java.io.IOException; @@ -17,7 +14,6 @@ import java.util.Arrays; import java.util.function.Function; import java.util.stream.Collectors; -import java.util.stream.Stream; import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; @@ -38,6 +34,13 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE ), metadata.coreArrayMetadata); } + /** + * Opens an existing Zarr array at a specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened + */ public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { return new Array( storeHandle, @@ -49,11 +52,32 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. This + * method will raise an exception if a Zarr array already exists at the specified storage + * location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created + */ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { return Array.create(storeHandle, arrayMetadata, false); } + /** + * Creates a new Zarr array with the provided metadata at a specified storage location. If + * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the + * specified storage location. + * + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException throws IOException if the metadata cannot be serialized + * @throws ZarrException throws ZarrException if the Zarr array cannot be created + */ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) throws IOException, ZarrException { StoreHandle metadataHandle = storeHandle.resolve(ZARRAY); @@ -85,8 +109,6 @@ public static ArrayMetadataBuilder metadataBuilder(ArrayMetadata existingMetadat return ArrayMetadataBuilder.fromArrayMetadata(existingMetadata); } - - @Override public String toString() { return String.format("", storeHandle, diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 973aa7e4..9e5c06b3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -3,7 +3,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.codec.CodecPipeline; import java.io.IOException; @@ -21,7 +20,7 @@ public class Array extends Node implements dev.zarr.zarrjava.interfaces.Array { CodecPipeline codecPipeline; protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) - throws IOException, ZarrException { + throws ZarrException { super(storeHandle); this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(arrayMetadata.codecs, arrayMetadata.coreArrayMetadata); @@ -30,9 +29,9 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) /** * Opens an existing Zarr array at a specified storage location. * - * @param storeHandle - * @throws IOException - * @throws ZarrException + * @param storeHandle the storage location of the Zarr array + * @throws IOException throws IOException if the metadata cannot be read + * @throws ZarrException throws ZarrException if the Zarr array cannot be opened */ public static Array open(StoreHandle storeHandle) throws IOException, ZarrException { return new Array( @@ -50,10 +49,10 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept * method will raise an exception if a Zarr array already exists at the specified storage * location. * - * @param storeHandle - * @param arrayMetadata - * @throws IOException - * @throws ZarrException + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created */ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOException, ZarrException { @@ -65,11 +64,11 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata) * `existsOk` is false, this method will raise an exception if a Zarr array already exists at the * specified storage location. * - * @param storeHandle - * @param arrayMetadata - * @param existsOk - * @throws IOException - * @throws ZarrException + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadata the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException throws IOException if the metadata cannot be serialized + * @throws ZarrException throws ZarrException if the Zarr array cannot be created */ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, boolean existsOk) throws IOException, ZarrException { @@ -91,11 +90,12 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, * be used to construct the metadata of the Zarr array. If `existsOk` is false, this method will * raise an exception if a Zarr array already exists at the specified storage location. * - * @param storeHandle - * @param arrayMetadataBuilderMapper - * @param existsOk - * @throws IOException - * @throws ZarrException + * @param storeHandle the storage location of the Zarr array + * @param arrayMetadataBuilderMapper a callback of ArrayMetadataBuilder -> ArrayMetadataBuilder that is + * used to construct the metadata of the Zarr array + * @param existsOk if true, no exception is raised if the Zarr array already exists + * @throws IOException if the metadata cannot be serialized + * @throws ZarrException if the Zarr array cannot be created */ public static Array create(StoreHandle storeHandle, Function arrayMetadataBuilderMapper, @@ -142,9 +142,9 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException * deleted. This method returns a new instance of the Zarr array class and the old instance * becomes invalid. * - * @param newShape - * @throws ZarrException - * @throws IOException + * @param newShape the new shape of the Zarr array + * @throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array resize(long[] newShape) throws ZarrException, IOException { if (newShape.length != metadata.ndim()) { @@ -162,9 +162,9 @@ public Array resize(long[] newShape) throws ZarrException, IOException { * Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. * - * @param newAttributes - * @throws ZarrException - * @throws IOException + * @param newAttributes the new attributes of the Zarr array + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array setAttributes(Map newAttributes) throws ZarrException, IOException { ArrayMetadata newArrayMetadata = @@ -180,9 +180,10 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * callback may be mutated. This method overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. * - * @param attributeMapper - * @throws ZarrException - * @throws IOException + * @param attributeMapper a callback of Map -> Map that is used to construct the new + * attributes + * @throws ZarrException throws ZarrException if the new metadata is invalid + * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function, Map> attributeMapper) throws ZarrException, IOException { diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 96713f77..9a16abf8 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -12,7 +12,6 @@ import dev.zarr.zarrjava.v3.*; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; From 02f3a27a39d8f336330d674454e1215ddad98555 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Sep 2025 17:40:49 +0200 Subject: [PATCH 04/27] adopt allocateFillValueChunk() and chunkKeyEncoding() --- src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java | 11 +++++++---- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 1 - 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index ed41e31b..0cc8a004 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -4,9 +4,11 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import ucar.ma2.Array; @@ -108,14 +110,15 @@ public DataType dataType() { @Override public Array allocateFillValueChunk() { - //TODO - return null; + ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunks); + MultiArrayUtils.fill(outputArray, parsedFillValue); + return outputArray; } @Override public ChunkKeyEncoding chunkKeyEncoding() { - //TODO - return null; + Separator separator = dimensionSeparator == null ? Separator.DOT : dimensionSeparator; + return new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(separator)); } @Override diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 9a16abf8..dc20a395 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -697,7 +697,6 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I )); } - // TODO test withBloscCompressor @ParameterizedTest @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException { From aac56722480e7d62c16df12bc93b88ff9422fa03 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Sep 2025 22:59:41 +0200 Subject: [PATCH 05/27] zarrita tests with uv --- .github/workflows/ci.yml | 12 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 1246 ++++++++--------- 2 files changed, 608 insertions(+), 650 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 29852b58..65e935e6 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,15 +28,11 @@ jobs: distribution: 'temurin' cache: maven - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - - name: Install zarrita + - name: Set up uv and zarrita run: | - python -m venv venv_zarrita - if [ "${{ runner.os }}" = "Windows" ]; then venv_zarrita/Scripts/pip install zarrita; else venv_zarrita/bin/pip install zarrita; fi + curl -Ls https://github.com/astral-sh/uv/releases/latest/download/uv-installer.sh | bash + uv venv && uv init + uv add zarrita - name: Download testdata run: | diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index dc20a395..1c167580 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -40,681 +40,643 @@ public class ZarrTest { - final static Path TESTDATA = Paths.get("testdata"); - final static Path TESTOUTPUT = Paths.get("testoutput"); - final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + final static Path TESTDATA = Paths.get("testdata"); + final static Path TESTOUTPUT = Paths.get("testoutput"); + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + @BeforeAll + public static void clearTestoutputFolder() throws IOException { + if (Files.exists(TESTOUTPUT)) { + try (Stream walk = Files.walk(TESTOUTPUT)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + Files.createDirectory(TESTOUTPUT); + } - public static String pythonPath() { - if (System.getProperty("os.name").startsWith("Windows")) { - return "venv_zarrita\\Scripts\\python.exe"; + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + + public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder("uv", "run", PYTHON_TEST_PATH.resolve("zarrita_write.py") + .toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + + Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); + ucar.ma2.Array result = array.read(); + + //for expected values see zarrita_write.py + Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); + Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); + Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + + int[] expectedData = new int[16 * 16]; + Arrays.setAll(expectedData, p -> p); + Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); } - return "venv_zarrita/bin/python"; - } - - @BeforeAll - public static void clearTestoutputFolder() throws IOException { - if (Files.exists(TESTOUTPUT)) { - try (Stream walk = Files.walk(TESTOUTPUT)) { - walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); - } + + @CsvSource({"0,true", "0,false", "5, true", "10, false"}) + @ParameterizedTest + public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { + //compress using ZstdCompressCtx + int number = 123456; + byte[] src = ByteBuffer.allocate(4).putInt(number).array(); + byte[] compressed; + try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { + ctx.setLevel(clevel); + ctx.setChecksum(checksumFlag); + compressed = ctx.compress(src); + } + //decompress with Zstd.decompress + long originalSize = Zstd.decompressedSize(compressed); + byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); + Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); + + //write compressed to file + String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); + try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { + fos.write(compressed); + } + + //decompress in python + Process process = new ProcessBuilder( + "uv", + "run", + PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), + compressedDataPath, + Integer.toString(number) + ).start(); + int exitCode = process.waitFor(); + assert exitCode == 0; } - Files.createDirectory(TESTOUTPUT); - } - - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - - public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { - String command = pythonPath(); - ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_write.py") - .toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + Map attributes = new HashMap<>(); + attributes.put("test_key", "test_value"); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withAttributes(attributes); + + switch (codec) { + case "blosc": + String cname = codecParam.split("_")[0]; + String shuffle = codecParam.split("_")[1]; + int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); + builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); + break; + case "gzip": + builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); + break; + case "zstd": + int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); + boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); + builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); + break; + case "bytes": + builder = builder.withCodecs(c -> c.withBytes(codecParam)); + break; + case "transpose": + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); + break; + case "sharding": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); + break; + case "sharding_nested": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); + break; + case "crc32c": + builder = builder.withCodecs(CodecBuilder::withCrc32c); + break; + default: + throw new IllegalArgumentException("Invalid Codec: " + codec); + } + + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + + //read in zarrita + ProcessBuilder pb = new ProcessBuilder("uv", "run", PYTHON_TEST_PATH.resolve("zarrita_read.py") + .toString(), codec, codecParam, TESTOUTPUT.toString()); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + static Stream> invalidCodecBuilder() { + return Stream.of( + c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), + c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), + c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) + ); + } + + @ParameterizedTest + @MethodSource("invalidCodecBuilder") + public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(new long[]{4, 4}) + .withDataType(DataType.UINT32) + .withChunkShape(new int[]{2, 2}) + .withCodecs(codecBuilder); + + assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); + } + + @Test + public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(32, 32, 32) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); } - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); + static Stream invalidChunkSizes() { + return Stream.of( + new int[]{1}, + new int[]{1, 1, 1} + ); } - int exitCode = process.waitFor(); - assert exitCode == 0; - - Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); - ucar.ma2.Array result = array.read(); - - //for expected values see zarrita_write.py - Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); - Assertions.assertEquals(42, array.metadata.attributes.get("answer")); - - int[] expectedData = new int[16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); - } - - @CsvSource({"0,true", "0,false", "5, true", "10, false"}) - @ParameterizedTest - public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { - //compress using ZstdCompressCtx - int number = 123456; - byte[] src = ByteBuffer.allocate(4).putInt(number).array(); - byte[] compressed; - try (ZstdCompressCtx ctx = new ZstdCompressCtx()) { - ctx.setLevel(clevel); - ctx.setChecksum(checksumFlag); - compressed = ctx.compress(src); + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidChunkDimensions(int[] chunkSize) { + long[] shape = new long[]{4, 4}; + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32) + .withChunkShape(chunkSize); + + assertThrows(ZarrException.class, builder::build); } - //decompress with Zstd.decompress - long originalSize = Zstd.decompressedSize(compressed); - byte[] decompressed = Zstd.decompress(compressed, (int) originalSize); - Assertions.assertEquals(number, ByteBuffer.wrap(decompressed).getInt()); - - //write compressed to file - String compressedDataPath = TESTOUTPUT.resolve("compressed" + clevel + checksumFlag + ".bin").toString(); - try (FileOutputStream fos = new FileOutputStream(compressedDataPath)) { - fos.write(compressed); + + static Stream invalidShardSizes() { + return Stream.of( + new int[]{4}, //wrong dims + new int[]{4, 4, 4}, //wrong dims + new int[]{1, 1}, //smaller than inner chunk shape + new int[]{5, 5}, //no exact multiple of inner chunk shape + new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension + new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension + ); } - //decompress in python - Process process = new ProcessBuilder( - pythonPath(), - PYTHON_TEST_PATH.resolve("zstd_decompress.py").toString(), - compressedDataPath, - Integer.toString(number) - ).start(); - int exitCode = process.waitFor(); - assert exitCode == 0; - } - - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - Map attributes = new HashMap<>(); - attributes.put("test_key", "test_value"); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withAttributes(attributes); - - switch (codec) { - case "blosc": - String cname = codecParam.split("_")[0]; - String shuffle = codecParam.split("_")[1]; - int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); - builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); - break; - case "gzip": - builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); - break; - case "zstd": - int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); - boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); - builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); - break; - case "bytes": - builder = builder.withCodecs(c -> c.withBytes(codecParam)); - break; - case "transpose": - builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); - break; - case "sharding": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); - break; - case "sharding_nested": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); - break; - case "crc32c": - builder = builder.withCodecs(CodecBuilder::withCrc32c); - break; - default: - throw new IllegalArgumentException("Invalid Codec: " + codec); + @ParameterizedTest + @MethodSource("invalidShardSizes") + public void testCheckShardingBounds(int[] shardSize) throws Exception { + long[] shape = new long[]{10, 10}; + int[] innerChunkSize = new int[]{2, 2}; + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(shape) + .withDataType(DataType.UINT32).withChunkShape(shardSize); + + if (false) { + int[] nestedChunkSize = new int[]{4, 4}; + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + } + builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); + assertThrows(ZarrException.class, builder::build); } - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + @ParameterizedTest + @CsvSource({"0,true", "0,false", "5, true", "5, false"}) + public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withCodecs(c -> c.withZstd(clevel, checksum)); + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); - Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + } - //read in zarrita - String command = pythonPath(); + @Test + public void testTransposeCodec() throws ZarrException { + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); + ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ + 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + new long[]{2, 3, 3}, + new int[]{2, 3, 3}, + DataType.UINT32, + null)); + + assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); + assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); + } - ProcessBuilder pb = new ProcessBuilder(command, PYTHON_TEST_PATH.resolve("zarrita_read.py") - .toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); + static Stream invalidTransposeOrder() { + return Stream.of( + new int[]{1, 0, 0}, + new int[]{1, 2, 3}, + new int[]{1, 2, 3, 0}, + new int[]{1, 2} + ); + } - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); + @ParameterizedTest + @MethodSource("invalidChunkSizes") + public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { + int[] shapeInt = new int[]{2, 3, 3}; + long[] shapeLong = new long[]{2, 3, 3}; + + TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); + transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( + shapeLong, + shapeInt, + DataType.UINT32, + null)); + + ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); + assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); } - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); + @Test + public void testFileSystemStores() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTDATA); + ObjectMapper objectMapper = Node.makeObjectMapper(); + + GroupMetadata group = objectMapper.readValue( + Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), + GroupMetadata.class + ); + + System.out.println(group); + System.out.println(objectMapper.writeValueAsString(group)); + + ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( + "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), + ArrayMetadata.class); + + System.out.println(arrayMetadata); + System.out.println(objectMapper.writeValueAsString(arrayMetadata)); + + System.out.println( + Array.open(fsStore.resolve("l4_sample", "color", "1"))); + System.out.println( + Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); + System.out.println( + Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() + .toArray(Node[]::new))); } - int exitCode = process.waitFor(); - assert exitCode == 0; - } - - static Stream> invalidCodecBuilder() { - return Stream.of( - c -> c.withBytes(BytesCodec.Endian.LITTLE).withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBlosc().withBytes(BytesCodec.Endian.LITTLE), - c -> c.withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}), - c -> c.withTranspose(new int[]{1, 0}).withBytes(BytesCodec.Endian.LITTLE).withTranspose(new int[]{1, 0}) - ); - } - - @ParameterizedTest - @MethodSource("invalidCodecBuilder") - public void testCheckInvalidCodecConfiguration(Function codecBuilder) throws Exception { - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_codec_config", String.valueOf(codecBuilder.hashCode())); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(new long[]{4, 4}) - .withDataType(DataType.UINT32) - .withChunkShape(new int[]{2, 2}) - .withCodecs(codecBuilder); - - assertThrows(ZarrException.class, () -> Array.create(storeHandle, builder.build())); - } - - @Test - public void testLargerChunkSizeThanArraySize() throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("larger_chunk_size_than_array"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(32, 32, 32) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - static Stream invalidChunkSizes() { - return Stream.of( - new int[]{1}, - new int[]{1, 1, 1} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidChunkDimensions(int[] chunkSize) { - long[] shape = new long[]{4, 4}; - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("invalid_chunksize"); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32) - .withChunkShape(chunkSize); - - assertThrows(ZarrException.class, builder::build); - } - - static Stream invalidShardSizes() { - return Stream.of( - new int[]{4}, //wrong dims - new int[]{4, 4, 4}, //wrong dims - new int[]{1, 1}, //smaller than inner chunk shape - new int[]{5, 5}, //no exact multiple of inner chunk shape - new int[]{2, 1}, //smaller than inner chunk shape in 2nd dimension - new int[]{2, 5} //no exact multiple of inner chunk shape in 2nd dimension - ); - } - - @ParameterizedTest - @MethodSource("invalidShardSizes") - public void testCheckShardingBounds(int[] shardSize) throws Exception { - long[] shape = new long[]{10, 10}; - int[] innerChunkSize = new int[]{2, 2}; - - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(shape) - .withDataType(DataType.UINT32).withChunkShape(shardSize); - - if (false) { - int[] nestedChunkSize = new int[]{4, 4}; - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2}, c1 -> c1.withSharding(nestedChunkSize, c2 -> c2.withBytes("LITTLE")))); + @Test + public void testS3Store() throws IOException, ZarrException { + S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() + .withRegion("eu-west-1") + .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) + .build(), "static.webknossos.org", "data"); + System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); } - builder = builder.withCodecs(c -> c.withSharding(innerChunkSize, c1 -> c1.withBytes("LITTLE"))); - assertThrows(ZarrException.class, builder::build); - } - - @ParameterizedTest - @CsvSource({"0,true", "0,false", "5, true", "5, false"}) - public void testZstdCodecReadWrite(int clevel, boolean checksum) throws ZarrException, IOException { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testZstdCodecReadWrite", "checksum_" + checksum, "clevel_" + clevel); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withCodecs(c -> c.withZstd(clevel, checksum)); - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - } - - @Test - public void testTransposeCodec() throws ZarrException { - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{2, 3, 3}, new int[]{ - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17}); - ucar.ma2.Array testDataTransposed120 = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{3, 3, 2}, new int[]{ - 0, 9, 1, 10, 2, 11, 3, 12, 4, 13, 5, 14, 6, 15, 7, 16, 8, 17}); - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(new int[]{1, 2, 0})); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - new long[]{2, 3, 3}, - new int[]{2, 3, 3}, - DataType.UINT32, - null)); - - assert MAMath.equals(testDataTransposed120, transposeCodec.encode(testData)); - assert MAMath.equals(testData, transposeCodec.decode(testDataTransposed120)); - } - - static Stream invalidTransposeOrder() { - return Stream.of( - new int[]{1, 0, 0}, - new int[]{1, 2, 3}, - new int[]{1, 2, 3, 0}, - new int[]{1, 2} - ); - } - - @ParameterizedTest - @MethodSource("invalidChunkSizes") - public void testCheckInvalidTransposeOrder(int[] transposeOrder) throws Exception { - int[] shapeInt = new int[]{2, 3, 3}; - long[] shapeLong = new long[]{2, 3, 3}; - - TransposeCodec transposeCodec = new TransposeCodec(new TransposeCodec.Configuration(transposeOrder)); - transposeCodec.setCoreArrayMetadata(new ArrayMetadata.CoreArrayMetadata( - shapeLong, - shapeInt, - DataType.UINT32, - null)); - - ucar.ma2.Array testData = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, shapeInt); - assertThrows(ZarrException.class, () -> transposeCodec.encode(testData)); - } - - @Test - public void testFileSystemStores() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTDATA); - ObjectMapper objectMapper = Node.makeObjectMapper(); - - GroupMetadata group = objectMapper.readValue( - Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")), - GroupMetadata.class - ); - - System.out.println(group); - System.out.println(objectMapper.writeValueAsString(group)); - - ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve( - "l4_sample").resolve("color").resolve("1").resolve("zarr.json")), - ArrayMetadata.class); - - System.out.println(arrayMetadata); - System.out.println(objectMapper.writeValueAsString(arrayMetadata)); - - System.out.println( - Array.open(fsStore.resolve("l4_sample", "color", "1"))); - System.out.println( - Arrays.toString(Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new))); - System.out.println( - Arrays.toString(((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list() - .toArray(Node[]::new))); - } - - @Test - public void testS3Store() throws IOException, ZarrException { - S3Store s3Store = new S3Store(AmazonS3ClientBuilder.standard() - .withRegion("eu-west-1") - .withCredentials(new AWSStaticCredentialsProvider(new AnonymousAWSCredentials())) - .build(), "static.webknossos.org", "data"); - System.out.println(Array.open(s3Store.resolve("zarr_v3", "l4_sample", "color", "1"))); - } - - @Test - public void testHttpStore() throws IOException, ZarrException { - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data/"); - dev.zarr.zarrjava.v2.Array v2Array = dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1")); - Array v3Array = Array.open(httpStore.resolve("zarr_v3", "l4_sample", "color", "1")); - - System.out.println(v2Array.metadata.fillValue); - System.out.println(v2Array.metadata.dataType); - System.out.println(v2Array.metadata.chunks[0]); - System.out.println(v2Array.metadata.filters); - System.out.println(v2Array.metadata.compressor); - System.out.println(v2Array.metadata.order); - System.out.println(v2Array.metadata.dimensionSeparator); - System.out.println(v3Array); - for (Codec codec : ((ShardingIndexedCodec) v3Array.metadata.codecs[0]).configuration.codecs) { - System.out.println(codec); + + @Test + public void testV3ShardingReadCutout() throws IOException, ZarrException { + Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); } - } - - @Test - public void testV3ShardingReadCutout() throws IOException, ZarrException { - Array array = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = array.read(new long[]{0, 3073, 3073, 513}, new int[]{1, 64, 64, 64}); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - } - - @Test - public void testV3Access() throws IOException, ZarrException { - Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); - - ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) - .withShape(1, 64, 64, 64) - .read(); - Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); - Assertions.assertEquals(outArray.getByte(0), -98); - - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), - readArray.metadata - ); - writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); - } - - @ParameterizedTest - @ValueSource(strings = {"start", "end"}) - public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); - ucar.ma2.Array readArrayContent = readArray.read(); - Array writeArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), - readArray.metadata - ); - writeArray.write(readArrayContent); - ucar.ma2.Array outArray = writeArray.read(); - - assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); - } - - @Test - public void testV3Codecs() throws IOException, ZarrException { - int[] readShape = new int[]{1, 1, 1024, 1024}; - Array readArray = Array.open( - new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); - ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); - { - Array gzipArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() - ); - gzipArray.write(readArrayContent); - ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + + @Test + public void testV3Access() throws IOException, ZarrException { + Array readArray = Array.open(new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "1")); + + ucar.ma2.Array outArray = readArray.access().withOffset(0, 3073, 3073, 513) + .withShape(1, 64, 64, 64) + .read(); + Assertions.assertEquals(outArray.getSize(), 64 * 64 * 64); + Assertions.assertEquals(outArray.getByte(0), -98); + + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_2", "color", "1"), + readArray.metadata + ); + writeArray.access().withOffset(0, 3073, 3073, 513).write(outArray); } - { - Array bloscArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() - ); - bloscArray.write(readArrayContent); - ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + + @ParameterizedTest + @ValueSource(strings = {"start", "end"}) + public void testV3ShardingReadWrite(String indexLocation) throws IOException, ZarrException { + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("sharding_index_location", indexLocation)); + ucar.ma2.Array readArrayContent = readArray.read(); + Array writeArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("sharding_index_location", indexLocation), + readArray.metadata + ); + writeArray.write(readArrayContent); + ucar.ma2.Array outArray = writeArray.read(); + + assert MultiArrayUtils.allValuesEqual(readArrayContent, outArray); } - { - Array zstdArray = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), - Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() - ); - zstdArray.write(readArrayContent); - ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); - assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + + @Test + public void testV3Codecs() throws IOException, ZarrException { + int[] readShape = new int[]{1, 1, 1024, 1024}; + Array readArray = Array.open( + new FilesystemStore(TESTDATA).resolve("l4_sample", "color", "8-8-2")); + ucar.ma2.Array readArrayContent = readArray.read(new long[4], readShape); + { + Array gzipArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_gzip", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withGzip(5)).build() + ); + gzipArray.write(readArrayContent); + ucar.ma2.Array outGzipArray = gzipArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outGzipArray, readArrayContent); + } + { + Array bloscArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_blosc", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withBlosc("zstd", 5)).build() + ); + bloscArray.write(readArrayContent); + ucar.ma2.Array outBloscArray = bloscArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outBloscArray, readArrayContent); + } + { + Array zstdArray = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("l4_sample_zstd", "color", "8-8-2"), + Array.metadataBuilder(readArray.metadata).withCodecs(c -> c.withZstd(10)).build() + ); + zstdArray.write(readArrayContent); + ucar.ma2.Array outZstdArray = zstdArray.read(new long[4], readShape); + assert MultiArrayUtils.allValuesEqual(outZstdArray, readArrayContent); + } } - } - - @Test - public void testV3ArrayMetadataBuilder() throws ZarrException { - Array.metadataBuilder() - .withShape(1, 4096, 4096, 1536) - .withDataType(DataType.UINT32) - .withChunkShape(1, 1024, 1024, 1024) - .withFillValue(0) - .withCodecs( - c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) - .build(); - } - - @Test - public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); - } - - @Test - public void testV3Group() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); - - Map attributes = new HashMap<>(); - attributes.put("hello", "world"); - - Group group = Group.create(fsStore.resolve("testgroup")); - Group group2 = group.createGroup("test2", attributes); - Array array = group2.createArray("array", b -> - b.withShape(10, 10) - .withDataType(DataType.UINT8) - .withChunkShape(5, 5) - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); - } - - @Test - public void testV2() throws IOException, ZarrException { - FilesystemStore fsStore = new FilesystemStore(""); - HttpStore httpStore = new HttpStore("https://static.webknossos.org/data"); - - System.out.println(dev.zarr.zarrjava.v2.Array.open(httpStore.resolve("l4_sample", "color", "1"))); - } - - - @Test - public void testReadme1() throws IOException, ZarrException { - Group hierarchy = Group.open( - new HttpStore("https://static.webknossos.org/data/zarr_v3") - .resolve("l4_sample") - ); - Group color = (Group) hierarchy.get("color"); - Array array = (Array) color.get("1"); - ucar.ma2.Array outArray = array.read( - new long[]{0, 3073, 3073, 513}, // offset - new int[]{1, 64, 64, 64} // shape - ); - } - - @Test - public void testReadme2() throws IOException, ZarrException { - Array array = Array.create( - new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), + + @Test + public void testV3ArrayMetadataBuilder() throws ZarrException { Array.metadataBuilder() .withShape(1, 4096, 4096, 1536) .withDataType(DataType.UINT32) .withChunkShape(1, 1024, 1024, 1024) .withFillValue(0) - .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) - .build() - ); - ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); - array.write( - new long[]{0, 0, 0, 0}, // offset - data - ); - ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); - assert MultiArrayUtils.allValuesEqual(data, output); - - } - - @ParameterizedTest - @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) - public void testReadL4Sample(String mag) throws IOException, ZarrException { - StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); - StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); - - Array httpArray = Array.open(httpStoreHandle); - Array localArray = Array.open(localStoreHandle); - System.out.println(httpArray); - System.out.println(localArray); - - ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData1, localData1); - - //offset to where l4_sample contains non-zero values - long[] offset = new long[4]; - long[] originalOffset = new long[]{0, 3073, 3073, 513}; - long[] originalShape = new long[]{1, 4096, 4096, 2048}; - long[] arrayShape = httpArray.metadata.shape; - for (int i = 0; i < 4; i++) { - offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); + .withCodecs( + c -> c.withSharding(new int[]{1, 32, 32, 32}, CodecBuilder::withBlosc)) + .build(); } - ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); - ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); - - assert MultiArrayUtils.allValuesEqual(httpData2, localData2); - } - - @ParameterizedTest - @ValueSource(booleans = {false, true}) - public void testParallel(boolean useParallel) throws IOException, ZarrException { - int[] testData = new int[512 * 512 * 512]; - Arrays.setAll(testData, p -> p); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); - ArrayMetadata metadata = Array.metadataBuilder() - .withShape(512, 512, 512) - .withDataType(DataType.UINT32) - .withChunkShape(100, 100, 100) - .withFillValue(0) - .build(); - Array writeArray = Array.create(storeHandle, metadata); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); - - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(useParallel); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - clearTestoutputFolder(); - } - - @Test - public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { - // non-empty storage transformers are currently not supported - - Map[] storageTransformersEmpty = Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") - ).metadata.storageTransformers; - assert storageTransformersEmpty.length == 0; - - assertThrows(JsonMappingException.class, () -> Array.open( - new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) - ); - - ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() - .withShape(1) - .withChunkShape(1) - .withDataType(DataType.UINT8) - .withStorageTransformers(new HashMap[]{new HashMap() {{ - put("some", "value"); - }}}); - - assertThrows(ZarrException.class, () -> Array.create( - new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), - builderWithStorageTransformer.build() - )); - } - - @ParameterizedTest - @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) - public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException { - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) + @Test + public void testV3FillValue() throws ZarrException { + Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); + Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); + Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); + assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); + } + + @Test + public void testV3Group() throws IOException, ZarrException { + FilesystemStore fsStore = new FilesystemStore(TESTOUTPUT); + + Map attributes = new HashMap<>(); + attributes.put("hello", "world"); + + Group group = Group.create(fsStore.resolve("testgroup")); + Group group2 = group.createGroup("test2", attributes); + Array array = group2.createArray("array", b -> + b.withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + Assertions.assertArrayEquals(((Array) ((Group) group.listAsArray()[0]).listAsArray()[0]).metadata.chunkShape(), new int[]{5, 5}); + } + + @Test + public void testReadme1() throws IOException, ZarrException { + Group hierarchy = Group.open( + new HttpStore("https://static.webknossos.org/data/zarr_v3") + .resolve("l4_sample") + ); + Group color = (Group) hierarchy.get("color"); + Array array = (Array) color.get("1"); + ucar.ma2.Array outArray = array.read( + new long[]{0, 3073, 3073, 513}, // offset + new int[]{1, 64, 64, 64} // shape + ); + } + + @Test + public void testReadme2() throws IOException, ZarrException { + Array array = Array.create( + new FilesystemStore(TESTOUTPUT).resolve("testoutput", "color", "1"), + Array.metadataBuilder() + .withShape(1, 4096, 4096, 1536) + .withDataType(DataType.UINT32) + .withChunkShape(1, 1024, 1024, 1024) + .withFillValue(0) + .withCodecs(c -> c.withSharding(new int[]{1, 32, 32, 32}, c1 -> c1.withBlosc())) + .build() + ); + ucar.ma2.Array data = ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1, 1, 2, 2}, new int[]{1, 2, 3, 4}); + array.write( + new long[]{0, 0, 0, 0}, // offset + data + ); + ucar.ma2.Array output = array.read(new long[]{0, 0, 0, 0}, new int[]{1, 1, 2, 2}); + assert MultiArrayUtils.allValuesEqual(data, output); + + } + + @ParameterizedTest + @ValueSource(strings = {"1", "2-2-1", "4-4-1", "16-16-4"}) + public void testReadL4Sample(String mag) throws IOException, ZarrException { + StoreHandle httpStoreHandle = new HttpStore("https://static.webknossos.org/data/zarr_v3/").resolve("l4_sample", "color", mag); + StoreHandle localStoreHandle = new FilesystemStore(TESTDATA).resolve("l4_sample", "color", mag); + + Array httpArray = Array.open(httpStoreHandle); + Array localArray = Array.open(localStoreHandle); + System.out.println(httpArray); + System.out.println(localArray); + + ucar.ma2.Array httpData1 = httpArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData1 = localArray.read(new long[]{0, 0, 0, 0}, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData1, localData1); + + //offset to where l4_sample contains non-zero values + long[] offset = new long[4]; + long[] originalOffset = new long[]{0, 3073, 3073, 513}; + long[] originalShape = new long[]{1, 4096, 4096, 2048}; + long[] arrayShape = httpArray.metadata.shape; + for (int i = 0; i < 4; i++) { + offset[i] = originalOffset[i] / (originalShape[i] / arrayShape[i]); + } + + ucar.ma2.Array httpData2 = httpArray.read(offset, new int[]{1, 64, 64, 64}); + ucar.ma2.Array localData2 = localArray.read(offset, new int[]{1, 64, 64, 64}); + + assert MultiArrayUtils.allValuesEqual(httpData2, localData2); + } + + @ParameterizedTest + @ValueSource(booleans = {false, true}) + public void testParallel(boolean useParallel) throws IOException, ZarrException { + int[] testData = new int[512 * 512 * 512]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testParallelRead"); + ArrayMetadata metadata = Array.metadataBuilder() + .withShape(512, 512, 512) + .withDataType(DataType.UINT32) + .withChunkShape(100, 100, 100) + .withFillValue(0) + .build(); + Array writeArray = Array.create(storeHandle, metadata); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{512, 512, 512}, testData), useParallel); + + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(useParallel); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + clearTestoutputFolder(); + } + + @Test + public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, IOException { + // non-empty storage transformers are currently not supported + + Map[] storageTransformersEmpty = Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "empty") + ).metadata.storageTransformers; + assert storageTransformersEmpty.length == 0; + + assertThrows(JsonMappingException.class, () -> Array.open( + new FilesystemStore(TESTDATA).resolve("storage_transformer", "exists")) + ); + + ArrayMetadataBuilder builderWithStorageTransformer = Array.metadataBuilder() + .withShape(1) + .withChunkShape(1) .withDataType(DataType.UINT8) - .withChunks(5, 5) - .withFillValue(1) - .withBloscCompressor(cname, shuffle, clevel) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(outArray.getSize(), 8 * 8); - Assertions.assertEquals(outArray.getByte(0), 0); - } -} + .withStorageTransformers(new HashMap[]{new HashMap() {{ + put("some", "value"); + }}}); + + assertThrows(ZarrException.class, () -> Array.create( + new FilesystemStore(TESTOUTPUT).resolve("storage_transformer"), + builderWithStorageTransformer.build() + )); + } + @ParameterizedTest + @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) + public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException { + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withFillValue(1) + .withBloscCompressor(cname, shuffle, clevel) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(outArray.getSize(), 8 * 8); + Assertions.assertEquals(outArray.getByte(0), 0); + } +} From 135ebe8e10abbabb100bbd932919fd7ecc9a5507 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Mon, 15 Sep 2025 23:11:09 +0200 Subject: [PATCH 06/27] fix ci --- .github/workflows/ci.yml | 6 ++++-- src/main/java/dev/zarr/zarrjava/v3/Array.java | 6 ++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 65e935e6..90519cd1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,9 +28,11 @@ jobs: distribution: 'temurin' cache: maven - - name: Set up uv and zarrita + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Set up zarrita run: | - curl -Ls https://github.com/astral-sh/uv/releases/latest/download/uv-installer.sh | bash uv venv && uv init uv add zarrita diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 9e5c06b3..2162c92d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -91,8 +91,7 @@ public static Array create(StoreHandle storeHandle, ArrayMetadata arrayMetadata, * raise an exception if a Zarr array already exists at the specified storage location. * * @param storeHandle the storage location of the Zarr array - * @param arrayMetadataBuilderMapper a callback of ArrayMetadataBuilder -> ArrayMetadataBuilder that is - * used to construct the metadata of the Zarr array + * @param arrayMetadataBuilderMapper a callback that is used to construct the metadata of the Zarr array * @param existsOk if true, no exception is raised if the Zarr array already exists * @throws IOException if the metadata cannot be serialized * @throws ZarrException if the Zarr array cannot be created @@ -180,8 +179,7 @@ public Array setAttributes(Map newAttributes) throws ZarrExcepti * callback may be mutated. This method overwrites and removes any existing attributes. This * method returns a new instance of the Zarr array class and the old instance becomes invalid. * - * @param attributeMapper a callback of Map -> Map that is used to construct the new - * attributes + * @param attributeMapper the callback that is used to construct the new attributes * @throws ZarrException throws ZarrException if the new metadata is invalid * @throws IOException throws IOException if the new metadata cannot be serialized */ From 8fc91873d7420096390087e8debb0c7364ffed40 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 16 Sep 2025 11:41:03 +0200 Subject: [PATCH 07/27] add zlib --- .gitignore | 5 +- .../zarrjava/v2/ArrayMetadataBuilder.java | 11 +++ .../zarrjava/v3/codec/BytesBytesCodec.java | 10 +++ .../zarr/zarrjava/v3/codec/CodecBuilder.java | 22 ++++-- .../zarr/zarrjava/v3/codec/CodecRegistry.java | 10 +-- .../zarrjava/v3/codec/core/GzipCodec.java | 10 +-- .../zarrjava/v3/codec/core/ZlibCodec.java | 74 +++++++++++++++++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 64 +++++++++++++++- 8 files changed, 181 insertions(+), 25 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java diff --git a/.gitignore b/.gitignore index 49917206..2d55ee74 100644 --- a/.gitignore +++ b/.gitignore @@ -38,4 +38,7 @@ build/ ### Custom ### /testdata/l4_sample /testoutput -/venv_zarrita +/.python-version +/main.py +/pyproject.toml +/uv.lock diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index bcbdf46c..a96f55c7 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -101,6 +101,17 @@ public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, i return this; } + public ArrayMetadataBuilder withZlibCompressor(int level) { + try { + this.compressor = new CodecBuilder(dataTypeV2.toV3()) + .withZlib(level) + .build(false)[0]; + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + public ArrayMetadata build() throws ZarrException { if (shape == null) { throw new IllegalStateException("Please call `withShape` first."); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java index 64354632..3304a6c6 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java @@ -2,6 +2,9 @@ import dev.zarr.zarrjava.ZarrException; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; public abstract class BytesBytesCodec extends Codec { @@ -10,4 +13,11 @@ public abstract class BytesBytesCodec extends Codec { public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; + protected void copy(InputStream inputStream, OutputStream outputStream) throws IOException { + byte[] buffer = new byte[4096]; + int len; + while ((len = inputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, len); + } + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 7d9a3b01..edbbe50e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -3,15 +3,10 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.DataType; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v3.codec.core.*; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Endian; -import dev.zarr.zarrjava.v3.codec.core.Crc32cCodec; -import dev.zarr.zarrjava.v3.codec.core.GzipCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import dev.zarr.zarrjava.v3.codec.core.ZstdCodec; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -65,6 +60,19 @@ public CodecBuilder withBlosc() { return withBlosc("zstd"); } + public CodecBuilder withZlib(int level) { + try { + codecs.add(new ZlibCodec(new ZlibCodec.Configuration(level))); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withZlib() { + return withZlib(5); + } + public CodecBuilder withTranspose(int[] order) { codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); return this; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index efed1f21..c30c618a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -1,13 +1,8 @@ package dev.zarr.zarrjava.v3.codec; import com.fasterxml.jackson.databind.jsontype.NamedType; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.Crc32cCodec; -import dev.zarr.zarrjava.v3.codec.core.GzipCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; -import dev.zarr.zarrjava.v3.codec.core.ZstdCodec; +import dev.zarr.zarrjava.v3.codec.core.*; + import java.util.HashMap; import java.util.Map; @@ -20,6 +15,7 @@ public class CodecRegistry { addType("bytes", BytesCodec.class); addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); + addType("zlib", ZlibCodec.class); addType("zstd", ZstdCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index 3ff5acd9..b526ce58 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -9,8 +9,6 @@ import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; import java.nio.ByteBuffer; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; @@ -28,13 +26,7 @@ public GzipCodec( this.configuration = configuration; } - private void copy(InputStream inputStream, OutputStream outputStream) throws IOException { - byte[] buffer = new byte[4096]; - int len; - while ((len = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, len); - } - } + @Override public ByteBuffer decode(ByteBuffer chunkBytes) diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java new file mode 100644 index 00000000..53ae5895 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java @@ -0,0 +1,74 @@ +package dev.zarr.zarrjava.v3.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.zip.*; +import javax.annotation.Nonnull; + +public class ZlibCodec extends BytesBytesCodec { + + public final String name = "zlib"; + @Nonnull + public final Configuration configuration; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public ZlibCodec( + @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { + this.configuration = configuration; + } + + + @Override + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InflaterInputStream inputStream = new InflaterInputStream( + new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { + copy(inputStream, outputStream); + inputStream.close(); + return ByteBuffer.wrap(outputStream.toByteArray()); + } catch (IOException ex) { + throw new ZarrException("Error in decoding gzip.", ex); + } + } + + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); + DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.configuration.level))) { + dos.write(Utils.toArray(chunkBytes)); + dos.close(); + return ByteBuffer.wrap(outputStream.toByteArray()); + } catch (IOException ex) { + throw new ZarrException("Error in encoding zlib.", ex); + } + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Zlib codec."); + } + + + public static final class Configuration { + + public final int level; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public Configuration(@JsonProperty(value = "level", defaultValue = "1") int level) + throws ZarrException { + if (level < 0 || level > 9) { + throw new ZarrException("'level' needs to be between 0 and 9."); + } + this.level = level; + } + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 1c167580..d00cdbb5 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -662,7 +662,7 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I @ParameterizedTest @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) - public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException { + public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel), dev.zarr.zarrjava.v2.Array.metadataBuilder() @@ -679,4 +679,66 @@ public void testV2create(String cname, String shuffle, int clevel) throws IOExce Assertions.assertEquals(outArray.getSize(), 8 * 8); Assertions.assertEquals(outArray.getByte(0), 0); } + + @Test + public void testV2create() throws IOException, ZarrException { + DataType dataType = DataType.UINT32; + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dataType) + .withChunks(5, 5) + .withFillValue(2) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(outArray.getSize(), 8 * 8); + Assertions.assertEquals(outArray.getByte(0), 0); + } + + @Test + public void testV2Filters() throws IOException, ZarrException { + DataType dataType = DataType.UINT32; + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(10, 10) + .withDataType(dataType) + .withChunks(5, 5) + .withFillValue(2) + .withFilters(f -> f.withTranspose(new int[]{1, 0})) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); + Assertions.assertEquals(outArray.getSize(), 8 * 8); + Assertions.assertEquals(outArray.getByte(0), 0); + } + + + @ParameterizedTest + @ValueSource(ints = {0, 1, 5, 9}) + public void testV2createZlib(int level) throws IOException, ZarrException { + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(15, 10) + .withDataType(DataType.UINT8) + .withChunks(4, 5) + .withFillValue(5) + .withZlibCompressor(level) + .build() + ); + array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); + + ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); + Assertions.assertEquals(outArray.getSize(), 7 * 6); + Assertions.assertEquals(outArray.getByte(0), 0); + } } From f3a80976807d019f66c9c0c74f48386d03b6e978 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 16 Sep 2025 12:01:17 +0200 Subject: [PATCH 08/27] fix testoutput paths --- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index d00cdbb5..721a1aac 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -664,7 +664,7 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I @CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"}) public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException { dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel), + new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(10, 10) .withDataType(DataType.UINT8) @@ -705,7 +705,7 @@ public void testV2Filters() throws IOException, ZarrException { DataType dataType = DataType.UINT32; dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create"), + new FilesystemStore(TESTOUTPUT).resolve("v2_create_filters"), dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(10, 10) .withDataType(dataType) From a8c6347c1356e9d8104dfd4a1dff341fbc78397c Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 17 Sep 2025 14:52:45 +0200 Subject: [PATCH 09/27] split v2 and v3 codecs to fix serialization --- .../zarr/zarrjava/codec/ArrayArrayCodec.java | 14 ++ .../{v3 => }/codec/ArrayBytesCodec.java | 12 +- .../zarr/zarrjava/codec/BytesBytesCodec.java | 23 ++++ .../java/dev/zarr/zarrjava/codec/Codec.java | 14 ++ .../dev/zarr/zarrjava/codec/CodecBuilder.java | 14 ++ .../{v3 => }/codec/CodecPipeline.java | 2 +- .../dev/zarr/zarrjava/interfaces/Array.java | 2 +- src/main/java/dev/zarr/zarrjava/v2/Array.java | 21 ++- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 9 +- .../zarrjava/v2/ArrayMetadataBuilder.java | 12 +- .../dev/zarr/zarrjava/v2/codec/Codec.java | 26 ++++ .../zarr/zarrjava/v2/codec/CodecBuilder.java | 75 ++++++++++ .../zarr/zarrjava/v2/codec/CodecRegistry.java | 29 ++++ .../zarrjava/v2/codec/core/BloscCodec.java | 130 ++++++++++++++++++ src/main/java/dev/zarr/zarrjava/v3/Array.java | 2 +- .../zarrjava/v3/ArrayMetadataBuilder.java | 2 - .../zarrjava/v3/codec/ArrayArrayCodec.java | 14 -- .../zarrjava/v3/codec/BytesBytesCodec.java | 23 ---- .../dev/zarr/zarrjava/v3/codec/Codec.java | 7 +- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 32 +---- .../zarr/zarrjava/v3/codec/CodecRegistry.java | 2 +- .../zarrjava/v3/codec/core/BloscCodec.java | 7 +- .../zarrjava/v3/codec/core/BytesCodec.java | 5 +- .../zarrjava/v3/codec/core/Crc32cCodec.java | 5 +- .../zarrjava/v3/codec/core/GzipCodec.java | 5 +- .../v3/codec/core/ShardingIndexedCodec.java | 6 +- .../v3/codec/core/TransposeCodec.java | 7 +- 27 files changed, 389 insertions(+), 111 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java rename src/main/java/dev/zarr/zarrjava/{v3 => }/codec/ArrayBytesCodec.java (57%) create mode 100644 src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java create mode 100644 src/main/java/dev/zarr/zarrjava/codec/Codec.java create mode 100644 src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java rename src/main/java/dev/zarr/zarrjava/{v3 => }/codec/CodecPipeline.java (99%) create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java delete mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java delete mode 100644 src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java diff --git a/src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java new file mode 100644 index 00000000..2ecba670 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java @@ -0,0 +1,14 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import ucar.ma2.Array; + +public interface ArrayArrayCodec { + + Array encode(Array chunkArray) + throws ZarrException; + + Array decode(Array chunkArray) + throws ZarrException; + +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java similarity index 57% rename from src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java rename to src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java index 361ae61f..e080786e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java @@ -1,24 +1,24 @@ -package dev.zarr.zarrjava.v3.codec; +package dev.zarr.zarrjava.codec; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import java.nio.ByteBuffer; import ucar.ma2.Array; -public abstract class ArrayBytesCodec extends Codec { +public interface ArrayBytesCodec { - protected abstract ByteBuffer encode(Array chunkArray) + ByteBuffer encode(Array chunkArray) throws ZarrException; - protected abstract Array decode(ByteBuffer chunkBytes) + Array decode(ByteBuffer chunkBytes) throws ZarrException; - public abstract static class WithPartialDecode extends ArrayBytesCodec { + interface WithPartialDecode extends ArrayBytesCodec { public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; public abstract ByteBuffer encode(Array shardArray) throws ZarrException; - protected abstract Array decodePartial( + Array decodePartial( StoreHandle handle, long[] offset, int[] shape ) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java new file mode 100644 index 00000000..b48d4adf --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java @@ -0,0 +1,23 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.ByteBuffer; + +public interface BytesBytesCodec { + + ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; + + ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; + + default void copy(InputStream inputStream, OutputStream outputStream) throws IOException { + byte[] buffer = new byte[4096]; + int len; + while ((len = inputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, len); + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/codec/Codec.java new file mode 100644 index 00000000..441ed603 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/codec/Codec.java @@ -0,0 +1,14 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.ArrayMetadata; + +public interface Codec { + ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException; + + long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) + throws ZarrException; + + void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; +} + diff --git a/src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java new file mode 100644 index 00000000..0b5e9d64 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java @@ -0,0 +1,14 @@ +package dev.zarr.zarrjava.codec; + +import dev.zarr.zarrjava.v3.DataType; + +public abstract class CodecBuilder { + + final protected DataType dataType; + + public CodecBuilder(DataType dataType) { + this.dataType = dataType; + } + + public abstract Codec[] build(); +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java similarity index 99% rename from src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java rename to src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java index 920a1f4f..781c8059 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.v3.codec; +package dev.zarr.zarrjava.codec; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java index 4dcbdea1..6bf25318 100644 --- a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java +++ b/src/main/java/dev/zarr/zarrjava/interfaces/Array.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.codec.CodecPipeline; +import dev.zarr.zarrjava.codec.CodecPipeline; import ucar.ma2.InvalidRangeException; import javax.annotation.Nonnull; diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 8cf4d77f..df1a9f2c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -1,11 +1,13 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.datatype.jdk8.Jdk8Module; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.codec.CodecPipeline; -import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.codec.CodecPipeline; +import dev.zarr.zarrjava.v2.codec.Codec; +import dev.zarr.zarrjava.v2.codec.CodecRegistry; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import javax.annotation.Nonnull; @@ -15,8 +17,6 @@ import java.util.function.Function; import java.util.stream.Collectors; -import static dev.zarr.zarrjava.v3.Node.makeObjectMapper; - public class Array implements dev.zarr.zarrjava.interfaces.Array { static final String ZARRAY = ".zarray"; @@ -28,8 +28,9 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE this.storeHandle = storeHandle; this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( - metadata.filters, - new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, + new dev.zarr.zarrjava.codec.Codec[]{}, + metadata.filters == null ? new Codec[]{} : metadata.filters, + new dev.zarr.zarrjava.codec.Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} ), metadata.coreArrayMetadata); } @@ -52,6 +53,14 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept ); } + public static ObjectMapper makeObjectMapper() { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.registerModule(new Jdk8Module()); + objectMapper.registerSubtypes(CodecRegistry.getNamedTypes()); + return objectMapper; + } + + /** * Creates a new Zarr array with the provided metadata at a specified storage location. This * method will raise an exception if a Zarr array already exists at the specified storage diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 0cc8a004..96bf400c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -9,7 +9,7 @@ import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; -import dev.zarr.zarrjava.v3.codec.Codec; +import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; import javax.annotation.Nullable; @@ -58,18 +58,17 @@ public ArrayMetadata( @JsonProperty(value = "shape", required = true) long[] shape, @JsonProperty(value = "chunks", required = true) int[] chunks, @JsonProperty(value = "dtype", required = true) DataTypeV2 dataTypeV2, - @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo can be "null" + @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo test when null @JsonProperty(value = "order", required = true) Order order, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator, - @Nullable @JsonProperty(value = "filters") Codec[] filters, //todo can be "null" - @Nullable @JsonProperty(value = "compressor") Codec compressor //todo can be "null" + @Nullable @JsonProperty(value = "filters") Codec[] filters, + @Nullable @JsonProperty(value = "compressor") Codec compressor ) throws ZarrException { super(); if (zarrFormat != this.zarrFormat) { throw new ZarrException( "Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'."); } - this.shape = shape; this.chunks = chunks; this.dataTypeV2 = dataTypeV2; diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index a96f55c7..c625348c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -3,8 +3,8 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; -import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import dev.zarr.zarrjava.v2.codec.Codec; +import dev.zarr.zarrjava.v2.codec.CodecBuilder; import java.util.function.Function; @@ -15,7 +15,7 @@ public class ArrayMetadataBuilder { Order order = Order.C; Separator dimensionSeparator = Separator.DOT; Object fillValue = 0; - Codec[] filters = new Codec[]{}; + Codec[] filters = null; Codec compressor = null; @@ -81,7 +81,7 @@ public ArrayMetadataBuilder withFilters(Function cod } CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3()); this.filters = codecBuilder.apply(nestedCodecBuilder) - .build(false); + .build(); return this; } @@ -94,7 +94,7 @@ public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, i try { this.compressor = new CodecBuilder(dataTypeV2.toV3()) .withBlosc(cname, shuffle, clevel) - .build(false)[0]; + .build()[0]; } catch (ZarrException e) { throw new RuntimeException(e); } @@ -105,7 +105,7 @@ public ArrayMetadataBuilder withZlibCompressor(int level) { try { this.compressor = new CodecBuilder(dataTypeV2.toV3()) .withZlib(level) - .build(false)[0]; + .build()[0]; } catch (ZarrException e) { throw new RuntimeException(e); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java new file mode 100644 index 00000000..94ad5828 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -0,0 +1,26 @@ +package dev.zarr.zarrjava.v2.codec; + +import com.fasterxml.jackson.annotation.JsonTypeInfo; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.ArrayMetadata; + +@JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") +public abstract class Codec implements dev.zarr.zarrjava.codec.Codec { + + protected ArrayMetadata.CoreArrayMetadata arrayMetadata; + + public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + if (arrayMetadata == null) { + throw new ZarrException("arrayMetadata needs to get set in for every codec"); + } + return this.arrayMetadata; + } + + public abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) + throws ZarrException; + + public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + this.arrayMetadata = arrayMetadata; + } +} + diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java new file mode 100644 index 00000000..cc9203b7 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java @@ -0,0 +1,75 @@ +package dev.zarr.zarrjava.v2.codec; + +import com.scalableminds.bloscjava.Blosc; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v2.codec.core.*; + +import java.util.ArrayList; +import java.util.List; + +public class CodecBuilder extends dev.zarr.zarrjava.codec.CodecBuilder { + + protected List codecs; + + public CodecBuilder(DataType dataType) { + super(dataType); + this.codecs = new ArrayList<>(); + } + + public CodecBuilder withBlosc( + Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, + int blockSize + ) { + try { + codecs.add(new BloscCodec(cname, shuffle, clevel, typeSize, blockSize)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withBlosc(String cname, String shuffle, int clevel, int blockSize) { + if (shuffle.equals("shuffle")){ + shuffle = "byteshuffle"; + } + return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, + dataType.getByteCount(), blockSize + ); + } + + public CodecBuilder withBlosc(String cname, String shuffle, int clevel) { + return withBlosc(cname, shuffle, clevel, 0); + } + + public CodecBuilder withBlosc(String cname, int clevel) { + return withBlosc(cname, "noshuffle", clevel); + } + + public CodecBuilder withBlosc(String cname) { + return withBlosc(cname, 5); + } + + public CodecBuilder withBlosc() { + return withBlosc("zstd"); + } + + public CodecBuilder withZlib(int level) { + try { + codecs.add(new ZlibCodec(level)); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public CodecBuilder withZlib() { + return withZlib(5); + } + + public Codec[] build() { + return codecs.toArray(new Codec[0]); + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java new file mode 100644 index 00000000..2b0f7a6c --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecRegistry.java @@ -0,0 +1,29 @@ +package dev.zarr.zarrjava.v2.codec; + +import com.fasterxml.jackson.databind.jsontype.NamedType; +import dev.zarr.zarrjava.v2.codec.core.*; + +import java.util.HashMap; +import java.util.Map; + +public class CodecRegistry { + + static Map> map = new HashMap<>(); + + static { + addType("blosc", BloscCodec.class); + addType("zlib", ZlibCodec.class); + } + + public static void addType(String name, Class codecClass) { + map.put(name, codecClass); + } + + public static NamedType[] getNamedTypes() { + return map.entrySet() + .stream() + .map(entry -> new NamedType(entry.getValue(), entry.getKey())) + .toArray( + NamedType[]::new); + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java new file mode 100644 index 00000000..5f97b5d0 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -0,0 +1,130 @@ +package dev.zarr.zarrjava.v2.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.scalableminds.bloscjava.Blosc; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.codec.BytesBytesCodec; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v2.codec.Codec; +import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v3.codec.core.BloscCodec.CustomCompressorDeserializer; +import dev.zarr.zarrjava.v3.codec.core.BloscCodec.CustomCompressorSerializer; +import javax.annotation.Nonnull; +import java.io.IOException; +import java.nio.ByteBuffer; + +public class BloscCodec extends Codec implements BytesBytesCodec { + + public final String id = "blosc"; + + @Nonnull + @JsonSerialize(using = CustomCompressorSerializer.class) + public final Blosc.Compressor cname; + @Nonnull + @JsonSerialize(using = CustomShuffleSerializer.class) + public final Blosc.Shuffle shuffle; + public final int clevel; + public final int typesize; + public final int blocksize; + + @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) + public BloscCodec( + @Nonnull @JsonProperty(value = "cname", defaultValue = "zstd") + @JsonDeserialize(using = CustomCompressorDeserializer.class) + Blosc.Compressor cname, + @Nonnull @JsonProperty(value = "shuffle", defaultValue = "noshuffle") + @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, + @JsonProperty(value = "clevel", defaultValue = "5") int clevel, + @JsonProperty(value = "typesize", defaultValue = "0") int typesize, + @JsonProperty(value = "blocksize", defaultValue = "0") + int blocksize + ) throws ZarrException { + if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { + typesize = 4; // in v2 typesize is not required. todo: deflault to correct value based on dtype + } + if (clevel < 0 || clevel > 9) { + throw new ZarrException("'clevel' needs to be between 0 and 9."); + } + this.cname = cname; + this.shuffle = shuffle; + this.clevel = clevel; + this.typesize = typesize; + this.blocksize = blocksize; + } + + @Override + public ByteBuffer decode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); + } catch (Exception ex) { + throw new ZarrException("Error in decoding blosc.", ex); + } + } + + @Override + public ByteBuffer encode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap( + Blosc.compress(Utils.toArray(chunkBytes), this.typesize, this.cname, + this.clevel, + this.shuffle, this.blocksize + )); + } catch (Exception ex) { + throw new ZarrException("Error in encoding blosc.", ex); + } + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + throw new ZarrException("Not implemented for Blosc codec."); + } + + public static final class CustomShuffleSerializer extends StdSerializer { + + public CustomShuffleSerializer() { + super(Blosc.Shuffle.class); + } + + public CustomShuffleSerializer(Class t) { + super(t); + } + + @Override + public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, + SerializerProvider provider) + throws IOException { + generator.writeNumber(shuffle.ordinal()); + } + } + + public static final class CustomShuffleDeserializer extends StdDeserializer { + + public CustomShuffleDeserializer() { + this(null); + } + + public CustomShuffleDeserializer(Class vc) { + super(vc); + } + + @Override + public Blosc.Shuffle deserialize(JsonParser jsonParser, DeserializationContext ctxt) + throws IOException { + int shuffle = jsonParser.getCodec() + .readValue(jsonParser, int.class); + return Blosc.Shuffle.values()[shuffle]; + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 2162c92d..3488fb58 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -4,7 +4,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.codec.CodecPipeline; +import dev.zarr.zarrjava.codec.CodecPipeline; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index c16a9e54..a844802e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -11,9 +11,7 @@ import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Endian; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; -import java.util.Arrays; import java.util.HashMap; import java.util.Map; import java.util.function.Function; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java deleted file mode 100644 index a488d306..00000000 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/ArrayArrayCodec.java +++ /dev/null @@ -1,14 +0,0 @@ -package dev.zarr.zarrjava.v3.codec; - -import dev.zarr.zarrjava.ZarrException; -import ucar.ma2.Array; - -public abstract class ArrayArrayCodec extends Codec { - - protected abstract Array encode(Array chunkArray) - throws ZarrException; - - protected abstract Array decode(Array chunkArray) - throws ZarrException; - -} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java deleted file mode 100644 index 3304a6c6..00000000 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java +++ /dev/null @@ -1,23 +0,0 @@ -package dev.zarr.zarrjava.v3.codec; - -import dev.zarr.zarrjava.ZarrException; - -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.ByteBuffer; - -public abstract class BytesBytesCodec extends Codec { - - protected abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; - - public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; - - protected void copy(InputStream inputStream, OutputStream outputStream) throws IOException { - byte[] buffer = new byte[4096]; - int len; - while ((len = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, len); - } - } -} diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index 988dd1dc..c239d4ee 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,20 +5,17 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public abstract class Codec { +public abstract class Codec implements dev.zarr.zarrjava.codec.Codec { protected ArrayMetadata.CoreArrayMetadata arrayMetadata; - protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { if (arrayMetadata == null) { throw new ZarrException("arrayMetadata needs to get set in for every codec"); } return this.arrayMetadata; } - protected abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException; - public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ this.arrayMetadata = arrayMetadata; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index edbbe50e..a25071b3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -2,6 +2,9 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.core.*; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration; @@ -12,13 +15,11 @@ import java.util.List; import java.util.function.Function; -public class CodecBuilder { - - final private DataType dataType; - private List codecs; +public class CodecBuilder extends dev.zarr.zarrjava.codec.CodecBuilder { + protected List codecs; public CodecBuilder(DataType dataType) { - this.dataType = dataType; + super(dataType); this.codecs = new ArrayList<>(); } @@ -60,19 +61,6 @@ public CodecBuilder withBlosc() { return withBlosc("zstd"); } - public CodecBuilder withZlib(int level) { - try { - codecs.add(new ZlibCodec(new ZlibCodec.Configuration(level))); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withZlib() { - return withZlib(5); - } - public CodecBuilder withTranspose(int[] order) { codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order))); return this; @@ -168,13 +156,7 @@ private void autoInsertBytesCodec() { } public Codec[] build() { - return build(true); - } - - public Codec[] build(boolean includeBytesCodec) { - if (includeBytesCodec) { - autoInsertBytesCodec(); - } + autoInsertBytesCodec(); return codecs.toArray(new Codec[0]); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index c30c618a..770dfced 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -2,6 +2,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import dev.zarr.zarrjava.v3.codec.core.*; +import dev.zarr.zarrjava.codec.Codec; import java.util.HashMap; import java.util.Map; @@ -15,7 +16,6 @@ public class CodecRegistry { addType("bytes", BytesCodec.class); addType("blosc", BloscCodec.class); addType("gzip", GzipCodec.class); - addType("zlib", ZlibCodec.class); addType("zstd", ZstdCodec.class); addType("crc32c", Crc32cCodec.class); addType("sharding_indexed", ShardingIndexedCodec.class); diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index 819a610f..7601622a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -13,14 +13,15 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import java.io.IOException; import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class BloscCodec extends BytesBytesCodec { +public class BloscCodec extends Codec implements BytesBytesCodec { public final String name = "blosc"; @Nonnull @@ -192,7 +193,7 @@ public Configuration( @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize ) throws ZarrException { - if (typesize < 1) { + if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { throw new ZarrException("'typesize' needs to be larger than 0."); } if (clevel < 0 || clevel > 9) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index 1415da92..1f4d824f 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -4,14 +4,15 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.codec.ArrayBytesCodec; import java.nio.ByteBuffer; import java.nio.ByteOrder; import javax.annotation.Nonnull; import ucar.ma2.Array; -public class BytesCodec extends ArrayBytesCodec { +public class BytesCodec extends Codec implements ArrayBytesCodec { public final String name = "bytes"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index a1e3cb52..00e829f2 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -2,14 +2,15 @@ import com.fasterxml.jackson.annotation.JsonCreator; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.CRC32C; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class Crc32cCodec extends BytesBytesCodec { +public class Crc32cCodec extends Codec implements BytesBytesCodec { public final String name = "crc32c"; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index b526ce58..589b274b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -3,9 +3,10 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; @@ -14,7 +15,7 @@ import java.util.zip.GZIPOutputStream; import javax.annotation.Nonnull; -public class GzipCodec extends BytesBytesCodec { +public class GzipCodec extends Codec implements BytesBytesCodec { public final String name = "gzip"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 0dce7348..e6341468 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -10,9 +10,9 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; import dev.zarr.zarrjava.v3.DataType; -import dev.zarr.zarrjava.v3.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.codec.ArrayBytesCodec; import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.v3.codec.CodecPipeline; +import dev.zarr.zarrjava.codec.CodecPipeline; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; @@ -22,7 +22,7 @@ import ucar.ma2.InvalidRangeException; -public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode { +public class ShardingIndexedCodec extends Codec implements ArrayBytesCodec.WithPartialDecode { public final String name = "sharding_indexed"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index 4d614ae9..bc24ea06 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -3,8 +3,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.codec.ArrayArrayCodec; import ucar.ma2.Array; import javax.annotation.Nonnull; @@ -13,7 +14,7 @@ import static dev.zarr.zarrjava.utils.Utils.inversePermutation; import static dev.zarr.zarrjava.utils.Utils.isPermutation; -public class TransposeCodec extends ArrayArrayCodec { +public class TransposeCodec extends Codec implements ArrayArrayCodec { @Nonnull public final String name = "transpose"; @@ -70,7 +71,7 @@ public Configuration(@JsonProperty(value = "order") int[] order) { } @Override - protected ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { super.resolveArrayMetadata(); assert arrayMetadata.ndim() == configuration.order.length; From b93de235671405ed84668991c56957dc9c1a40e3 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 17 Sep 2025 14:54:21 +0200 Subject: [PATCH 10/27] zarr-python tests for v2, v3 --- .github/workflows/ci.yml | 4 +- .../dev/zarr/zarrjava/store/StoreHandle.java | 8 + .../{v3 => v2}/codec/core/ZlibCodec.java | 38 +-- .../zarrjava/v3/codec/core/ZstdCodec.java | 5 +- .../dev/zarr/zarrjava/ZarrPythonTests.java | 247 ++++++++++++++++++ src/test/java/dev/zarr/zarrjava/ZarrTest.java | 162 ------------ src/test/python-scripts/parse_codecs.py | 83 ++++++ src/test/python-scripts/zarr_python_read.py | 33 +++ .../python-scripts/zarr_python_read_v2.py | 35 +++ src/test/python-scripts/zarr_python_write.py | 27 ++ .../python-scripts/zarr_python_write_v2.py | 29 ++ src/test/python-scripts/zarrita_read.py | 28 +- src/test/python-scripts/zarrita_write.py | 34 +-- 13 files changed, 487 insertions(+), 246 deletions(-) rename src/main/java/dev/zarr/zarrjava/{v3 => v2}/codec/core/ZlibCodec.java (66%) create mode 100644 src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java create mode 100644 src/test/python-scripts/parse_codecs.py create mode 100644 src/test/python-scripts/zarr_python_read.py create mode 100644 src/test/python-scripts/zarr_python_read_v2.py create mode 100644 src/test/python-scripts/zarr_python_write.py create mode 100644 src/test/python-scripts/zarr_python_write_v2.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 90519cd1..5314ef9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,10 +31,10 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v6 - - name: Set up zarrita + - name: Set up zarr-python run: | uv venv && uv init - uv add zarrita + uv add zarr - name: Download testdata run: | diff --git a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java index 8ffc5f46..b82424f7 100644 --- a/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java +++ b/src/main/java/dev/zarr/zarrjava/store/StoreHandle.java @@ -3,6 +3,7 @@ import dev.zarr.zarrjava.utils.Utils; import java.nio.ByteBuffer; import java.nio.file.NoSuchFileException; +import java.nio.file.Path; import java.util.stream.Stream; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -70,4 +71,11 @@ public String toString() { public StoreHandle resolve(String... subKeys) { return new StoreHandle(store, Utils.concatArrays(keys, subKeys)); } + + public Path toPath() { + if (!(store instanceof FilesystemStore)) { + throw new UnsupportedOperationException("The underlying store is not a filesystem store."); + } + return ((FilesystemStore) store).resolveKeys(keys); + } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java similarity index 66% rename from src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java rename to src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 53ae5895..517732ef 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -1,29 +1,32 @@ -package dev.zarr.zarrjava.v3.codec.core; +package dev.zarr.zarrjava.v2.codec.core; import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.nio.ByteBuffer; import java.util.zip.*; -import javax.annotation.Nonnull; -public class ZlibCodec extends BytesBytesCodec { +public class ZlibCodec extends Codec implements BytesBytesCodec { + + public final String id = "zlib"; + public final int level; - public final String name = "zlib"; - @Nonnull - public final Configuration configuration; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) public ZlibCodec( - @Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) { - this.configuration = configuration; + @JsonProperty(value = "level", defaultValue = "1") int level) throws ZarrException { + if (level < 0 || level > 9) { + throw new ZarrException("'level' needs to be between 0 and 9."); + } + this.level = level; } @@ -42,7 +45,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); - DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.configuration.level))) { + DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.level))) { dos.write(Utils.toArray(chunkBytes)); dos.close(); return ByteBuffer.wrap(outputStream.toByteArray()); @@ -56,19 +59,4 @@ public long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { throw new ZarrException("Not implemented for Zlib codec."); } - - - public static final class Configuration { - - public final int level; - - @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) - public Configuration(@JsonProperty(value = "level", defaultValue = "1") int level) - throws ZarrException { - if (level < 0 || level > 9) { - throw new ZarrException("'level' needs to be between 0 and 9."); - } - this.level = level; - } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index f042f11c..208d74b3 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -5,13 +5,14 @@ import com.github.luben.zstd.Zstd; import com.github.luben.zstd.ZstdCompressCtx; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.BytesBytesCodec; +import dev.zarr.zarrjava.codec.BytesBytesCodec; import javax.annotation.Nonnull; import java.nio.ByteBuffer; -public class ZstdCodec extends BytesBytesCodec { +public class ZstdCodec extends Codec implements BytesBytesCodec { public final String name = "zstd"; @Nonnull diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java new file mode 100644 index 00000000..054d250c --- /dev/null +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -0,0 +1,247 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.ArrayMetadataBuilder; +import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v3.codec.CodecBuilder; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.CsvSource; + +import java.io.BufferedReader; +import java.io.File; +import java.io.IOException; +import java.io.InputStreamReader; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.Comparator; +import java.util.HashMap; +import java.util.Map; +import java.util.stream.Stream; + +public class ZarrPythonTests { + + final static Path TESTOUTPUT = Paths.get("testoutput"); + final static Path PYTHON_TEST_PATH = Paths.get("src/test/python-scripts/"); + + @BeforeAll + public static void clearTestoutputFolder() throws IOException { + if (Files.exists(TESTOUTPUT)) { + try (Stream walk = Files.walk(TESTOUTPUT)) { + walk.sorted(Comparator.reverseOrder()).map(Path::toFile).forEach(File::delete); + } + } + Files.createDirectory(TESTOUTPUT); + } + + public void run_python_script(String scriptName, String... args) throws IOException, InterruptedException { + ProcessBuilder pb = new ProcessBuilder(); + pb.command().add("uv"); + pb.command().add("run"); + pb.command().add(PYTHON_TEST_PATH.resolve(scriptName).toString()); + pb.command().addAll(Arrays.asList(args)); + Process process = pb.start(); + + BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + System.out.println(line); + } + + BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); + while ((line = readerErr.readLine()) != null) { + System.err.println(line); + } + + int exitCode = process.waitFor(); + assert exitCode == 0; + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + public void testReadFromZarrPythonV3(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python", codec, codecParam); + run_python_script("zarr_python_write.py", codec, codecParam, storeHandle.toPath().toString()); + Array array = Array.open(storeHandle); + ucar.ma2.Array result = array.read(); + + //for expected values see zarr_python_write.py + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); + Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + + int[] expectedData = new int[16 * 16 * 16]; + Arrays.setAll(expectedData, p -> p); + Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + + @ParameterizedTest + @CsvSource({ + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + "gzip,0", "gzip,5", + "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", + "bytes,BIG", "bytes,LITTLE", + "transpose,_", + "sharding,start", "sharding,end", + "sharding_nested,_", + "crc32c,_", + }) + public void testWriteReadWithZarrPythonV3(String codec, String codecParam) throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + + Map attributes = new HashMap<>(); + attributes.put("test_key", "test_value"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python", codec, codecParam); + + ArrayMetadataBuilder builder = Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunkShape(2, 4, 8) + .withFillValue(0) + .withAttributes(attributes); + + switch (codec) { + case "blosc": + String cname = codecParam.split("_")[0]; + String shuffle = codecParam.split("_")[1]; + int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); + builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); + break; + case "gzip": + builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); + break; + case "zstd": + int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); + boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); + builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); + break; + case "bytes": + builder = builder.withCodecs(c -> c.withBytes(codecParam)); + break; + case "transpose": + builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); + break; + case "sharding": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); + break; + case "sharding_nested": + builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); + break; + case "crc32c": + builder = builder.withCodecs(CodecBuilder::withCrc32c); + break; + default: + throw new IllegalArgumentException("Invalid Codec: " + codec); + } + + Array writeArray = Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + Array readArray = Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); + Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + + //read in zarr_python + run_python_script("zarr_python_read.py", codec, codecParam, storeHandle.toPath().toString()); + } + + + @ParameterizedTest + @CsvSource({ + "zlib,0", "zlib,5", + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + }) + public void testReadFromZarrPythonV2(String compressor, String compressorParam) throws IOException, ZarrException, InterruptedException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("read_from_zarr_python_v2", compressor, compressorParam); + run_python_script("zarr_python_write_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle); + ucar.ma2.Array result = array.read(); + + //for expected values see zarr_python_write.py + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); +// Assertions.assertEquals(42, array.metadata.attributes.get("answer")); + + int[] expectedData = new int[16 * 16 * 16]; + Arrays.setAll(expectedData, p -> p); + Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); + } + + + @ParameterizedTest + @CsvSource({ + "zlib,0", "zlib,5", + "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", + }) + public void testWriteReadWithZarrPythonV2(String compressor, String compressorParam) throws Exception { + int[] testData = new int[16 * 16 * 16]; + Arrays.setAll(testData, p -> p); + +// Map attributes = new HashMap<>(); +// attributes.put("test_key", "test_value"); + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarr_python_v2", compressor, compressorParam); + + dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(16, 16, 16) + .withDataType(DataType.UINT32) + .withChunks(2, 4, 8) +// .withAttributes(attributes) + .withFillValue(0); + + switch (compressor) { + case "blosc": + String cname = compressorParam.split("_")[0]; + String shuffle = compressorParam.split("_")[1]; + int clevel_blosc = Integer.parseInt(compressorParam.split("_")[2]); + builder = builder.withBloscCompressor(cname, shuffle, clevel_blosc); + break; + case "zlib": + builder = builder.withZlibCompressor(Integer.parseInt(compressorParam)); + break; + default: + throw new IllegalArgumentException("Invalid compressor: " + compressor); + } + + dev.zarr.zarrjava.v2.Array writeArray = dev.zarr.zarrjava.v2.Array.create(storeHandle, builder.build()); + writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); + + //read in zarr-java + dev.zarr.zarrjava.v2.Array readArray = dev.zarr.zarrjava.v2.Array.open(storeHandle); + ucar.ma2.Array result = readArray.read(); + + Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); + Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); +// Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); + + Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); + + //read in zarr_python + run_python_script("zarr_python_read_v2.py", compressor, compressorParam, storeHandle.toPath().toString()); + } +} diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 721a1aac..03573626 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -10,10 +10,8 @@ import dev.zarr.zarrjava.store.*; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.v3.*; -import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; import dev.zarr.zarrjava.v3.codec.core.TransposeCodec; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeAll; @@ -54,51 +52,6 @@ public static void clearTestoutputFolder() throws IOException { Files.createDirectory(TESTOUTPUT); } - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - - public void testReadFromZarrita(String codec, String codecParam) throws IOException, ZarrException, InterruptedException { - ProcessBuilder pb = new ProcessBuilder("uv", "run", PYTHON_TEST_PATH.resolve("zarrita_write.py") - .toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); - } - - int exitCode = process.waitFor(); - assert exitCode == 0; - - Array array = Array.open(new FilesystemStore(TESTOUTPUT).resolve("read_from_zarrita", codec, codecParam)); - ucar.ma2.Array result = array.read(); - - //for expected values see zarrita_write.py - Assertions.assertArrayEquals(new int[]{16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 8}, array.metadata.chunkShape()); - Assertions.assertEquals(42, array.metadata.attributes.get("answer")); - - int[] expectedData = new int[16 * 16]; - Arrays.setAll(expectedData, p -> p); - Assertions.assertArrayEquals(expectedData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.INT)); - } - @CsvSource({"0,true", "0,false", "5, true", "10, false"}) @ParameterizedTest public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException, InterruptedException { @@ -134,99 +87,6 @@ public void testZstdLibrary(int clevel, boolean checksumFlag) throws IOException assert exitCode == 0; } - @ParameterizedTest - @CsvSource({ - "blosc,blosclz_noshuffle_0", "blosc,lz4_shuffle_6", "blosc,lz4hc_bitshuffle_3", "blosc,zlib_shuffle_5", "blosc,zstd_bitshuffle_9", - "gzip,0", "gzip,5", - "zstd,0_true", "zstd,5_true", "zstd,0_false", "zstd,5_false", - "bytes,BIG", "bytes,LITTLE", - "transpose,_", - "sharding,start", "sharding,end", - "sharding_nested,_", - "crc32c,_", - }) - public void testWriteReadWithZarrita(String codec, String codecParam) throws Exception { - int[] testData = new int[16 * 16 * 16]; - Arrays.setAll(testData, p -> p); - - Map attributes = new HashMap<>(); - attributes.put("test_key", "test_value"); - - StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("write_to_zarrita", codec, codecParam); - ArrayMetadataBuilder builder = Array.metadataBuilder() - .withShape(16, 16, 16) - .withDataType(DataType.UINT32) - .withChunkShape(2, 4, 8) - .withFillValue(0) - .withAttributes(attributes); - - switch (codec) { - case "blosc": - String cname = codecParam.split("_")[0]; - String shuffle = codecParam.split("_")[1]; - int clevel_blosc = Integer.parseInt(codecParam.split("_")[2]); - builder = builder.withCodecs(c -> c.withBlosc(cname, shuffle, clevel_blosc)); - break; - case "gzip": - builder = builder.withCodecs(c -> c.withGzip(Integer.parseInt(codecParam))); - break; - case "zstd": - int clevel_zstd = Integer.parseInt(codecParam.split("_")[0]); - boolean checksum = Boolean.parseBoolean(codecParam.split("_")[1]); - builder = builder.withCodecs(c -> c.withZstd(clevel_zstd, checksum)); - break; - case "bytes": - builder = builder.withCodecs(c -> c.withBytes(codecParam)); - break; - case "transpose": - builder = builder.withCodecs(c -> c.withTranspose(new int[]{1, 0, 2})); - break; - case "sharding": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withBytes("LITTLE"), codecParam)); - break; - case "sharding_nested": - builder = builder.withCodecs(c -> c.withSharding(new int[]{2, 2, 4}, c1 -> c1.withSharding(new int[]{2, 1, 2}, c2 -> c2.withBytes("LITTLE")))); - break; - case "crc32c": - builder = builder.withCodecs(CodecBuilder::withCrc32c); - break; - default: - throw new IllegalArgumentException("Invalid Codec: " + codec); - } - - Array writeArray = Array.create(storeHandle, builder.build()); - writeArray.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{16, 16, 16}, testData)); - - //read in zarr-java - Array readArray = Array.open(storeHandle); - ucar.ma2.Array result = readArray.read(); - - Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); - Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); - Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); - - Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT)); - - //read in zarrita - ProcessBuilder pb = new ProcessBuilder("uv", "run", PYTHON_TEST_PATH.resolve("zarrita_read.py") - .toString(), codec, codecParam, TESTOUTPUT.toString()); - Process process = pb.start(); - - BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream())); - String line; - while ((line = reader.readLine()) != null) { - System.out.println(line); - } - - BufferedReader readerErr = new BufferedReader(new InputStreamReader(process.getErrorStream())); - while ((line = readerErr.readLine()) != null) { - System.err.println(line); - } - - int exitCode = process.waitFor(); - assert exitCode == 0; - } static Stream> invalidCodecBuilder() { return Stream.of( @@ -700,28 +560,6 @@ public void testV2create() throws IOException, ZarrException { Assertions.assertEquals(outArray.getByte(0), 0); } - @Test - public void testV2Filters() throws IOException, ZarrException { - DataType dataType = DataType.UINT32; - - dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( - new FilesystemStore(TESTOUTPUT).resolve("v2_create_filters"), - dev.zarr.zarrjava.v2.Array.metadataBuilder() - .withShape(10, 10) - .withDataType(dataType) - .withChunks(5, 5) - .withFillValue(2) - .withFilters(f -> f.withTranspose(new int[]{1, 0})) - .build() - ); - array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); - - ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(outArray.getSize(), 8 * 8); - Assertions.assertEquals(outArray.getByte(0), 0); - } - - @ParameterizedTest @ValueSource(ints = {0, 1, 5, 9}) public void testV2createZlib(int level) throws IOException, ZarrException { diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py new file mode 100644 index 00000000..d7207064 --- /dev/null +++ b/src/test/python-scripts/parse_codecs.py @@ -0,0 +1,83 @@ +import zarr +from zarr.codecs.blosc import BloscCodec +from zarr.codecs.bytes import BytesCodec +from zarr.codecs.crc32c_ import Crc32cCodec +from zarr.codecs.gzip import GzipCodec +from zarr.codecs.sharding import ShardingCodec, ShardingCodecIndexLocation +from zarr.codecs.transpose import TransposeCodec +from zarr.codecs.zstd import ZstdCodec +import zarrita +import numcodecs + +def parse_codecs_zarr_python(codec_string: str, param_string: str, zarr_version: int = 3): + compressor = None + filters = "auto" + serializer = "auto" + + if codec_string == "blosc" and zarr_version == 3: + cname, shuffle, clevel = param_string.split("_") + compressor = BloscCodec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel)) + elif codec_string == "blosc" and zarr_version == 2: + cname, shuffle, clevel = param_string.split("_") + if shuffle == "noshuffle": + shuffle = numcodecs.Blosc.NOSHUFFLE + elif shuffle == "shuffle": + shuffle = numcodecs.Blosc.SHUFFLE + elif shuffle == "bitshuffle": + shuffle = numcodecs.Blosc.BITSHUFFLE + else: + raise ValueError(f"Invalid shuffle: {shuffle}") + compressor = numcodecs.Blosc(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel)) + elif codec_string == "zlib" and zarr_version == 2: + compressor = numcodecs.Zlib(level=int(param_string)) + elif codec_string == "gzip" and zarr_version == 3: + compressor = GzipCodec(level=int(param_string)) + elif codec_string == "zstd" and zarr_version == 3: + level, checksum = param_string.split("_") + compressor = ZstdCodec(checksum=checksum == 'true', level=int(level)) + elif codec_string == "bytes" and zarr_version == 3: + serializer = BytesCodec(endian=param_string.lower()) + elif codec_string == "transpose" and zarr_version == 3: + filters = [TransposeCodec(order=(1, 0, 2))] + elif codec_string == "sharding" and zarr_version == 3: + serializer = ShardingCodec(chunk_shape=(2, 2, 4), codecs=(BytesCodec(endian="little"),), + index_location=ShardingCodecIndexLocation.start if param_string == "start" + else ShardingCodecIndexLocation.end) + elif codec_string == "sharding_nested" and zarr_version == 3: + serializer = ShardingCodec(chunk_shape=(2, 2, 4), codecs=(ShardingCodec(chunk_shape=(2, 1, 2), + codecs=[BytesCodec(endian="little")]),)) + elif codec_string == "crc32c" and zarr_version == 3: + compressor = Crc32cCodec() + else: + raise ValueError(f"Invalid codec: {codec_string}, zarr_version: {zarr_version}") + + return compressor, serializer, filters + +def parse_codecs_zarrita(codec_string: str, param_string: str): + codec = [] + if codec_string == "blosc": + cname, shuffle, clevel = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), + zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] + elif codec_string == "gzip": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] + elif codec_string == "zstd": + level, checksum = param_string.split("_") + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] + elif codec_string == "bytes": + codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] + elif codec_string == "transpose": + codec = [zarrita.codecs.transpose_codec((1, 0, 2)), zarrita.codecs.bytes_codec()] + elif codec_string == "sharding": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), codecs=[zarrita.codecs.bytes_codec("little")], + index_location=zarrita.metadata.ShardingCodecIndexLocation.start if param_string == "start" + else zarrita.metadata.ShardingCodecIndexLocation.end), + elif codec_string == "sharding_nested": + codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), + codecs=[zarrita.codecs.sharding_codec(chunk_shape=(2, 1, 2), codecs=[ + zarrita.codecs.bytes_codec("little")])]), + elif codec_string == "crc32c": + codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] + else: + raise ValueError(f"Invalid codec: {codec_string}") + return codec diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py new file mode 100644 index 00000000..1e31ff52 --- /dev/null +++ b/src/test/python-scripts/zarr_python_read.py @@ -0,0 +1,33 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr +from zarr.storage import LocalStore +from parse_codecs import parse_codecs_zarr_python + +codec_string = sys.argv[1] +param_string = sys.argv[2] +compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) +store_path = Path(sys.argv[3]) + +expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +a = zarr.open_array(store=LocalStore(store_path)) +read_data = a[:, :] +assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" + +b = zarr.create_array( + LocalStore(store_path / "expected"), + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype="uint32", + fill_value=0, + filters=filters, + serializer=serializer, + compressors=compressor, + attributes={'test_key': 'test_value'}, +) + +assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py new file mode 100644 index 00000000..5390a231 --- /dev/null +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -0,0 +1,35 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr + +from zarr.storage import LocalStore +from parse_codecs import parse_codecs_zarr_python + +codec_string = sys.argv[1] +param_string = sys.argv[2] +compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) +store_path = Path(sys.argv[3]) + +expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +a = zarr.open_array(store=LocalStore(store_path)) +read_data = a[:, :] +assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" + +b = zarr.create_array( + LocalStore(store_path / "expected"), + zarr_format=2, + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype='uint32', + fill_value=0, + filters=filters, + serializer=serializer, + compressors=compressor, +# attributes={'test_key': 'test_value'}, +) + +assert a.metadata == b.metadata, f"not equal: \n{a.metadata=}\n{b.metadata=}" diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py new file mode 100644 index 00000000..531d356d --- /dev/null +++ b/src/test/python-scripts/zarr_python_write.py @@ -0,0 +1,27 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr +from zarr.storage import LocalStore +from parse_codecs import parse_codecs_zarr_python + +codec_string = sys.argv[1] +param_string = sys.argv[2] +compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string) +store_path = Path(sys.argv[3]) + +testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +a = zarr.create_array( + LocalStore(store_path), + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype='int32', + filters=filters, + serializer=serializer, + compressors=compressor, + attributes={'answer': 42} +) +a[:, :] = testdata diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py new file mode 100644 index 00000000..9c124c12 --- /dev/null +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -0,0 +1,29 @@ +import sys +from pathlib import Path + +import numpy as np + +import zarr + +from zarr.storage import LocalStore +from parse_codecs import parse_codecs_zarr_python + +codec_string = sys.argv[1] +param_string = sys.argv[2] +compressor, serializer, filters = parse_codecs_zarr_python(codec_string, param_string, zarr_version=2) +store_path = Path(sys.argv[3]) + +testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) + +a = zarr.create_array( + LocalStore(store_path), + zarr_format=2, + shape=(16, 16, 16), + chunks=(2, 4, 8), + dtype='int32', + filters=filters, + serializer=serializer, + compressors=compressor, + attributes={'answer': 42} +) +a[:, :] = testdata diff --git a/src/test/python-scripts/zarrita_read.py b/src/test/python-scripts/zarrita_read.py index f84bf9bd..2769694e 100644 --- a/src/test/python-scripts/zarrita_read.py +++ b/src/test/python-scripts/zarrita_read.py @@ -3,35 +3,11 @@ import numpy as np import zarrita from zarrita.metadata import ShardingCodecIndexLocation +from parse_codecs import parse_codecs_zarrita codec_string = sys.argv[1] param_string = sys.argv[2] -codec = [] -if codec_string == "blosc": - cname, shuffle, clevel = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), - zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] -elif codec_string == "gzip": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] -elif codec_string == "zstd": - level, checksum = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] -elif codec_string == "bytes": - codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] -elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec((1, 0, 2)), zarrita.codecs.bytes_codec()] -elif codec_string == "sharding": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), codecs=[zarrita.codecs.bytes_codec("little")], - index_location=ShardingCodecIndexLocation.start if param_string == "start" - else ShardingCodecIndexLocation.end), -elif codec_string == "sharding_nested": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 2, 4), - codecs=[zarrita.codecs.sharding_codec(chunk_shape=(2, 1, 2), codecs=[ - zarrita.codecs.bytes_codec("little")])]), -elif codec_string == "crc32c": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] -else: - raise ValueError(f"Invalid {codec=}") +codec = parse_codecs_zarrita(codec_string, param_string) store = zarrita.LocalStore(sys.argv[3]) expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) diff --git a/src/test/python-scripts/zarrita_write.py b/src/test/python-scripts/zarrita_write.py index 2eb0fc23..4000f60b 100644 --- a/src/test/python-scripts/zarrita_write.py +++ b/src/test/python-scripts/zarrita_write.py @@ -3,44 +3,20 @@ import zarrita import numpy as np from zarrita.metadata import ShardingCodecIndexLocation +from parse_codecs import parse_codecs_zarrita codec_string = sys.argv[1] param_string = sys.argv[2] -codec = [] -if codec_string == "blosc": - cname, shuffle, clevel = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), - zarrita.codecs.blosc_codec(typesize=4, cname=cname, shuffle=shuffle, clevel=int(clevel))] -elif codec_string == "gzip": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.gzip_codec(level=int(param_string))] -elif codec_string == "zstd": - level, checksum = param_string.split("_") - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.zstd_codec(checksum=checksum == 'true', level=int(level))] -elif codec_string == "bytes": - codec = [zarrita.codecs.bytes_codec(endian=param_string.lower())] -elif codec_string == "transpose": - codec = [zarrita.codecs.transpose_codec((0, 1)), zarrita.codecs.bytes_codec()] -elif codec_string == "sharding": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 4), codecs=[zarrita.codecs.bytes_codec("little")], - index_location=ShardingCodecIndexLocation.start if param_string == "start" - else ShardingCodecIndexLocation.end), -elif codec_string == "sharding_nested": - codec = zarrita.codecs.sharding_codec(chunk_shape=(2, 4), - codecs=[zarrita.codecs.sharding_codec(chunk_shape=(1, 2), codecs=[ - zarrita.codecs.bytes_codec("little")])]), -elif codec_string == "crc32c": - codec = [zarrita.codecs.bytes_codec(), zarrita.codecs.crc32c_codec()] -else: - raise ValueError(f"Invalid {codec_string=}") +codec = parse_codecs_zarrita(codec_string, param_string) store = zarrita.LocalStore(sys.argv[3]) -testdata = np.arange(16 * 16, dtype='int32').reshape((16, 16)) +testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarrita.Array.create( store / 'read_from_zarrita' / codec_string / param_string, - shape=(16, 16), + shape=(16, 16, 16), + chunk_shape=(2, 4, 8), dtype='int32', - chunk_shape=(2, 8), codecs=codec, attributes={'answer': 42} ) From 84001243b8908f13c90ae76ea76cc11a99bcd3b2 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 17 Sep 2025 16:09:08 +0200 Subject: [PATCH 11/27] isolate v2 and v3 with common core --- .../zarrjava/{interfaces => core}/Array.java | 4 +- .../dev/zarr/zarrjava/core/ArrayMetadata.java | 167 ++++++++++++++++++ .../{interfaces => core}/DataType.java | 4 +- .../{ => core}/codec/ArrayArrayCodec.java | 2 +- .../{ => core}/codec/ArrayBytesCodec.java | 2 +- .../{ => core}/codec/BytesBytesCodec.java | 2 +- .../zarr/zarrjava/{ => core}/codec/Codec.java | 2 +- .../{ => core}/codec/CodecBuilder.java | 2 +- .../{ => core}/codec/CodecPipeline.java | 4 +- .../zarrjava/core/codec/core/BloscCodec.java | 73 ++++++++ .../zarrjava/interfaces/ArrayMetadata.java | 21 --- src/main/java/dev/zarr/zarrjava/v2/Array.java | 8 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 13 +- .../zarrjava/v2/ArrayMetadataBuilder.java | 10 +- .../v2/{DataTypeV2.java => DataType.java} | 37 ++-- .../dev/zarr/zarrjava/v2/codec/Codec.java | 2 +- .../zarr/zarrjava/v2/codec/CodecBuilder.java | 2 +- .../zarrjava/v2/codec/core/BloscCodec.java | 15 +- .../zarrjava/v2/codec/core/ZlibCodec.java | 4 +- src/main/java/dev/zarr/zarrjava/v3/Array.java | 4 +- .../dev/zarr/zarrjava/v3/ArrayMetadata.java | 151 +--------------- .../java/dev/zarr/zarrjava/v3/DataType.java | 2 +- .../dev/zarr/zarrjava/v3/codec/Codec.java | 2 +- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 8 +- .../zarr/zarrjava/v3/codec/CodecRegistry.java | 2 +- .../zarrjava/v3/codec/core/BloscCodec.java | 58 +----- .../zarrjava/v3/codec/core/BytesCodec.java | 2 +- .../zarrjava/v3/codec/core/Crc32cCodec.java | 4 +- .../zarrjava/v3/codec/core/GzipCodec.java | 2 +- .../v3/codec/core/ShardingIndexedCodec.java | 6 +- .../v3/codec/core/TransposeCodec.java | 2 +- .../zarrjava/v3/codec/core/ZstdCodec.java | 2 +- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 11 +- 33 files changed, 322 insertions(+), 308 deletions(-) rename src/main/java/dev/zarr/zarrjava/{interfaces => core}/Array.java (99%) create mode 100644 src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java rename src/main/java/dev/zarr/zarrjava/{interfaces => core}/DataType.java (55%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/ArrayArrayCodec.java (85%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/ArrayBytesCodec.java (94%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/BytesBytesCodec.java (94%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/Codec.java (91%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/CodecBuilder.java (85%) rename src/main/java/dev/zarr/zarrjava/{ => core}/codec/CodecPipeline.java (98%) create mode 100644 src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java delete mode 100644 src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java rename src/main/java/dev/zarr/zarrjava/v2/{DataTypeV2.java => DataType.java} (74%) diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java similarity index 99% rename from src/main/java/dev/zarr/zarrjava/interfaces/Array.java rename to src/main/java/dev/zarr/zarrjava/core/Array.java index 6bf25318..23bc4084 100644 --- a/src/main/java/dev/zarr/zarrjava/interfaces/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -1,11 +1,11 @@ -package dev.zarr.zarrjava.interfaces; +package dev.zarr.zarrjava.core; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.IndexingUtils; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.codec.CodecPipeline; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import ucar.ma2.InvalidRangeException; import javax.annotation.Nonnull; diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java new file mode 100644 index 00000000..7ea8a4bc --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -0,0 +1,167 @@ +package dev.zarr.zarrjava.core; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.utils.MultiArrayUtils; +import dev.zarr.zarrjava.utils.Utils; +import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; +import ucar.ma2.Array; + +import javax.annotation.Nonnull; +import java.nio.ByteBuffer; +import java.util.Arrays; + +public interface ArrayMetadata { + int ndim(); + + int[] chunkShape(); + + long[] shape(); + + DataType dataType(); + + Array allocateFillValueChunk(); + + ChunkKeyEncoding chunkKeyEncoding(); + + Object parsedFillValue(); + + static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) + throws ZarrException { + boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL; + boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8; + boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16; + boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32; + boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64; + boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32; + boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64; + + if (fillValue instanceof Boolean) { + Boolean fillValueBool = (Boolean) fillValue; + if (dataTypeIsBool) { + return fillValueBool; + } + } + if (fillValue instanceof Number) { + Number fillValueNumber = (Number) fillValue; + if (dataTypeIsBool) { + return fillValueNumber.byteValue() != 0; + } else if (dataTypeIsByte) { + return fillValueNumber.byteValue(); + } else if (dataTypeIsShort) { + return fillValueNumber.shortValue(); + } else if (dataTypeIsInt) { + return fillValueNumber.intValue(); + } else if (dataTypeIsLong) { + return fillValueNumber.longValue(); + } else if (dataTypeIsFloat) { + return fillValueNumber.floatValue(); + } else if (dataTypeIsDouble) { + return fillValueNumber.doubleValue(); + } + // Fallback to throwing below + } else if (fillValue instanceof String) { + String fillValueString = (String) fillValue; + if (fillValueString.equals("NaN")) { + if (dataTypeIsFloat) { + return Float.NaN; + } else if (dataTypeIsDouble) { + return Double.NaN; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } else if (fillValueString.equals("+Infinity")) { + if (dataTypeIsFloat) { + return Float.POSITIVE_INFINITY; + } else if (dataTypeIsDouble) { + return Double.POSITIVE_INFINITY; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } else if (fillValueString.equals("-Infinity")) { + if (dataTypeIsFloat) { + return Float.NEGATIVE_INFINITY; + } else if (dataTypeIsDouble) { + return Double.NEGATIVE_INFINITY; + } + throw new ZarrException( + "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); + } + else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { + ByteBuffer buf = null; + if (fillValueString.startsWith("0b")) { + buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { + for (int i = 0; i < dataType.getByteCount(); i++) { + b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 8, 2 + (i + 1) * 8), + 2)); + } + return b; + }); + } else if (fillValueString.startsWith("0x")) { + buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { + for (int i = 0; i < dataType.getByteCount(); i++) { + b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 2, 2 + (i + 1) * 2), + 16)); + } + return b; + }); + } + if (buf != null) { + if (dataTypeIsBool) { + return buf.get() != 0; + } else if (dataTypeIsByte) { + return buf.get(); + } else if (dataTypeIsShort) { + return buf.getShort(); + } else if (dataTypeIsInt) { + return buf.getInt(); + } else if (dataTypeIsLong) { + return buf.getLong(); + } else if (dataTypeIsFloat) { + return buf.getFloat(); + } else if (dataTypeIsDouble) { + return buf.getDouble(); + // Fallback to throwing below + } + } + } + } + throw new ZarrException("Invalid fill value '" + fillValue + "'."); + } + + final class CoreArrayMetadata { + + public final long[] shape; + public final int[] chunkShape; + public final dev.zarr.zarrjava.v3.DataType dataType; + public final Object parsedFillValue; + + public CoreArrayMetadata(long[] shape, int[] chunkShape, dev.zarr.zarrjava.v3.DataType dataType, + Object parsedFillValue) { + this.shape = shape; + this.chunkShape = chunkShape; + this.dataType = dataType; + this.parsedFillValue = parsedFillValue; + } + + public int ndim() { + return shape.length; + } + + public int chunkSize() { + return Arrays.stream(chunkShape) + .reduce(1, (acc, a) -> acc * a); + } + + public int chunkByteLength() { + return this.dataType.getByteCount() * chunkSize(); + } + + public ucar.ma2.Array allocateFillValueChunk() { + ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunkShape); + MultiArrayUtils.fill(outputArray, parsedFillValue); + return outputArray; + } + } + + +} diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/DataType.java b/src/main/java/dev/zarr/zarrjava/core/DataType.java similarity index 55% rename from src/main/java/dev/zarr/zarrjava/interfaces/DataType.java rename to src/main/java/dev/zarr/zarrjava/core/DataType.java index 04112047..d63de069 100644 --- a/src/main/java/dev/zarr/zarrjava/interfaces/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/core/DataType.java @@ -1,5 +1,7 @@ -package dev.zarr.zarrjava.interfaces; +package dev.zarr.zarrjava.core; public interface DataType { ucar.ma2.DataType getMA2DataType(); + + int getByteCount(); } diff --git a/src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java similarity index 85% rename from src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java rename to src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java index 2ecba670..9acf1dde 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; import ucar.ma2.Array; diff --git a/src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java similarity index 94% rename from src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java rename to src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java index e080786e..ada36fff 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; diff --git a/src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java similarity index 94% rename from src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java rename to src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java index b48d4adf..79a2271e 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; diff --git a/src/main/java/dev/zarr/zarrjava/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java similarity index 91% rename from src/main/java/dev/zarr/zarrjava/codec/Codec.java rename to src/main/java/dev/zarr/zarrjava/core/codec/Codec.java index 441ed603..89e52fdd 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.ArrayMetadata; diff --git a/src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java similarity index 85% rename from src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java rename to src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java index 0b5e9d64..c155584b 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.v3.DataType; diff --git a/src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java similarity index 98% rename from src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java rename to src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java index 781c8059..9d83836e 100644 --- a/src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java @@ -1,8 +1,8 @@ -package dev.zarr.zarrjava.codec; +package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; import java.nio.ByteBuffer; import java.util.Arrays; import javax.annotation.Nonnull; diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java new file mode 100644 index 00000000..b233a06d --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java @@ -0,0 +1,73 @@ +package dev.zarr.zarrjava.core.codec.core; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParseException; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.scalableminds.bloscjava.Blosc; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.utils.Utils; + +import java.io.IOException; +import java.nio.ByteBuffer; + +public interface BloscCodec extends BytesBytesCodec { + + @Override + default ByteBuffer decode(ByteBuffer chunkBytes) + throws ZarrException { + try { + return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); + } catch (Exception ex) { + throw new ZarrException("Error in decoding blosc.", ex); + } + } + + final class CustomCompressorDeserializer extends StdDeserializer { + + public CustomCompressorDeserializer() { + this(null); + } + + public CustomCompressorDeserializer(Class vc) { + super(vc); + } + + @Override + public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContext ctxt) + throws IOException { + String cname = jsonParser.getCodec() + .readValue(jsonParser, String.class); + Blosc.Compressor compressor = Blosc.Compressor.fromString(cname); + if (compressor == null) { + throw new JsonParseException( + jsonParser, + String.format("Could not parse the Blosc.Compressor. Got '%s'", cname) + ); + } + return compressor; + } + } + + final class CustomCompressorSerializer extends StdSerializer { + + public CustomCompressorSerializer() { + super(Blosc.Compressor.class); + } + + public CustomCompressorSerializer(Class t) { + super(t); + } + + @Override + public void serialize(Blosc.Compressor compressor, JsonGenerator generator, + SerializerProvider provider) + throws IOException { + generator.writeString(compressor.getValue()); + } + } +} diff --git a/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java deleted file mode 100644 index 7e79e0dd..00000000 --- a/src/main/java/dev/zarr/zarrjava/interfaces/ArrayMetadata.java +++ /dev/null @@ -1,21 +0,0 @@ -package dev.zarr.zarrjava.interfaces; - -import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; -import ucar.ma2.Array; - -public interface ArrayMetadata { - int ndim(); - - int[] chunkShape(); - - long[] shape(); - - DataType dataType(); - - Array allocateFillValueChunk(); - - ChunkKeyEncoding chunkKeyEncoding(); - - Object parsedFillValue(); - -} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index df1a9f2c..7ee7ee11 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.codec.CodecPipeline; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.CodecRegistry; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; @@ -17,7 +17,7 @@ import java.util.function.Function; import java.util.stream.Collectors; -public class Array implements dev.zarr.zarrjava.interfaces.Array { +public class Array implements dev.zarr.zarrjava.core.Array { static final String ZARRAY = ".zarray"; public ArrayMetadata metadata; @@ -28,9 +28,9 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE this.storeHandle = storeHandle; this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( - new dev.zarr.zarrjava.codec.Codec[]{}, + new dev.zarr.zarrjava.core.codec.Codec[]{}, metadata.filters == null ? new Codec[]{} : metadata.filters, - new dev.zarr.zarrjava.codec.Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, + new dev.zarr.zarrjava.core.codec.Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} ), metadata.coreArrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 96bf400c..02be9107 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; @@ -14,10 +13,10 @@ import javax.annotation.Nullable; -import static dev.zarr.zarrjava.v3.ArrayMetadata.parseFillValue; +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; -public class ArrayMetadata implements dev.zarr.zarrjava.interfaces.ArrayMetadata { +public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { static final int ZARR_FORMAT = 2; @JsonProperty("zarr_format") @@ -27,9 +26,9 @@ public class ArrayMetadata implements dev.zarr.zarrjava.interfaces.ArrayMetadata public int[] chunks; @JsonProperty("dtype") - public DataTypeV2 dataTypeV2; + public DataType dataTypeV2; @JsonIgnore - public final DataType dataType; + public final dev.zarr.zarrjava.v3.DataType dataType; @JsonIgnore public final Endianness endianness; @@ -57,7 +56,7 @@ public ArrayMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "shape", required = true) long[] shape, @JsonProperty(value = "chunks", required = true) int[] chunks, - @JsonProperty(value = "dtype", required = true) DataTypeV2 dataTypeV2, + @JsonProperty(value = "dtype", required = true) DataType dataTypeV2, @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo test when null @JsonProperty(value = "order", required = true) Order order, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator, @@ -103,7 +102,7 @@ public long[] shape() { } @Override - public DataType dataType() { + public dev.zarr.zarrjava.v3.DataType dataType() { return dataType; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index c625348c..0b131a4b 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -1,7 +1,6 @@ package dev.zarr.zarrjava.v2; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.CodecBuilder; @@ -11,7 +10,7 @@ public class ArrayMetadataBuilder { long[] shape = null; int[] chunks = null; - DataTypeV2 dataTypeV2 = null; + DataType dataTypeV2 = null; Order order = Order.C; Separator dimensionSeparator = Separator.DOT; Object fillValue = 0; @@ -45,13 +44,14 @@ public ArrayMetadataBuilder withChunks(int... chunks) { return this; } - public ArrayMetadataBuilder withDataType(DataTypeV2 dataTypeV2) { + public ArrayMetadataBuilder withDataType(DataType dataTypeV2) { this.dataTypeV2 = dataTypeV2; return this; } - public ArrayMetadataBuilder withDataType(DataType dataType) { - this.dataTypeV2 = DataTypeV2.fromDataType(dataType); + //TODO remove v3? + public ArrayMetadataBuilder withDataType(dev.zarr.zarrjava.v3.DataType dataType) { + this.dataTypeV2 = DataType.fromDataType(dataType); return this; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java similarity index 74% rename from src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java rename to src/main/java/dev/zarr/zarrjava/v2/DataType.java index 734d4863..b1c502cb 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataTypeV2.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -2,9 +2,8 @@ import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.DataType; -public enum DataTypeV2 implements dev.zarr.zarrjava.interfaces.DataType{ +public enum DataType implements dev.zarr.zarrjava.core.DataType { BOOL("b1", Endianness.UNSPECIFIED), INT8("i1", Endianness.UNSPECIFIED), INT16("i2", Endianness.LITTLE), @@ -20,12 +19,13 @@ public enum DataTypeV2 implements dev.zarr.zarrjava.interfaces.DataType{ private final String dtype; private final Endianness endianness; - DataTypeV2(String dtype, Endianness endianness) { + + DataType(String dtype, Endianness endianness) { this.dtype = dtype; this.endianness = endianness; } - public static DataTypeV2 fromDataType(DataType dataType) { + public static DataType fromDataType(dev.zarr.zarrjava.v3.DataType dataType) { switch (dataType) { case BOOL: return BOOL; @@ -58,29 +58,29 @@ public Endianness getEndianness() { return endianness; } - public DataType toV3() throws ZarrException { + public dev.zarr.zarrjava.v3.DataType toV3() throws ZarrException { if (this.dtype.equals(BOOL.dtype)) - return DataType.BOOL; + return dev.zarr.zarrjava.v3.DataType.BOOL; if (this.dtype.equals(INT8.dtype)) - return DataType.INT8; + return dev.zarr.zarrjava.v3.DataType.INT8; if (this.dtype.equals(INT16.dtype)) - return DataType.INT16; + return dev.zarr.zarrjava.v3.DataType.INT16; if (this.dtype.equals(INT32.dtype)) - return DataType.INT32; + return dev.zarr.zarrjava.v3.DataType.INT32; if (this.dtype.equals(INT64.dtype)) - return DataType.INT64; + return dev.zarr.zarrjava.v3.DataType.INT64; if (this.dtype.equals(UINT8.dtype)) - return DataType.UINT8; + return dev.zarr.zarrjava.v3.DataType.UINT8; if (this.dtype.equals(UINT16.dtype)) - return DataType.UINT16; + return dev.zarr.zarrjava.v3.DataType.UINT16; if (this.dtype.equals(UINT32.dtype)) - return DataType.UINT32; + return dev.zarr.zarrjava.v3.DataType.UINT32; if (this.dtype.equals(UINT64.dtype)) - return DataType.UINT64; + return dev.zarr.zarrjava.v3.DataType.UINT64; if (this.dtype.equals(FLOAT32.dtype)) - return DataType.FLOAT32; + return dev.zarr.zarrjava.v3.DataType.FLOAT32; if (this.dtype.equals(FLOAT64.dtype)) - return DataType.FLOAT64; + return dev.zarr.zarrjava.v3.DataType.FLOAT64; throw new ZarrException("Unknown DataTypeV2: " + this.dtype); } @@ -119,4 +119,9 @@ public ucar.ma2.DataType getMA2DataType() { } } + @Override + public int getByteCount() { + return Integer.parseInt(dtype.substring(1)); + } + } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index 94ad5828..58cec409 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") -public abstract class Codec implements dev.zarr.zarrjava.codec.Codec { +public abstract class Codec implements dev.zarr.zarrjava.core.codec.Codec { protected ArrayMetadata.CoreArrayMetadata arrayMetadata; diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java index cc9203b7..bb0ac5eb 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java @@ -9,7 +9,7 @@ import java.util.ArrayList; import java.util.List; -public class CodecBuilder extends dev.zarr.zarrjava.codec.CodecBuilder { +public class CodecBuilder extends dev.zarr.zarrjava.core.codec.CodecBuilder { protected List codecs; diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 5f97b5d0..4f615c94 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -12,17 +12,15 @@ import com.fasterxml.jackson.databind.ser.std.StdSerializer; import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.codec.BytesBytesCodec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec.CustomCompressorDeserializer; -import dev.zarr.zarrjava.v3.codec.core.BloscCodec.CustomCompressorSerializer; + import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; -public class BloscCodec extends Codec implements BytesBytesCodec { +public class BloscCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BloscCodec { public final String id = "blosc"; @@ -61,15 +59,6 @@ public BloscCodec( this.blocksize = blocksize; } - @Override - public ByteBuffer decode(ByteBuffer chunkBytes) - throws ZarrException { - try { - return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); - } catch (Exception ex) { - throw new ZarrException("Error in decoding blosc.", ex); - } - } @Override public ByteBuffer encode(ByteBuffer chunkBytes) diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 517732ef..5497cb62 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -5,8 +5,8 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.ArrayMetadata; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; diff --git a/src/main/java/dev/zarr/zarrjava/v3/Array.java b/src/main/java/dev/zarr/zarrjava/v3/Array.java index 3488fb58..f21b020a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -4,7 +4,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.store.StoreHandle; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.codec.CodecPipeline; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import java.io.IOException; import java.nio.ByteBuffer; import java.util.Arrays; @@ -14,7 +14,7 @@ import java.util.stream.Collectors; import javax.annotation.Nonnull; -public class Array extends Node implements dev.zarr.zarrjava.interfaces.Array { +public class Array extends Node implements dev.zarr.zarrjava.core.Array { public ArrayMetadata metadata; CodecPipeline codecPipeline; diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java index fb7cf87a..d50d9d05 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadata.java @@ -4,15 +4,13 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.chunkgrid.ChunkGrid; import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec; +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; -import java.nio.ByteBuffer; import java.util.Arrays; import java.util.Map; import java.util.Optional; @@ -20,7 +18,7 @@ import javax.annotation.Nullable; -public final class ArrayMetadata implements dev.zarr.zarrjava.interfaces.ArrayMetadata { +public final class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { static final String NODE_TYPE = "array"; static final int ZARR_FORMAT = 3; @@ -142,117 +140,6 @@ public ArrayMetadata( ); } - public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType) - throws ZarrException { - if (fillValue instanceof Boolean) { - Boolean fillValueBool = (Boolean) fillValue; - if (dataType == DataType.BOOL) { - return fillValueBool; - } - } - if (fillValue instanceof Number) { - Number fillValueNumber = (Number) fillValue; - switch (dataType) { - case BOOL: - return fillValueNumber.byteValue() != 0; - case INT8: - case UINT8: - return fillValueNumber.byteValue(); - case INT16: - case UINT16: - return fillValueNumber.shortValue(); - case INT32: - case UINT32: - return fillValueNumber.intValue(); - case INT64: - case UINT64: - return fillValueNumber.longValue(); - case FLOAT32: - return fillValueNumber.floatValue(); - case FLOAT64: - return fillValueNumber.doubleValue(); - default: - // Fallback to throwing below - } - } else if (fillValue instanceof String) { - String fillValueString = (String) fillValue; - if (fillValueString.equals("NaN")) { - switch (dataType) { - case FLOAT32: - return Float.NaN; - case FLOAT64: - return Double.NaN; - default: - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } - } else if (fillValueString.equals("+Infinity")) { - switch (dataType) { - case FLOAT32: - return Float.POSITIVE_INFINITY; - case FLOAT64: - return Double.POSITIVE_INFINITY; - default: - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } - } else if (fillValueString.equals("-Infinity")) { - switch (dataType) { - case FLOAT32: - return Float.NEGATIVE_INFINITY; - case FLOAT64: - return Double.NEGATIVE_INFINITY; - default: - throw new ZarrException( - "Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'."); - } - } else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) { - ByteBuffer buf = null; - if (fillValueString.startsWith("0b")) { - buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { - for (int i = 0; i < dataType.getByteCount(); i++) { - b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 8, 2 + (i + 1) * 8), - 2)); - } - return b; - }); - } else if (fillValueString.startsWith("0x")) { - buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> { - for (int i = 0; i < dataType.getByteCount(); i++) { - b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 2, 2 + (i + 1) * 2), - 16)); - } - return b; - }); - } - if (buf != null) { - switch (dataType) { - case BOOL: - return buf.get() != 0; - case INT8: - case UINT8: - return buf.get(); - case INT16: - case UINT16: - return buf.getShort(); - case INT32: - case UINT32: - return buf.getInt(); - case INT64: - case UINT64: - return buf.getLong(); - case FLOAT32: - return buf.getFloat(); - case FLOAT64: - return buf.getDouble(); - default: - // Fallback to throwing below - } - } - } - } - throw new ZarrException("Invalid fill value '" + fillValue + "'."); - } public ucar.ma2.Array allocateFillValueChunk() { return coreArrayMetadata.allocateFillValueChunk(); @@ -298,39 +185,5 @@ public int chunkByteLength() { return coreArrayMetadata.chunkByteLength(); } - public static final class CoreArrayMetadata { - - public final long[] shape; - public final int[] chunkShape; - public final DataType dataType; - public final Object parsedFillValue; - - public CoreArrayMetadata(long[] shape, int[] chunkShape, DataType dataType, - Object parsedFillValue) { - this.shape = shape; - this.chunkShape = chunkShape; - this.dataType = dataType; - this.parsedFillValue = parsedFillValue; - } - - public int ndim() { - return shape.length; - } - - public int chunkSize() { - return Arrays.stream(chunkShape) - .reduce(1, (acc, a) -> acc * a); - } - - public int chunkByteLength() { - return this.dataType.getByteCount() * chunkSize(); - } - - public ucar.ma2.Array allocateFillValueChunk() { - ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunkShape); - MultiArrayUtils.fill(outputArray, parsedFillValue); - return outputArray; - } - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/DataType.java b/src/main/java/dev/zarr/zarrjava/v3/DataType.java index 46384f56..9d9842c0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v3/DataType.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonValue; -public enum DataType implements dev.zarr.zarrjava.interfaces.DataType { +public enum DataType implements dev.zarr.zarrjava.core.DataType { BOOL("bool", 1), INT8("int8", 1), INT16("int16", 2), diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index c239d4ee..439e872d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public abstract class Codec implements dev.zarr.zarrjava.codec.Codec { +public abstract class Codec implements dev.zarr.zarrjava.core.codec.Codec { protected ArrayMetadata.CoreArrayMetadata arrayMetadata; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index a25071b3..e452db99 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -2,9 +2,9 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.codec.ArrayArrayCodec; -import dev.zarr.zarrjava.codec.ArrayBytesCodec; -import dev.zarr.zarrjava.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.core.*; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration; @@ -15,7 +15,7 @@ import java.util.List; import java.util.function.Function; -public class CodecBuilder extends dev.zarr.zarrjava.codec.CodecBuilder { +public class CodecBuilder extends dev.zarr.zarrjava.core.codec.CodecBuilder { protected List codecs; public CodecBuilder(DataType dataType) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index 770dfced..a02fc621 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import dev.zarr.zarrjava.v3.codec.core.*; -import dev.zarr.zarrjava.codec.Codec; +import dev.zarr.zarrjava.core.codec.Codec; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index 7601622a..f9d70f8a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -16,12 +16,12 @@ import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.BytesBytesCodec; + import java.io.IOException; import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class BloscCodec extends Codec implements BytesBytesCodec { +public class BloscCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BloscCodec { public final String name = "blosc"; @Nonnull @@ -33,16 +33,6 @@ public BloscCodec( this.configuration = configuration; } - @Override - public ByteBuffer decode(ByteBuffer chunkBytes) - throws ZarrException { - try { - return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); - } catch (Exception ex) { - throw new ZarrException("Error in decoding blosc.", ex); - } - } - @Override public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { @@ -91,50 +81,6 @@ public void serialize(Blosc.Shuffle shuffle, JsonGenerator generator, } } - public static final class CustomCompressorDeserializer extends StdDeserializer { - - public CustomCompressorDeserializer() { - this(null); - } - - public CustomCompressorDeserializer(Class vc) { - super(vc); - } - - @Override - public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContext ctxt) - throws IOException { - String cname = jsonParser.getCodec() - .readValue(jsonParser, String.class); - Blosc.Compressor compressor = Blosc.Compressor.fromString(cname); - if (compressor == null) { - throw new JsonParseException( - jsonParser, - String.format("Could not parse the Blosc.Compressor. Got '%s'", cname) - ); - } - return compressor; - } - } - - public static final class CustomCompressorSerializer extends StdSerializer { - - public CustomCompressorSerializer() { - super(Blosc.Compressor.class); - } - - public CustomCompressorSerializer(Class t) { - super(t); - } - - @Override - public void serialize(Blosc.Compressor compressor, JsonGenerator generator, - SerializerProvider provider) - throws IOException { - generator.writeString(compressor.getValue()); - } - } - public static final class CustomShuffleDeserializer extends StdDeserializer { public CustomShuffleDeserializer() { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index 1f4d824f..f655eb74 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -6,7 +6,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import java.nio.ByteBuffer; import java.nio.ByteOrder; import javax.annotation.Nonnull; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 00e829f2..85e8bf79 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -5,8 +5,8 @@ import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.CRC32C; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.codec.BytesBytesCodec; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; import java.nio.ByteBuffer; import java.nio.ByteOrder; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index 589b274b..bea531f6 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -6,7 +6,7 @@ import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; import java.io.IOException; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index e6341468..3b71bf77 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -8,11 +8,11 @@ import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata; +import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata; import dev.zarr.zarrjava.v3.DataType; -import dev.zarr.zarrjava.codec.ArrayBytesCodec; +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; import dev.zarr.zarrjava.v3.codec.Codec; -import dev.zarr.zarrjava.codec.CodecPipeline; +import dev.zarr.zarrjava.core.codec.CodecPipeline; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Arrays; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index bc24ea06..9127968a 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.ArrayArrayCodec; +import dev.zarr.zarrjava.core.codec.ArrayArrayCodec; import ucar.ma2.Array; import javax.annotation.Nonnull; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 208d74b3..5cede681 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -7,7 +7,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.codec.BytesBytesCodec; +import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import javax.annotation.Nonnull; import java.nio.ByteBuffer; diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 03573626..43953a28 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -34,6 +34,7 @@ import java.util.function.Function; import java.util.stream.Stream; +import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue; import static org.junit.Assert.assertThrows; public class ZarrTest { @@ -377,11 +378,11 @@ public void testV3ArrayMetadataBuilder() throws ZarrException { @Test public void testV3FillValue() throws ZarrException { - Assertions.assertEquals((int) ArrayMetadata.parseFillValue(0, DataType.UINT32), 0); - Assertions.assertEquals((int) ArrayMetadata.parseFillValue("0x00010203", DataType.UINT32), 50462976); - Assertions.assertEquals((byte) ArrayMetadata.parseFillValue("0b00000010", DataType.UINT8), 2); - assert Double.isNaN((double) ArrayMetadata.parseFillValue("NaN", DataType.FLOAT64)); - assert Double.isInfinite((double) ArrayMetadata.parseFillValue("-Infinity", DataType.FLOAT64)); + Assertions.assertEquals((int) parseFillValue(0, DataType.UINT32), 0); + Assertions.assertEquals((int) parseFillValue("0x00010203", DataType.UINT32), 50462976); + Assertions.assertEquals((byte) parseFillValue("0b00000010", DataType.UINT8), 2); + assert Double.isNaN((double) parseFillValue("NaN", DataType.FLOAT64)); + assert Double.isInfinite((double) parseFillValue("-Infinity", DataType.FLOAT64)); } @Test From 494b01ae7d63cac5141f8dd774c93e1e572a33fc Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 17 Sep 2025 17:32:46 +0200 Subject: [PATCH 12/27] isolate v2 and v3 with common core.chunkkeyencoding --- .../dev/zarr/zarrjava/core/ArrayMetadata.java | 6 +-- .../chunkkeyencoding/ChunkKeyEncoding.java | 7 ++++ .../chunkkeyencoding/Separator.java | 2 +- .../zarrjava/core/codec/CodecBuilder.java | 2 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 22 +++++------ .../zarrjava/v2/ArrayMetadataBuilder.java | 38 ++++++------------- src/main/java/dev/zarr/zarrjava/v2/Codec.java | 5 --- .../chunkkeyencoding/V2ChunkKeyEncoding.java | 33 ++++++++++++++++ .../zarr/zarrjava/v2/codec/CodecBuilder.java | 2 +- .../zarrjava/v3/ArrayMetadataBuilder.java | 2 +- .../v3/chunkkeyencoding/ChunkKeyEncoding.java | 2 +- .../DefaultChunkKeyEncoding.java | 2 + .../chunkkeyencoding/V2ChunkKeyEncoding.java | 2 + .../zarr/zarrjava/v3/codec/CodecBuilder.java | 2 +- .../dev/zarr/zarrjava/ZarrPythonTests.java | 6 +-- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 18 ++++----- 16 files changed, 87 insertions(+), 64 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java rename src/main/java/dev/zarr/zarrjava/{v3 => core}/chunkkeyencoding/Separator.java (84%) delete mode 100644 src/main/java/dev/zarr/zarrjava/v2/Codec.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java diff --git a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java index 7ea8a4bc..fc42294a 100644 --- a/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java @@ -3,7 +3,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.MultiArrayUtils; import dev.zarr.zarrjava.utils.Utils; -import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; +import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import ucar.ma2.Array; import javax.annotation.Nonnull; @@ -132,10 +132,10 @@ final class CoreArrayMetadata { public final long[] shape; public final int[] chunkShape; - public final dev.zarr.zarrjava.v3.DataType dataType; + public final DataType dataType; public final Object parsedFillValue; - public CoreArrayMetadata(long[] shape, int[] chunkShape, dev.zarr.zarrjava.v3.DataType dataType, + public CoreArrayMetadata(long[] shape, int[] chunkShape, DataType dataType, Object parsedFillValue) { this.shape = shape; this.chunkShape = chunkShape; diff --git a/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java new file mode 100644 index 00000000..c82cb759 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/ChunkKeyEncoding.java @@ -0,0 +1,7 @@ +package dev.zarr.zarrjava.core.chunkkeyencoding; + +public interface ChunkKeyEncoding { + + String[] encodeChunkKey(long[] chunkCoords); + +} diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/Separator.java b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java similarity index 84% rename from src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/Separator.java rename to src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java index 0aa49109..5074cd50 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/Separator.java +++ b/src/main/java/dev/zarr/zarrjava/core/chunkkeyencoding/Separator.java @@ -1,4 +1,4 @@ -package dev.zarr.zarrjava.v3.chunkkeyencoding; +package dev.zarr.zarrjava.core.chunkkeyencoding; import com.fasterxml.jackson.annotation.JsonValue; diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java index c155584b..f4f17069 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java @@ -1,6 +1,6 @@ package dev.zarr.zarrjava.core.codec; -import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.core.DataType; public abstract class CodecBuilder { diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 02be9107..edfb3a7c 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -4,10 +4,10 @@ import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonProperty; import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.utils.MultiArrayUtils; -import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; -import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; -import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.v2.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v2.codec.Codec; import ucar.ma2.Array; @@ -26,9 +26,8 @@ public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { public int[] chunks; @JsonProperty("dtype") - public DataType dataTypeV2; - @JsonIgnore - public final dev.zarr.zarrjava.v3.DataType dataType; + public DataType dataType; + @JsonIgnore public final Endianness endianness; @@ -56,7 +55,7 @@ public ArrayMetadata( @JsonProperty(value = "zarr_format", required = true) int zarrFormat, @JsonProperty(value = "shape", required = true) long[] shape, @JsonProperty(value = "chunks", required = true) int[] chunks, - @JsonProperty(value = "dtype", required = true) DataType dataTypeV2, + @JsonProperty(value = "dtype", required = true) DataType dataType, @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo test when null @JsonProperty(value = "order", required = true) Order order, @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator, @@ -70,9 +69,8 @@ public ArrayMetadata( } this.shape = shape; this.chunks = chunks; - this.dataTypeV2 = dataTypeV2; - this.endianness = dataTypeV2.getEndianness(); - this.dataType = dataTypeV2.toV3(); + this.dataType = dataType; + this.endianness = dataType.getEndianness(); this.fillValue = fillValue; this.parsedFillValue = parseFillValue(fillValue, this.dataType); this.order = order; @@ -102,7 +100,7 @@ public long[] shape() { } @Override - public dev.zarr.zarrjava.v3.DataType dataType() { + public DataType dataType() { return dataType; } @@ -116,7 +114,7 @@ public Array allocateFillValueChunk() { @Override public ChunkKeyEncoding chunkKeyEncoding() { Separator separator = dimensionSeparator == null ? Separator.DOT : dimensionSeparator; - return new V2ChunkKeyEncoding(new V2ChunkKeyEncoding.Configuration(separator)); + return new V2ChunkKeyEncoding(separator); } @Override diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 0b131a4b..5bb59421 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -1,7 +1,7 @@ package dev.zarr.zarrjava.v2; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.CodecBuilder; @@ -10,7 +10,7 @@ public class ArrayMetadataBuilder { long[] shape = null; int[] chunks = null; - DataType dataTypeV2 = null; + DataType dataType = null; Order order = Order.C; Separator dimensionSeparator = Separator.DOT; Object fillValue = 0; @@ -25,7 +25,7 @@ protected static ArrayMetadataBuilder fromArrayMetadata(ArrayMetadata arrayMetad ArrayMetadataBuilder builder = new ArrayMetadataBuilder(); builder.shape = arrayMetadata.shape; builder.chunks = arrayMetadata.chunks; - builder.dataTypeV2 = arrayMetadata.dataTypeV2; + builder.dataType = arrayMetadata.dataType; builder.order = arrayMetadata.order; builder.dimensionSeparator = arrayMetadata.dimensionSeparator; builder.fillValue = arrayMetadata.parsedFillValue; @@ -45,13 +45,7 @@ public ArrayMetadataBuilder withChunks(int... chunks) { } public ArrayMetadataBuilder withDataType(DataType dataTypeV2) { - this.dataTypeV2 = dataTypeV2; - return this; - } - - //TODO remove v3? - public ArrayMetadataBuilder withDataType(dev.zarr.zarrjava.v3.DataType dataType) { - this.dataTypeV2 = DataType.fromDataType(dataType); + this.dataType = dataTypeV2; return this; } @@ -76,10 +70,10 @@ public ArrayMetadataBuilder withFilters(Codec... filters) { } public ArrayMetadataBuilder withFilters(Function codecBuilder) throws ZarrException { - if (dataTypeV2 == null) { + if (dataType == null) { throw new IllegalStateException("Please call `withDataType` first."); } - CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3()); + CodecBuilder nestedCodecBuilder = new CodecBuilder(dataType); this.filters = codecBuilder.apply(nestedCodecBuilder) .build(); return this; @@ -91,25 +85,17 @@ public ArrayMetadataBuilder withCompressor(Codec compressor) { } public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { - try { - this.compressor = new CodecBuilder(dataTypeV2.toV3()) + this.compressor = new CodecBuilder(dataType) .withBlosc(cname, shuffle, clevel) .build()[0]; - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; + return this; } public ArrayMetadataBuilder withZlibCompressor(int level) { - try { - this.compressor = new CodecBuilder(dataTypeV2.toV3()) + this.compressor = new CodecBuilder(dataType) .withZlib(level) .build()[0]; - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; + return this; } public ArrayMetadata build() throws ZarrException { @@ -119,14 +105,14 @@ public ArrayMetadata build() throws ZarrException { if (chunks == null) { throw new IllegalStateException("Please call `withChunks` first."); } - if (dataTypeV2 == null) { + if (dataType == null) { throw new IllegalStateException("Please call `withDataType` first."); } return new ArrayMetadata( 2, shape, chunks, - dataTypeV2, + dataType, fillValue, order, dimensionSeparator, diff --git a/src/main/java/dev/zarr/zarrjava/v2/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/Codec.java deleted file mode 100644 index b12daed2..00000000 --- a/src/main/java/dev/zarr/zarrjava/v2/Codec.java +++ /dev/null @@ -1,5 +0,0 @@ -package dev.zarr.zarrjava.v2; - -public class Codec { - -} diff --git a/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java new file mode 100644 index 00000000..712306e9 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/chunkkeyencoding/V2ChunkKeyEncoding.java @@ -0,0 +1,33 @@ +package dev.zarr.zarrjava.v2.chunkkeyencoding; + +import dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; + +import javax.annotation.Nonnull; +import java.util.Arrays; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public class V2ChunkKeyEncoding implements ChunkKeyEncoding { + + public final String name = "v2"; + @Nonnull + public final Separator separator; + + public V2ChunkKeyEncoding( + @Nonnull Separator separator + ) { + this.separator = separator; + } + + @Override + public String[] encodeChunkKey(long[] chunkCoords) { + Stream keys = Arrays.stream(chunkCoords) + .mapToObj(Long::toString); + if (separator == Separator.SLASH) { + return keys.toArray(String[]::new); + } + return new String[]{keys.collect(Collectors.joining(this.separator.getValue()))}; + } +} + diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java index bb0ac5eb..f724e585 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java @@ -3,7 +3,7 @@ import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; -import dev.zarr.zarrjava.v3.DataType; +import dev.zarr.zarrjava.v2.DataType; import dev.zarr.zarrjava.v2.codec.core.*; import java.util.ArrayList; diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index a844802e..cdfe90cd 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -5,7 +5,7 @@ import dev.zarr.zarrjava.v3.chunkgrid.RegularChunkGrid; import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding; import dev.zarr.zarrjava.v3.chunkkeyencoding.DefaultChunkKeyEncoding; -import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java index dc34f2ee..cf82ad84 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/ChunkKeyEncoding.java @@ -9,7 +9,7 @@ @JsonSubTypes.Type(value = DefaultChunkKeyEncoding.class, name = "default"), @JsonSubTypes.Type(value = V2ChunkKeyEncoding.class, name = "v2") }) -public abstract class ChunkKeyEncoding { +public abstract class ChunkKeyEncoding implements dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding { public abstract String[] encodeChunkKey(long[] chunkCoords); diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java index f4d91173..52b7ea6d 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/DefaultChunkKeyEncoding.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; + import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java index 3ac4f59e..d72b429b 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java +++ b/src/main/java/dev/zarr/zarrjava/v3/chunkkeyencoding/V2ChunkKeyEncoding.java @@ -2,6 +2,8 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; + import java.util.Arrays; import java.util.stream.Collectors; import java.util.stream.Stream; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index e452db99..6c5268fd 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -125,7 +125,7 @@ public CodecBuilder withSharding(int[] chunkShape, public CodecBuilder withSharding(int[] chunkShape, Function codecBuilder, String indexLocation) { - CodecBuilder nestedBuilder = new CodecBuilder(dataType); + CodecBuilder nestedBuilder = new CodecBuilder((DataType) dataType); try { codecs.add(new ShardingIndexedCodec( new ShardingIndexedCodec.Configuration(chunkShape, diff --git a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java index 054d250c..c5950054 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java @@ -183,7 +183,7 @@ public void testReadFromZarrPythonV2(String compressor, String compressorParam) //for expected values see zarr_python_write.py Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.INT32, array.metadata.dataType); + Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.INT32, array.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, array.metadata.chunkShape()); // Assertions.assertEquals(42, array.metadata.attributes.get("answer")); @@ -208,7 +208,7 @@ public void testWriteReadWithZarrPythonV2(String compressor, String compressorPa dev.zarr.zarrjava.v2.ArrayMetadataBuilder builder = dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(16, 16, 16) - .withDataType(DataType.UINT32) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT32) .withChunks(2, 4, 8) // .withAttributes(attributes) .withFillValue(0); @@ -235,7 +235,7 @@ public void testWriteReadWithZarrPythonV2(String compressor, String compressorPa ucar.ma2.Array result = readArray.read(); Assertions.assertArrayEquals(new int[]{16, 16, 16}, result.getShape()); - Assertions.assertEquals(DataType.UINT32, readArray.metadata.dataType); + Assertions.assertEquals(dev.zarr.zarrjava.v2.DataType.UINT32, readArray.metadata.dataType); Assertions.assertArrayEquals(new int[]{2, 4, 8}, readArray.metadata.chunkShape()); // Assertions.assertEquals("test_value", readArray.metadata.attributes.get("test_key")); diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 43953a28..33c28f98 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -528,7 +528,7 @@ public void testV2createBlosc(String cname, String shuffle, int clevel) throws I new FilesystemStore(TESTOUTPUT).resolve("v2_create_blosc", cname + "_" + shuffle + "_" + clevel), dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(10, 10) - .withDataType(DataType.UINT8) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) .withChunks(5, 5) .withFillValue(1) .withBloscCompressor(cname, shuffle, clevel) @@ -537,13 +537,13 @@ public void testV2createBlosc(String cname, String shuffle, int clevel) throws I array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{8, 8})); ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(outArray.getSize(), 8 * 8); - Assertions.assertEquals(outArray.getByte(0), 0); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); } @Test public void testV2create() throws IOException, ZarrException { - DataType dataType = DataType.UINT32; + dev.zarr.zarrjava.v2.DataType dataType = dev.zarr.zarrjava.v2.DataType.UINT32; dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( new FilesystemStore(TESTOUTPUT).resolve("v2_create"), @@ -557,8 +557,8 @@ public void testV2create() throws IOException, ZarrException { array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8})); ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8}); - Assertions.assertEquals(outArray.getSize(), 8 * 8); - Assertions.assertEquals(outArray.getByte(0), 0); + Assertions.assertEquals(8 * 8, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); } @ParameterizedTest @@ -568,7 +568,7 @@ public void testV2createZlib(int level) throws IOException, ZarrException { new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)), dev.zarr.zarrjava.v2.Array.metadataBuilder() .withShape(15, 10) - .withDataType(DataType.UINT8) + .withDataType(dev.zarr.zarrjava.v2.DataType.UINT8) .withChunks(4, 5) .withFillValue(5) .withZlibCompressor(level) @@ -577,7 +577,7 @@ public void testV2createZlib(int level) throws IOException, ZarrException { array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6})); ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6}); - Assertions.assertEquals(outArray.getSize(), 7 * 6); - Assertions.assertEquals(outArray.getByte(0), 0); + Assertions.assertEquals(7 * 6, outArray.getSize()); + Assertions.assertEquals(0, outArray.getByte(0)); } } From 01206215f26b1891357fb4aaec60eafeb1ef2846 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 17 Sep 2025 17:35:23 +0200 Subject: [PATCH 13/27] add zarrita to ci while not removed from tests --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5314ef9d..a9882cff 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,7 @@ jobs: run: | uv venv && uv init uv add zarr + uv add zarrita - name: Download testdata run: | From 49b3fb8b5658a3c44c4c77f30e847e230468160b Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 11:02:16 +0200 Subject: [PATCH 14/27] fix zarr-python tests on Windows? --- src/test/python-scripts/zarr_python_read.py | 5 ++--- src/test/python-scripts/zarr_python_read_v2.py | 5 ++--- src/test/python-scripts/zarr_python_write.py | 3 +-- src/test/python-scripts/zarr_python_write_v2.py | 3 +-- 4 files changed, 6 insertions(+), 10 deletions(-) diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py index 1e31ff52..39a62567 100644 --- a/src/test/python-scripts/zarr_python_read.py +++ b/src/test/python-scripts/zarr_python_read.py @@ -4,7 +4,6 @@ import numpy as np import zarr -from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -14,12 +13,12 @@ expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -a = zarr.open_array(store=LocalStore(store_path)) +a = zarr.open_array(store=zarr.storage._local.LocalStore(store_path)) read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" b = zarr.create_array( - LocalStore(store_path / "expected"), + zarr.storage._local.LocalStore(store_path / "expected"), shape=(16, 16, 16), chunks=(2, 4, 8), dtype="uint32", diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 5390a231..607acf65 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -5,7 +5,6 @@ import zarr -from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -15,12 +14,12 @@ expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -a = zarr.open_array(store=LocalStore(store_path)) +a = zarr.open_array(store=zarr.storage._local.LocalStore(store_path)) read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" b = zarr.create_array( - LocalStore(store_path / "expected"), + zarr.storage._local.LocalStore(store_path / "expected"), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py index 531d356d..c0b298a9 100644 --- a/src/test/python-scripts/zarr_python_write.py +++ b/src/test/python-scripts/zarr_python_write.py @@ -4,7 +4,6 @@ import numpy as np import zarr -from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -15,7 +14,7 @@ testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( - LocalStore(store_path), + zarr.storage._local.LocalStore(store_path), shape=(16, 16, 16), chunks=(2, 4, 8), dtype='int32', diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py index 9c124c12..97c17714 100644 --- a/src/test/python-scripts/zarr_python_write_v2.py +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -5,7 +5,6 @@ import zarr -from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -16,7 +15,7 @@ testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( - LocalStore(store_path), + zarr.storage._local.LocalStore(store_path), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), From 710c45d76859eb1418d16c970d17d41d67709a2b Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 13:50:21 +0200 Subject: [PATCH 15/27] debug windows CI --- .github/workflows/ci.yml | 12 +++++++++++- src/test/python-scripts/parse_codecs.py | 1 + 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a9882cff..c251e886 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -36,7 +36,17 @@ jobs: uv venv && uv init uv add zarr uv add zarrita - + uv add numcodecs + - name: Debug Python env + continue-on-error: true + run: | + uv run python -c "import zarr; print(zarr.__version__)" + uv run python -c "from zarr.codecs.blosc import BloscCodec; print('BloscCodec OK')" + - name: Debug zarr-python + continue-on-error: true + run: | + uv add "zarr[blosc]" + uv run python -c "from zarr.codecs.blosc import BloscCodec; print('BloscCodec OK')" - name: Download testdata run: | mkdir testoutput diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index d7207064..39e04ccb 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -1,4 +1,5 @@ import zarr +print("Using zarr version:", zarr.__version__) from zarr.codecs.blosc import BloscCodec from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec From 5b847ce6e8f65ff9f3de150f1b25d398b32692c4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 13:56:04 +0200 Subject: [PATCH 16/27] uv add "zarr>=3.1.2" --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c251e886..a34a0df9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -34,7 +34,7 @@ jobs: - name: Set up zarr-python run: | uv venv && uv init - uv add zarr + uv add "zarr>=3.1.2" uv add zarrita uv add numcodecs - name: Debug Python env From 5962f5fb08e66980d4fdcca457648d404df601c7 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 14:01:22 +0200 Subject: [PATCH 17/27] use python 3.11 --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a34a0df9..44a6dfb2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,6 +28,11 @@ jobs: distribution: 'temurin' cache: maven + - name: Set up Python 3.11 + uses: actions/setup-python@v4 + with: + python-version: '3.11' + - name: Install uv uses: astral-sh/setup-uv@v6 From fb7a972ed84cec61de12655dc597eadf7d72e4be Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 14:06:55 +0200 Subject: [PATCH 18/27] remove debug changes --- .github/workflows/ci.yml | 17 +++-------------- src/test/python-scripts/parse_codecs.py | 1 - src/test/python-scripts/zarr_python_read.py | 5 +++-- src/test/python-scripts/zarr_python_read_v2.py | 5 +++-- src/test/python-scripts/zarr_python_write.py | 3 ++- src/test/python-scripts/zarr_python_write_v2.py | 3 ++- 6 files changed, 13 insertions(+), 21 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 44a6dfb2..680b5251 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -28,8 +28,8 @@ jobs: distribution: 'temurin' cache: maven - - name: Set up Python 3.11 - uses: actions/setup-python@v4 + - name: Set up Python + uses: actions/setup-python@@v5 with: python-version: '3.11' @@ -39,19 +39,8 @@ jobs: - name: Set up zarr-python run: | uv venv && uv init - uv add "zarr>=3.1.2" + uv add zarr uv add zarrita - uv add numcodecs - - name: Debug Python env - continue-on-error: true - run: | - uv run python -c "import zarr; print(zarr.__version__)" - uv run python -c "from zarr.codecs.blosc import BloscCodec; print('BloscCodec OK')" - - name: Debug zarr-python - continue-on-error: true - run: | - uv add "zarr[blosc]" - uv run python -c "from zarr.codecs.blosc import BloscCodec; print('BloscCodec OK')" - name: Download testdata run: | mkdir testoutput diff --git a/src/test/python-scripts/parse_codecs.py b/src/test/python-scripts/parse_codecs.py index 39e04ccb..d7207064 100644 --- a/src/test/python-scripts/parse_codecs.py +++ b/src/test/python-scripts/parse_codecs.py @@ -1,5 +1,4 @@ import zarr -print("Using zarr version:", zarr.__version__) from zarr.codecs.blosc import BloscCodec from zarr.codecs.bytes import BytesCodec from zarr.codecs.crc32c_ import Crc32cCodec diff --git a/src/test/python-scripts/zarr_python_read.py b/src/test/python-scripts/zarr_python_read.py index 39a62567..1e31ff52 100644 --- a/src/test/python-scripts/zarr_python_read.py +++ b/src/test/python-scripts/zarr_python_read.py @@ -4,6 +4,7 @@ import numpy as np import zarr +from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -13,12 +14,12 @@ expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -a = zarr.open_array(store=zarr.storage._local.LocalStore(store_path)) +a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" b = zarr.create_array( - zarr.storage._local.LocalStore(store_path / "expected"), + LocalStore(store_path / "expected"), shape=(16, 16, 16), chunks=(2, 4, 8), dtype="uint32", diff --git a/src/test/python-scripts/zarr_python_read_v2.py b/src/test/python-scripts/zarr_python_read_v2.py index 607acf65..5390a231 100644 --- a/src/test/python-scripts/zarr_python_read_v2.py +++ b/src/test/python-scripts/zarr_python_read_v2.py @@ -5,6 +5,7 @@ import zarr +from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -14,12 +15,12 @@ expected_data = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) -a = zarr.open_array(store=zarr.storage._local.LocalStore(store_path)) +a = zarr.open_array(store=LocalStore(store_path)) read_data = a[:, :] assert np.array_equal(read_data, expected_data), f"got:\n {read_data} \nbut expected:\n {expected_data}" b = zarr.create_array( - zarr.storage._local.LocalStore(store_path / "expected"), + LocalStore(store_path / "expected"), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), diff --git a/src/test/python-scripts/zarr_python_write.py b/src/test/python-scripts/zarr_python_write.py index c0b298a9..531d356d 100644 --- a/src/test/python-scripts/zarr_python_write.py +++ b/src/test/python-scripts/zarr_python_write.py @@ -4,6 +4,7 @@ import numpy as np import zarr +from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -14,7 +15,7 @@ testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( - zarr.storage._local.LocalStore(store_path), + LocalStore(store_path), shape=(16, 16, 16), chunks=(2, 4, 8), dtype='int32', diff --git a/src/test/python-scripts/zarr_python_write_v2.py b/src/test/python-scripts/zarr_python_write_v2.py index 97c17714..9c124c12 100644 --- a/src/test/python-scripts/zarr_python_write_v2.py +++ b/src/test/python-scripts/zarr_python_write_v2.py @@ -5,6 +5,7 @@ import zarr +from zarr.storage import LocalStore from parse_codecs import parse_codecs_zarr_python codec_string = sys.argv[1] @@ -15,7 +16,7 @@ testdata = np.arange(16 * 16 * 16, dtype='int32').reshape(16, 16, 16) a = zarr.create_array( - zarr.storage._local.LocalStore(store_path), + LocalStore(store_path), zarr_format=2, shape=(16, 16, 16), chunks=(2, 4, 8), From 8124a5bdd8b20419b70c6b00b6e2e69e974b6dc1 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 14:22:21 +0200 Subject: [PATCH 19/27] remove v3 code in v2 --- .../zarrjava/core/codec/core/BytesCodec.java | 11 +++ src/main/java/dev/zarr/zarrjava/v2/Array.java | 2 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 4 +- .../java/dev/zarr/zarrjava/v2/DataType.java | 2 + .../java/dev/zarr/zarrjava/v2/Endianness.java | 2 +- .../dev/zarr/zarrjava/v2/codec/Codec.java | 2 +- .../zarrjava/v2/codec/core/BloscCodec.java | 2 +- .../zarrjava/v2/codec/core/BytesCodec.java | 72 +++++++++++++++++++ .../zarrjava/v3/codec/core/BytesCodec.java | 2 +- 9 files changed, 92 insertions(+), 7 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java create mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java new file mode 100644 index 00000000..f91d72f8 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -0,0 +1,11 @@ +package dev.zarr.zarrjava.core.codec.core; + +import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; + +public interface BytesCodec extends ArrayBytesCodec { + + /* + TODO: better style would be to make this a class and dev.zarr.zarrjava.v3.codec.Codec an interface + Would this work with the serialization? + */ +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 7ee7ee11..03d4a2d9 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -8,7 +8,7 @@ import dev.zarr.zarrjava.core.codec.CodecPipeline; import dev.zarr.zarrjava.v2.codec.Codec; import dev.zarr.zarrjava.v2.codec.CodecRegistry; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v2.codec.core.BytesCodec; import javax.annotation.Nonnull; import java.io.IOException; diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index edfb3a7c..18f52210 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -47,7 +47,7 @@ public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { public Codec compressor; @JsonIgnore - public dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata coreArrayMetadata; + public CoreArrayMetadata coreArrayMetadata; @JsonCreator(mode = JsonCreator.Mode.PROPERTIES) @@ -78,7 +78,7 @@ public ArrayMetadata( this.filters = filters; this.compressor = compressor; this.coreArrayMetadata = - new dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata(shape, chunks, + new ArrayMetadata.CoreArrayMetadata(shape, chunks, this.dataType, parsedFillValue ); diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java index b1c502cb..46811c6b 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -25,6 +25,7 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { this.endianness = endianness; } + //todo remove? public static DataType fromDataType(dev.zarr.zarrjava.v3.DataType dataType) { switch (dataType) { case BOOL: @@ -58,6 +59,7 @@ public Endianness getEndianness() { return endianness; } + //todo remove? public dev.zarr.zarrjava.v3.DataType toV3() throws ZarrException { if (this.dtype.equals(BOOL.dtype)) return dev.zarr.zarrjava.v3.DataType.BOOL; diff --git a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java index 9c01c673..1f064cb6 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Endianness.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Endianness.java @@ -1,7 +1,7 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.annotation.JsonValue; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec; +import dev.zarr.zarrjava.v2.codec.core.BytesCodec; public enum Endianness { LITTLE("<"), diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index 58cec409..c0199597 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -2,7 +2,7 @@ import com.fasterxml.jackson.annotation.JsonTypeInfo; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v2.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") public abstract class Codec implements dev.zarr.zarrjava.core.codec.Codec { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 4f615c94..d39ab996 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -14,7 +14,7 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.v2.ArrayMetadata; import javax.annotation.Nonnull; import java.io.IOException; diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java new file mode 100644 index 00000000..34cb9885 --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -0,0 +1,72 @@ +package dev.zarr.zarrjava.v2.codec.core; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonValue; +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v2.ArrayMetadata; +import dev.zarr.zarrjava.v2.codec.Codec; +import ucar.ma2.Array; + +import javax.annotation.Nonnull; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public class BytesCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BytesCodec { + @Nonnull + public final Endian endian; + + public final String name = "bytes"; + + @JsonCreator + public BytesCodec( + @JsonProperty(value = "endian", defaultValue = "little") Endian endian) { + this.endian = endian; + } + + + @Override + public Array decode(ByteBuffer chunkBytes) { + chunkBytes.order(endian.getByteOrder()); + return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, + chunkBytes); + } + + @Override + public ByteBuffer encode(Array chunkArray) { + return chunkArray.getDataAsByteBuffer(endian.getByteOrder()); + } + + @Override + public long computeEncodedSize(long inputByteLength, + ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { + return inputByteLength; + } + + public enum Endian { + LITTLE("little"), + BIG("big"); + private final String endian; + + Endian(String endian) { + this.endian = endian; + } + + @JsonValue + public String getValue() { + return endian; + } + + public ByteOrder getByteOrder() { + switch (this) { + case LITTLE: + return ByteOrder.LITTLE_ENDIAN; + case BIG: + return ByteOrder.BIG_ENDIAN; + default: + throw new RuntimeException("Unreachable"); + } + } + } +} + diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index f655eb74..c8d1be98 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -12,7 +12,7 @@ import javax.annotation.Nonnull; import ucar.ma2.Array; -public class BytesCodec extends Codec implements ArrayBytesCodec { +public class BytesCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BytesCodec { public final String name = "bytes"; @Nonnull From 56644d24a063cc5e4a7e3800efa2ee95fbfd46b9 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 15:31:32 +0200 Subject: [PATCH 20/27] change multi-inheritance: - codecs in core are abstrac classes - parent codec per v2/v3 (used for serialization) is interface --- .../zarrjava/core/codec/AbstractCodec.java | 20 ++++++++++++++++++ .../zarrjava/core/codec/ArrayArrayCodec.java | 6 +++--- .../zarrjava/core/codec/ArrayBytesCodec.java | 14 ++++++------- .../zarrjava/core/codec/BytesBytesCodec.java | 8 +++---- .../dev/zarr/zarrjava/core/codec/Codec.java | 12 +++++------ .../zarrjava/core/codec/core/BloscCodec.java | 8 +++---- .../zarrjava/core/codec/core/BytesCodec.java | 8 +------ src/main/java/dev/zarr/zarrjava/v2/Array.java | 4 ++-- .../dev/zarr/zarrjava/v2/codec/Codec.java | 21 +------------------ .../zarrjava/v2/codec/core/BloscCodec.java | 9 +------- .../zarrjava/v2/codec/core/BytesCodec.java | 11 ++-------- .../zarrjava/v2/codec/core/ZlibCodec.java | 8 +------ .../dev/zarr/zarrjava/v3/codec/Codec.java | 16 ++------------ .../zarr/zarrjava/v3/codec/CodecRegistry.java | 1 - .../zarrjava/v3/codec/core/BloscCodec.java | 2 +- .../zarrjava/v3/codec/core/BytesCodec.java | 2 +- .../zarrjava/v3/codec/core/Crc32cCodec.java | 2 +- .../zarrjava/v3/codec/core/GzipCodec.java | 2 +- .../v3/codec/core/ShardingIndexedCodec.java | 2 +- .../v3/codec/core/TransposeCodec.java | 2 +- .../zarrjava/v3/codec/core/ZstdCodec.java | 2 +- 21 files changed, 61 insertions(+), 99 deletions(-) create mode 100644 src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java new file mode 100644 index 00000000..35e1434b --- /dev/null +++ b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java @@ -0,0 +1,20 @@ +package dev.zarr.zarrjava.core.codec; + +import dev.zarr.zarrjava.ZarrException; +import dev.zarr.zarrjava.v3.ArrayMetadata; //TODO remove v3 dependency + +public abstract class AbstractCodec implements Codec{ + protected ArrayMetadata.CoreArrayMetadata arrayMetadata; + + public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { + if (arrayMetadata == null) { + throw new ZarrException("arrayMetadata needs to get set in for every codec"); + } + return this.arrayMetadata; + } + + public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + this.arrayMetadata = arrayMetadata; + } +} + diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java index 9acf1dde..be7a7bd4 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java @@ -3,12 +3,12 @@ import dev.zarr.zarrjava.ZarrException; import ucar.ma2.Array; -public interface ArrayArrayCodec { +public abstract class ArrayArrayCodec extends AbstractCodec { - Array encode(Array chunkArray) + public abstract Array encode(Array chunkArray) throws ZarrException; - Array decode(Array chunkArray) + public abstract Array decode(Array chunkArray) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java index ada36fff..cf4ab64d 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java @@ -5,22 +5,22 @@ import java.nio.ByteBuffer; import ucar.ma2.Array; -public interface ArrayBytesCodec { +public abstract class ArrayBytesCodec extends AbstractCodec { - ByteBuffer encode(Array chunkArray) + public abstract ByteBuffer encode(Array chunkArray) throws ZarrException; - Array decode(ByteBuffer chunkBytes) + public abstract Array decode(ByteBuffer chunkBytes) throws ZarrException; - interface WithPartialDecode extends ArrayBytesCodec { + public abstract static class WithPartialDecode extends ArrayBytesCodec { public abstract Array decode(ByteBuffer shardBytes) throws ZarrException; public abstract ByteBuffer encode(Array shardArray) throws ZarrException; - Array decodePartial( - StoreHandle handle, long[] offset, int[] shape - ) throws ZarrException; + protected abstract Array decodePartial( + StoreHandle handle, long[] offset, int[] shape + ) throws ZarrException; } } diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java index 79a2271e..b7c448fa 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java @@ -7,13 +7,13 @@ import java.io.OutputStream; import java.nio.ByteBuffer; -public interface BytesBytesCodec { +public abstract class BytesBytesCodec extends AbstractCodec { - ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; + public abstract ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException; - ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; + public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; - default void copy(InputStream inputStream, OutputStream outputStream) throws IOException { + protected void copy(InputStream inputStream, OutputStream outputStream) throws IOException { byte[] buffer = new byte[4096]; int len; while ((len = inputStream.read(buffer)) > 0) { diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java index 89e52fdd..04bab9ac 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/Codec.java @@ -1,14 +1,14 @@ package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata; +import dev.zarr.zarrjava.core.ArrayMetadata; public interface Codec { - ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException; + void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata codecArrayMetadata) throws ZarrException; - long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException; + ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException; - void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; + default long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ + throw new ZarrException("Not implemented for " + this.getClass()); + } } - diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java index b233a06d..bdbe3f16 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java @@ -15,10 +15,10 @@ import java.io.IOException; import java.nio.ByteBuffer; -public interface BloscCodec extends BytesBytesCodec { +public abstract class BloscCodec extends BytesBytesCodec { @Override - default ByteBuffer decode(ByteBuffer chunkBytes) + public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try { return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes))); @@ -27,7 +27,7 @@ default ByteBuffer decode(ByteBuffer chunkBytes) } } - final class CustomCompressorDeserializer extends StdDeserializer { + public static final class CustomCompressorDeserializer extends StdDeserializer { public CustomCompressorDeserializer() { this(null); @@ -53,7 +53,7 @@ public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContex } } - final class CustomCompressorSerializer extends StdSerializer { + public static final class CustomCompressorSerializer extends StdSerializer { public CustomCompressorSerializer() { super(Blosc.Compressor.class); diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index f91d72f8..8642e8c8 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -2,10 +2,4 @@ import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; -public interface BytesCodec extends ArrayBytesCodec { - - /* - TODO: better style would be to make this a class and dev.zarr.zarrjava.v3.codec.Codec an interface - Would this work with the serialization? - */ -} +public abstract class BytesCodec extends ArrayBytesCodec {} diff --git a/src/main/java/dev/zarr/zarrjava/v2/Array.java b/src/main/java/dev/zarr/zarrjava/v2/Array.java index 03d4a2d9..52d22688 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ b/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -28,9 +28,9 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE this.storeHandle = storeHandle; this.metadata = arrayMetadata; this.codecPipeline = new CodecPipeline(Utils.concatArrays( - new dev.zarr.zarrjava.core.codec.Codec[]{}, + new Codec[]{}, metadata.filters == null ? new Codec[]{} : metadata.filters, - new dev.zarr.zarrjava.core.codec.Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, + new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())}, metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor} ), metadata.coreArrayMetadata); } diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java index c0199597..43f02f11 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/Codec.java @@ -1,26 +1,7 @@ package dev.zarr.zarrjava.v2.codec; import com.fasterxml.jackson.annotation.JsonTypeInfo; -import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v2.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "id") -public abstract class Codec implements dev.zarr.zarrjava.core.codec.Codec { - - protected ArrayMetadata.CoreArrayMetadata arrayMetadata; - - public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { - if (arrayMetadata == null) { - throw new ZarrException("arrayMetadata needs to get set in for every codec"); - } - return this.arrayMetadata; - } - - public abstract long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) - throws ZarrException; - - public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ - this.arrayMetadata = arrayMetadata; - } -} +public interface Codec extends dev.zarr.zarrjava.core.codec.Codec {} diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index d39ab996..9a9b8bdc 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -14,13 +14,12 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v2.ArrayMetadata; import javax.annotation.Nonnull; import java.io.IOException; import java.nio.ByteBuffer; -public class BloscCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BloscCodec { +public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec { public final String id = "blosc"; @@ -74,12 +73,6 @@ public ByteBuffer encode(ByteBuffer chunkBytes) } } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Blosc codec."); - } - public static final class CustomShuffleSerializer extends StdSerializer { public CustomShuffleSerializer() { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index 34cb9885..c9719e54 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -12,7 +12,8 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class BytesCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BytesCodec { +// TODO might move some of this to core +public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { @Nonnull public final Endian endian; @@ -24,8 +25,6 @@ public BytesCodec( this.endian = endian; } - - @Override public Array decode(ByteBuffer chunkBytes) { chunkBytes.order(endian.getByteOrder()); return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, @@ -37,12 +36,6 @@ public ByteBuffer encode(Array chunkArray) { return chunkArray.getDataAsByteBuffer(endian.getByteOrder()); } - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - return inputByteLength; - } - public enum Endian { LITTLE("little"), BIG("big"); diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index 5497cb62..f7124ac9 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -14,7 +14,7 @@ import java.nio.ByteBuffer; import java.util.zip.*; -public class ZlibCodec extends Codec implements BytesBytesCodec { +public class ZlibCodec extends BytesBytesCodec implements Codec { public final String id = "zlib"; public final int level; @@ -53,10 +53,4 @@ public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException { throw new ZarrException("Error in encoding zlib.", ex); } } - - @Override - public long computeEncodedSize(long inputByteLength, - ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { - throw new ZarrException("Not implemented for Zlib codec."); - } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java index 439e872d..d9631a1e 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/Codec.java @@ -5,19 +5,7 @@ import dev.zarr.zarrjava.v3.ArrayMetadata; @JsonTypeInfo(use = JsonTypeInfo.Id.NAME, property = "name") -public abstract class Codec implements dev.zarr.zarrjava.core.codec.Codec { - - protected ArrayMetadata.CoreArrayMetadata arrayMetadata; - - public ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException { - if (arrayMetadata == null) { - throw new ZarrException("arrayMetadata needs to get set in for every codec"); - } - return this.arrayMetadata; - } - - public void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException{ - this.arrayMetadata = arrayMetadata; - } +public interface Codec extends dev.zarr.zarrjava.v2.codec.Codec { + long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException; } diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java index a02fc621..16b52a12 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java @@ -2,7 +2,6 @@ import com.fasterxml.jackson.databind.jsontype.NamedType; import dev.zarr.zarrjava.v3.codec.core.*; -import dev.zarr.zarrjava.core.codec.Codec; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java index f9d70f8a..caa385fa 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BloscCodec.java @@ -21,7 +21,7 @@ import java.nio.ByteBuffer; import javax.annotation.Nonnull; -public class BloscCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BloscCodec { +public class BloscCodec extends dev.zarr.zarrjava.core.codec.core.BloscCodec implements Codec{ public final String name = "blosc"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index c8d1be98..a28c0ea0 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -12,7 +12,7 @@ import javax.annotation.Nonnull; import ucar.ma2.Array; -public class BytesCodec extends Codec implements dev.zarr.zarrjava.core.codec.core.BytesCodec { +public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { public final String name = "bytes"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java index 85e8bf79..823106d7 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/Crc32cCodec.java @@ -10,7 +10,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; -public class Crc32cCodec extends Codec implements BytesBytesCodec { +public class Crc32cCodec extends BytesBytesCodec implements Codec { public final String name = "crc32c"; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index bea531f6..e92bfcab 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -15,7 +15,7 @@ import java.util.zip.GZIPOutputStream; import javax.annotation.Nonnull; -public class GzipCodec extends Codec implements BytesBytesCodec { +public class GzipCodec extends BytesBytesCodec implements Codec { public final String name = "gzip"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java index 3b71bf77..dd30a930 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ShardingIndexedCodec.java @@ -22,7 +22,7 @@ import ucar.ma2.InvalidRangeException; -public class ShardingIndexedCodec extends Codec implements ArrayBytesCodec.WithPartialDecode { +public class ShardingIndexedCodec extends ArrayBytesCodec.WithPartialDecode implements Codec { public final String name = "sharding_indexed"; @Nonnull diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java index 9127968a..5eebefeb 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/TransposeCodec.java @@ -14,7 +14,7 @@ import static dev.zarr.zarrjava.utils.Utils.inversePermutation; import static dev.zarr.zarrjava.utils.Utils.isPermutation; -public class TransposeCodec extends Codec implements ArrayArrayCodec { +public class TransposeCodec extends ArrayArrayCodec implements Codec{ @Nonnull public final String name = "transpose"; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java index 5cede681..b79b1ac9 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/ZstdCodec.java @@ -12,7 +12,7 @@ import javax.annotation.Nonnull; import java.nio.ByteBuffer; -public class ZstdCodec extends Codec implements BytesBytesCodec { +public class ZstdCodec extends BytesBytesCodec implements Codec { public final String name = "zstd"; @Nonnull From bb06e80af145f36f3fa78c2636a5e2d12f0df237 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 15:49:48 +0200 Subject: [PATCH 21/27] refactor code duplication --- .github/workflows/ci.yml | 3 +- .../zarrjava/core/codec/AbstractCodec.java | 2 +- .../zarrjava/core/codec/core/BytesCodec.java | 48 ++++++++++++++- .../java/dev/zarr/zarrjava/v2/DataType.java | 58 ------------------- .../zarrjava/v2/codec/core/BytesCodec.java | 45 +------------- .../zarrjava/v3/ArrayMetadataBuilder.java | 2 +- .../zarr/zarrjava/v3/codec/CodecBuilder.java | 2 +- .../zarrjava/v3/codec/core/BytesCodec.java | 45 ++------------ 8 files changed, 60 insertions(+), 145 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 680b5251..c0e4c099 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -29,7 +29,7 @@ jobs: cache: maven - name: Set up Python - uses: actions/setup-python@@v5 + uses: actions/setup-python@v5 with: python-version: '3.11' @@ -41,6 +41,7 @@ jobs: uv venv && uv init uv add zarr uv add zarrita + - name: Download testdata run: | mkdir testoutput diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java index 35e1434b..d4c634f2 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/AbstractCodec.java @@ -1,7 +1,7 @@ package dev.zarr.zarrjava.core.codec; import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v3.ArrayMetadata; //TODO remove v3 dependency +import dev.zarr.zarrjava.core.ArrayMetadata; public abstract class AbstractCodec implements Codec{ protected ArrayMetadata.CoreArrayMetadata arrayMetadata; diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java index 8642e8c8..7c70b54d 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java @@ -1,5 +1,51 @@ package dev.zarr.zarrjava.core.codec.core; +import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; +import ucar.ma2.Array; -public abstract class BytesCodec extends ArrayBytesCodec {} +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +public abstract class BytesCodec extends ArrayBytesCodec { + protected abstract ByteOrder getByteOrder(); + + @Override + public Array decode(ByteBuffer chunkBytes) { + chunkBytes.order(getByteOrder()); + return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, + chunkBytes); + } + + @Override + public ByteBuffer encode(Array chunkArray) { + return chunkArray.getDataAsByteBuffer(getByteOrder()); + } + + public enum Endian { + LITTLE("little"), + BIG("big"); + private final String endian; + + Endian(String endian) { + this.endian = endian; + } + + @JsonValue + public String getValue() { + return endian; + } + + public ByteOrder getByteOrder() { + switch (this) { + case LITTLE: + return ByteOrder.LITTLE_ENDIAN; + case BIG: + return ByteOrder.BIG_ENDIAN; + default: + throw new RuntimeException("Unreachable"); + } + } + } + +} diff --git a/src/main/java/dev/zarr/zarrjava/v2/DataType.java b/src/main/java/dev/zarr/zarrjava/v2/DataType.java index 46811c6b..45834874 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/DataType.java +++ b/src/main/java/dev/zarr/zarrjava/v2/DataType.java @@ -1,7 +1,6 @@ package dev.zarr.zarrjava.v2; import com.fasterxml.jackson.annotation.JsonValue; -import dev.zarr.zarrjava.ZarrException; public enum DataType implements dev.zarr.zarrjava.core.DataType { BOOL("b1", Endianness.UNSPECIFIED), @@ -25,67 +24,10 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType { this.endianness = endianness; } - //todo remove? - public static DataType fromDataType(dev.zarr.zarrjava.v3.DataType dataType) { - switch (dataType) { - case BOOL: - return BOOL; - case INT8: - return INT8; - case INT16: - return INT16; - case INT32: - return INT32; - case INT64: - return INT64; - case UINT8: - return UINT8; - case UINT16: - return UINT16; - case UINT32: - return UINT32; - case UINT64: - return UINT64; - case FLOAT32: - return FLOAT32; - case FLOAT64: - return FLOAT64; - default: - throw new IllegalArgumentException("Unsupported DataType: " + dataType); - } - } - public Endianness getEndianness() { return endianness; } - //todo remove? - public dev.zarr.zarrjava.v3.DataType toV3() throws ZarrException { - if (this.dtype.equals(BOOL.dtype)) - return dev.zarr.zarrjava.v3.DataType.BOOL; - if (this.dtype.equals(INT8.dtype)) - return dev.zarr.zarrjava.v3.DataType.INT8; - if (this.dtype.equals(INT16.dtype)) - return dev.zarr.zarrjava.v3.DataType.INT16; - if (this.dtype.equals(INT32.dtype)) - return dev.zarr.zarrjava.v3.DataType.INT32; - if (this.dtype.equals(INT64.dtype)) - return dev.zarr.zarrjava.v3.DataType.INT64; - if (this.dtype.equals(UINT8.dtype)) - return dev.zarr.zarrjava.v3.DataType.UINT8; - if (this.dtype.equals(UINT16.dtype)) - return dev.zarr.zarrjava.v3.DataType.UINT16; - if (this.dtype.equals(UINT32.dtype)) - return dev.zarr.zarrjava.v3.DataType.UINT32; - if (this.dtype.equals(UINT64.dtype)) - return dev.zarr.zarrjava.v3.DataType.UINT64; - if (this.dtype.equals(FLOAT32.dtype)) - return dev.zarr.zarrjava.v3.DataType.FLOAT32; - if (this.dtype.equals(FLOAT64.dtype)) - return dev.zarr.zarrjava.v3.DataType.FLOAT64; - throw new ZarrException("Unknown DataTypeV2: " + this.dtype); - } - @JsonValue public String getValue() { return String.format("%s%s", endianness.getValue(), dtype); diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java index c9719e54..ac0cd39a 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BytesCodec.java @@ -2,64 +2,25 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; -import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v2.ArrayMetadata; import dev.zarr.zarrjava.v2.codec.Codec; -import ucar.ma2.Array; import javax.annotation.Nonnull; -import java.nio.ByteBuffer; import java.nio.ByteOrder; -// TODO might move some of this to core public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { + public final String name = "bytes"; @Nonnull public final Endian endian; - public final String name = "bytes"; - @JsonCreator public BytesCodec( @JsonProperty(value = "endian", defaultValue = "little") Endian endian) { this.endian = endian; } - public Array decode(ByteBuffer chunkBytes) { - chunkBytes.order(endian.getByteOrder()); - return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, - chunkBytes); - } - @Override - public ByteBuffer encode(Array chunkArray) { - return chunkArray.getDataAsByteBuffer(endian.getByteOrder()); - } - - public enum Endian { - LITTLE("little"), - BIG("big"); - private final String endian; - - Endian(String endian) { - this.endian = endian; - } - - @JsonValue - public String getValue() { - return endian; - } - - public ByteOrder getByteOrder() { - switch (this) { - case LITTLE: - return ByteOrder.LITTLE_ENDIAN; - case BIG: - return ByteOrder.BIG_ENDIAN; - default: - throw new RuntimeException("Unreachable"); - } - } + protected ByteOrder getByteOrder() { + return endian.getByteOrder(); } } diff --git a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java index cdfe90cd..212f4732 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/ArrayMetadataBuilder.java @@ -10,7 +10,7 @@ import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.codec.CodecBuilder; import dev.zarr.zarrjava.v3.codec.core.BytesCodec; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Endian; +import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import java.util.HashMap; import java.util.Map; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java index 6c5268fd..ad2f70c4 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java @@ -8,7 +8,7 @@ import dev.zarr.zarrjava.v3.DataType; import dev.zarr.zarrjava.v3.codec.core.*; import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration; -import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Endian; +import dev.zarr.zarrjava.core.codec.core.BytesCodec.Endian; import java.util.ArrayList; import java.util.Collections; diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java index a28c0ea0..311e9f56 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/BytesCodec.java @@ -2,15 +2,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonValue; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.v3.codec.Codec; import dev.zarr.zarrjava.v3.ArrayMetadata; -import dev.zarr.zarrjava.core.codec.ArrayBytesCodec; -import java.nio.ByteBuffer; + import java.nio.ByteOrder; import javax.annotation.Nonnull; -import ucar.ma2.Array; public class BytesCodec extends dev.zarr.zarrjava.core.codec.core.BytesCodec implements Codec { @@ -29,50 +26,18 @@ public BytesCodec(Endian endian) { this(new BytesCodec.Configuration(endian)); } - @Override - public Array decode(ByteBuffer chunkBytes) { - chunkBytes.order(configuration.endian.getByteOrder()); - return Array.factory(arrayMetadata.dataType.getMA2DataType(), arrayMetadata.chunkShape, - chunkBytes); - } - - @Override - public ByteBuffer encode(Array chunkArray) { - return chunkArray.getDataAsByteBuffer(configuration.endian.getByteOrder()); - } - @Override public long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException { return inputByteLength; } - public enum Endian { - LITTLE("little"), - BIG("big"); - private final String endian; - - Endian(String endian) { - this.endian = endian; - } - - @JsonValue - public String getValue() { - return endian; - } - - public ByteOrder getByteOrder() { - switch (this) { - case LITTLE: - return ByteOrder.LITTLE_ENDIAN; - case BIG: - return ByteOrder.BIG_ENDIAN; - default: - throw new RuntimeException("Unreachable"); - } - } + @Override + protected ByteOrder getByteOrder() { + return configuration.endian.getByteOrder(); } + public static final class Configuration{ @Nonnull From 86e8f6ca48a821de6653c8d1ae2188c12596ace5 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 15:53:58 +0200 Subject: [PATCH 22/27] update ci actions versions --- .github/workflows/ci.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c0e4c099..aadf66c7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: shell: bash steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 - name: Set up JDK uses: actions/setup-java@v4 @@ -29,9 +29,7 @@ jobs: cache: maven - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '3.11' + uses: actions/setup-python@v6 - name: Install uv uses: astral-sh/setup-uv@v6 From e5222ce6ba377a18c04de822d6ea82fc3b15b4ef Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 16:19:48 +0200 Subject: [PATCH 23/27] allow fillValue null --- .../java/dev/zarr/zarrjava/core/Array.java | 3 +- .../dev/zarr/zarrjava/v2/ArrayMetadata.java | 17 +++++++---- .../zarrjava/v2/ArrayMetadataBuilder.java | 6 ++-- src/test/java/dev/zarr/zarrjava/ZarrTest.java | 28 +++++++++++++++++++ 4 files changed, 44 insertions(+), 10 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index 23bc4084..aaf21873 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -85,8 +85,9 @@ default void writeChunk(long[] chunkCoords, ucar.ma2.Array chunkArray) throws Za ArrayMetadata metadata = metadata(); String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords); StoreHandle chunkHandle = storeHandle().resolve(chunkKeys); + Object parsedFillValue = metadata.parsedFillValue(); - if (MultiArrayUtils.allValuesEqual(chunkArray, metadata.parsedFillValue())) { + if (parsedFillValue != null && MultiArrayUtils.allValuesEqual(chunkArray, parsedFillValue)) { chunkHandle.delete(); } else { ByteBuffer chunkBytes = codecPipeline().encode(chunkArray); diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java index 18f52210..d0fa0e1d 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java @@ -42,6 +42,7 @@ public class ArrayMetadata implements dev.zarr.zarrjava.core.ArrayMetadata { @JsonIgnore public final Object parsedFillValue; + @Nullable public Codec[] filters; @Nullable public Codec compressor; @@ -56,11 +57,11 @@ public ArrayMetadata( @JsonProperty(value = "shape", required = true) long[] shape, @JsonProperty(value = "chunks", required = true) int[] chunks, @JsonProperty(value = "dtype", required = true) DataType dataType, - @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo test when null + @Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, @JsonProperty(value = "order", required = true) Order order, - @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator, - @Nullable @JsonProperty(value = "filters") Codec[] filters, - @Nullable @JsonProperty(value = "compressor") Codec compressor + @Nullable @JsonProperty(value = "filters", required = true) Codec[] filters, + @Nullable @JsonProperty(value = "compressor", required = true) Codec compressor, + @Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator ) throws ZarrException { super(); if (zarrFormat != this.zarrFormat) { @@ -72,7 +73,11 @@ public ArrayMetadata( this.dataType = dataType; this.endianness = dataType.getEndianness(); this.fillValue = fillValue; - this.parsedFillValue = parseFillValue(fillValue, this.dataType); + if (fillValue == null) { + this.parsedFillValue = null; + } else { + this.parsedFillValue = parseFillValue(fillValue, this.dataType); + } this.order = order; this.dimensionSeparator = dimensionSeparator; this.filters = filters; @@ -107,7 +112,7 @@ public DataType dataType() { @Override public Array allocateFillValueChunk() { ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunks); - MultiArrayUtils.fill(outputArray, parsedFillValue); + if (parsedFillValue != null) MultiArrayUtils.fill(outputArray, parsedFillValue); return outputArray; } diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 5bb59421..3e79a759 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -13,7 +13,7 @@ public class ArrayMetadataBuilder { DataType dataType = null; Order order = Order.C; Separator dimensionSeparator = Separator.DOT; - Object fillValue = 0; + Object fillValue = null; Codec[] filters = null; Codec compressor = null; @@ -115,9 +115,9 @@ public ArrayMetadata build() throws ZarrException { dataType, fillValue, order, - dimensionSeparator, filters, - compressor + compressor, + dimensionSeparator ); } } \ No newline at end of file diff --git a/src/test/java/dev/zarr/zarrjava/ZarrTest.java b/src/test/java/dev/zarr/zarrjava/ZarrTest.java index 33c28f98..647a3b3e 100644 --- a/src/test/java/dev/zarr/zarrjava/ZarrTest.java +++ b/src/test/java/dev/zarr/zarrjava/ZarrTest.java @@ -580,4 +580,32 @@ public void testV2createZlib(int level) throws IOException, ZarrException { Assertions.assertEquals(7 * 6, outArray.getSize()); Assertions.assertEquals(0, outArray.getByte(0)); } + + @ParameterizedTest + @ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"}) + public void testV2noFillValue(dev.zarr.zarrjava.v2.DataType dataType) throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name()); + + dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create( + storeHandle, + dev.zarr.zarrjava.v2.Array.metadataBuilder() + .withShape(15, 10) + .withDataType(dataType) + .withChunks(4, 5) + .build() + ); + Assertions.assertNull(array.metadata().fillValue); + + ucar.ma2.Array outArray = array.read(new long[]{0, 0}, new int[]{1, 1}); + if (dataType == dev.zarr.zarrjava.v2.DataType.BOOL) { + Assertions.assertFalse(outArray.getBoolean(0)); + } else { + Assertions.assertEquals(0, outArray.getByte(0)); + } + + dev.zarr.zarrjava.v2.Array array2 = dev.zarr.zarrjava.v2.Array.open( + storeHandle + ); + Assertions.assertNull(array2.metadata().fillValue); + } } From e8fdea34c4c46285965ed0578b39a10553180d01 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 23 Sep 2025 16:33:36 +0200 Subject: [PATCH 24/27] add ci python-version: '3.11' label --- .github/workflows/ci.yml | 2 ++ .../java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java | 5 ++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aadf66c7..759a84d8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,6 +30,8 @@ jobs: - name: Set up Python uses: actions/setup-python@v6 + with: + python-version: '3.11' - name: Install uv uses: astral-sh/setup-uv@v6 diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java index 9a9b8bdc..4838488d 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/BloscCodec.java @@ -42,11 +42,10 @@ public BloscCodec( @JsonDeserialize(using = CustomShuffleDeserializer.class) Blosc.Shuffle shuffle, @JsonProperty(value = "clevel", defaultValue = "5") int clevel, @JsonProperty(value = "typesize", defaultValue = "0") int typesize, - @JsonProperty(value = "blocksize", defaultValue = "0") - int blocksize + @JsonProperty(value = "blocksize", defaultValue = "0") int blocksize ) throws ZarrException { if (typesize < 1 && shuffle != Blosc.Shuffle.NO_SHUFFLE) { - typesize = 4; // in v2 typesize is not required. todo: deflault to correct value based on dtype + typesize = 4; //todo: in v2 typesize is not a required parameter. default to correct value based on dtype } if (clevel < 0 || clevel > 9) { throw new ZarrException("'clevel' needs to be between 0 and 9."); From 33af68831eb50c852cd948486e1f0ff294b87233 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 24 Sep 2025 12:21:41 +0200 Subject: [PATCH 25/27] update deploy.yml --- .github/workflows/deploy.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index fa61257f..47fbbcab 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -32,14 +32,18 @@ jobs: # Begin copy from ci.yml. Refactor? - name: Set up Python - uses: actions/setup-python@v5 + uses: actions/setup-python@v6 with: python-version: '3.11' - - name: Install zarrita + - name: Install uv + uses: astral-sh/setup-uv@v6 + + - name: Set up zarr-python run: | - python -m venv venv_zarrita - if [ "${{ runner.os }}" = "Windows" ]; then venv_zarrita/Scripts/pip install zarrita; else venv_zarrita/bin/pip install zarrita; fi + uv venv && uv init + uv add zarr + uv add zarrita - name: Download testdata run: | From 574b1c0deabd542979d851df5b4075fe4ddf0ec4 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Wed, 24 Sep 2025 15:31:53 +0200 Subject: [PATCH 26/27] adopt feedback --- .../java/dev/zarr/zarrjava/core/Array.java | 9 +-- .../zarrjava/v2/ArrayMetadataBuilder.java | 67 +++++++++++------ .../zarr/zarrjava/v2/codec/CodecBuilder.java | 75 ------------------- 3 files changed, 48 insertions(+), 103 deletions(-) delete mode 100644 src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java diff --git a/src/main/java/dev/zarr/zarrjava/core/Array.java b/src/main/java/dev/zarr/zarrjava/core/Array.java index aaf21873..0401b6b8 100644 --- a/src/main/java/dev/zarr/zarrjava/core/Array.java +++ b/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -17,6 +17,10 @@ public interface Array { ArrayMetadata metadata(); + + StoreHandle storeHandle(); + + CodecPipeline codecPipeline(); /** * Writes a ucar.ma2.Array into the Zarr array at a specified offset. The shape of the Zarr array @@ -203,11 +207,6 @@ default boolean chunkIsInArray(long[] chunkCoords) { return true; } - - StoreHandle storeHandle(); - - CodecPipeline codecPipeline(); - /** * Reads a part of the Zarr array based on a requested offset and shape into an ucar.ma2.Array. * diff --git a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java index 3e79a759..ea29b165 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java +++ b/src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java @@ -1,11 +1,11 @@ package dev.zarr.zarrjava.v2; +import com.scalableminds.bloscjava.Blosc; import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.core.chunkkeyencoding.Separator; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.v2.codec.CodecBuilder; - -import java.util.function.Function; +import dev.zarr.zarrjava.v2.codec.core.BloscCodec; +import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; public class ArrayMetadataBuilder { long[] shape = null; @@ -64,38 +64,59 @@ public ArrayMetadataBuilder withFillValue(Object fillValue) { return this; } - public ArrayMetadataBuilder withFilters(Codec... filters) { - this.filters = filters; + public ArrayMetadataBuilder withCompressor(Codec compressor) { + this.compressor = compressor; return this; } - public ArrayMetadataBuilder withFilters(Function codecBuilder) throws ZarrException { - if (dataType == null) { - throw new IllegalStateException("Please call `withDataType` first."); + public ArrayMetadataBuilder withBloscCompressor( + Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, + int blockSize + ) { + try { + this.compressor = new BloscCodec(cname, shuffle, clevel, typeSize, blockSize); + } catch (ZarrException e) { + throw new RuntimeException(e); } - CodecBuilder nestedCodecBuilder = new CodecBuilder(dataType); - this.filters = codecBuilder.apply(nestedCodecBuilder) - .build(); return this; } - public ArrayMetadataBuilder withCompressor(Codec compressor) { - this.compressor = compressor; - return this; + public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel, int blockSize) { + if (shuffle.equals("shuffle")) { + shuffle = "byteshuffle"; + } + return withBloscCompressor(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, + dataType.getByteCount(), blockSize + ); } - public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { - this.compressor = new CodecBuilder(dataType) - .withBlosc(cname, shuffle, clevel) - .build()[0]; - return this; + public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, int clevel) { + return withBloscCompressor(cname, shuffle, clevel, 0); + } + + public ArrayMetadataBuilder withBloscCompressor(String cname, int clevel) { + return withBloscCompressor(cname, "noshuffle", clevel); + } + + public ArrayMetadataBuilder withBloscCompressor(String cname) { + return withBloscCompressor(cname, 5); + } + + public ArrayMetadataBuilder withBloscCompressor() { + return withBloscCompressor("zstd"); } public ArrayMetadataBuilder withZlibCompressor(int level) { - this.compressor = new CodecBuilder(dataType) - .withZlib(level) - .build()[0]; - return this; + try { + this.compressor = new ZlibCodec(level); + } catch (ZarrException e) { + throw new RuntimeException(e); + } + return this; + } + + public ArrayMetadataBuilder withZlibCompressor() { + return withZlibCompressor(5); } public ArrayMetadata build() throws ZarrException { diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java b/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java deleted file mode 100644 index f724e585..00000000 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/CodecBuilder.java +++ /dev/null @@ -1,75 +0,0 @@ -package dev.zarr.zarrjava.v2.codec; - -import com.scalableminds.bloscjava.Blosc; -import dev.zarr.zarrjava.ZarrException; -import dev.zarr.zarrjava.v2.codec.core.ZlibCodec; -import dev.zarr.zarrjava.v2.DataType; -import dev.zarr.zarrjava.v2.codec.core.*; - -import java.util.ArrayList; -import java.util.List; - -public class CodecBuilder extends dev.zarr.zarrjava.core.codec.CodecBuilder { - - protected List codecs; - - public CodecBuilder(DataType dataType) { - super(dataType); - this.codecs = new ArrayList<>(); - } - - public CodecBuilder withBlosc( - Blosc.Compressor cname, Blosc.Shuffle shuffle, int clevel, int typeSize, - int blockSize - ) { - try { - codecs.add(new BloscCodec(cname, shuffle, clevel, typeSize, blockSize)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withBlosc(String cname, String shuffle, int clevel, int blockSize) { - if (shuffle.equals("shuffle")){ - shuffle = "byteshuffle"; - } - return withBlosc(Blosc.Compressor.fromString(cname), Blosc.Shuffle.fromString(shuffle), clevel, - dataType.getByteCount(), blockSize - ); - } - - public CodecBuilder withBlosc(String cname, String shuffle, int clevel) { - return withBlosc(cname, shuffle, clevel, 0); - } - - public CodecBuilder withBlosc(String cname, int clevel) { - return withBlosc(cname, "noshuffle", clevel); - } - - public CodecBuilder withBlosc(String cname) { - return withBlosc(cname, 5); - } - - public CodecBuilder withBlosc() { - return withBlosc("zstd"); - } - - public CodecBuilder withZlib(int level) { - try { - codecs.add(new ZlibCodec(level)); - } catch (ZarrException e) { - throw new RuntimeException(e); - } - return this; - } - - public CodecBuilder withZlib() { - return withZlib(5); - } - - public Codec[] build() { - return codecs.toArray(new Codec[0]); - } - -} From d42453cf4ce3f07665c4482cb88e2bd5b2d64639 Mon Sep 17 00:00:00 2001 From: brokkoli71 Date: Tue, 30 Sep 2025 13:03:11 +0200 Subject: [PATCH 27/27] move BytesBytesCodec.copy to Utils.copyStream --- .../dev/zarr/zarrjava/core/codec/BytesBytesCodec.java | 7 ------- src/main/java/dev/zarr/zarrjava/utils/Utils.java | 10 ++++++++++ .../dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java | 3 +-- .../dev/zarr/zarrjava/v3/codec/core/GzipCodec.java | 2 +- 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java index b7c448fa..0574f2f6 100644 --- a/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java +++ b/src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java @@ -13,11 +13,4 @@ public abstract class BytesBytesCodec extends AbstractCodec { public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException; - protected void copy(InputStream inputStream, OutputStream outputStream) throws IOException { - byte[] buffer = new byte[4096]; - int len; - while ((len = inputStream.read(buffer)) > 0) { - outputStream.write(buffer, 0, len); - } - } } diff --git a/src/main/java/dev/zarr/zarrjava/utils/Utils.java b/src/main/java/dev/zarr/zarrjava/utils/Utils.java index 3ea4bfdf..7cc88d2d 100644 --- a/src/main/java/dev/zarr/zarrjava/utils/Utils.java +++ b/src/main/java/dev/zarr/zarrjava/utils/Utils.java @@ -3,6 +3,7 @@ import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.InputStream; +import java.io.OutputStream; import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; @@ -78,6 +79,15 @@ public static T[] concatArrays(T[] array1, T[]... arrays) { return result; } + public static void copyStream(InputStream inputStream, OutputStream outputStream) throws IOException { + byte[] buffer = new byte[4096]; + int len; + while ((len = inputStream.read(buffer)) > 0) { + outputStream.write(buffer, 0, len); + } + } + + public static boolean isPermutation(int[] array) { if (array.length==0){ return false; diff --git a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java index f7124ac9..8b2bed6a 100644 --- a/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v2/codec/core/ZlibCodec.java @@ -5,7 +5,6 @@ import dev.zarr.zarrjava.ZarrException; import dev.zarr.zarrjava.utils.Utils; import dev.zarr.zarrjava.v2.codec.Codec; -import dev.zarr.zarrjava.core.ArrayMetadata; import dev.zarr.zarrjava.core.codec.BytesBytesCodec; import java.io.ByteArrayInputStream; @@ -34,7 +33,7 @@ public ZlibCodec( public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InflaterInputStream inputStream = new InflaterInputStream( new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { - copy(inputStream, outputStream); + Utils.copyStream(inputStream, outputStream); inputStream.close(); return ByteBuffer.wrap(outputStream.toByteArray()); } catch (IOException ex) { diff --git a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java index e92bfcab..c197fc43 100644 --- a/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java +++ b/src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java @@ -34,7 +34,7 @@ public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException { try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); GZIPInputStream inputStream = new GZIPInputStream( new ByteArrayInputStream(Utils.toArray(chunkBytes)))) { - copy(inputStream, outputStream); + Utils.copyStream(inputStream, outputStream); inputStream.close(); return ByteBuffer.wrap(outputStream.toByteArray()); } catch (IOException ex) {