Skip to content

Commit 8fc9187

Browse files
brokkoli71brokkoli71
committed
add zlib
1 parent 135ebe8 commit 8fc9187

8 files changed

Lines changed: 181 additions & 25 deletions

File tree

‎.gitignore‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,4 +38,7 @@ build/
3838
### Custom ###
3939
/testdata/l4_sample
4040
/testoutput
41-
/venv_zarrita
41+
/.python-version
42+
/main.py
43+
/pyproject.toml
44+
/uv.lock

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,17 @@ public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, i
101101
return this;
102102
}
103103

104+
public ArrayMetadataBuilder withZlibCompressor(int level) {
105+
try {
106+
this.compressor = new CodecBuilder(dataTypeV2.toV3())
107+
.withZlib(level)
108+
.build(false)[0];
109+
} catch (ZarrException e) {
110+
throw new RuntimeException(e);
111+
}
112+
return this;
113+
}
114+
104115
public ArrayMetadata build() throws ZarrException {
105116
if (shape == null) {
106117
throw new IllegalStateException("Please call `withShape` first.");

‎src/main/java/dev/zarr/zarrjava/v3/codec/BytesBytesCodec.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
import dev.zarr.zarrjava.ZarrException;
44

5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.io.OutputStream;
58
import java.nio.ByteBuffer;
69

710
public abstract class BytesBytesCodec extends Codec {
@@ -10,4 +13,11 @@ public abstract class BytesBytesCodec extends Codec {
1013

1114
public abstract ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException;
1215

16+
protected void copy(InputStream inputStream, OutputStream outputStream) throws IOException {
17+
byte[] buffer = new byte[4096];
18+
int len;
19+
while ((len = inputStream.read(buffer)) > 0) {
20+
outputStream.write(buffer, 0, len);
21+
}
22+
}
1323
}

‎src/main/java/dev/zarr/zarrjava/v3/codec/CodecBuilder.java‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,10 @@
33
import com.scalableminds.bloscjava.Blosc;
44
import dev.zarr.zarrjava.ZarrException;
55
import dev.zarr.zarrjava.v3.DataType;
6-
import dev.zarr.zarrjava.v3.codec.core.BloscCodec;
7-
import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
6+
import dev.zarr.zarrjava.v3.codec.core.*;
87
import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Configuration;
98
import dev.zarr.zarrjava.v3.codec.core.BytesCodec.Endian;
10-
import dev.zarr.zarrjava.v3.codec.core.Crc32cCodec;
11-
import dev.zarr.zarrjava.v3.codec.core.GzipCodec;
12-
import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec;
13-
import dev.zarr.zarrjava.v3.codec.core.TransposeCodec;
14-
import dev.zarr.zarrjava.v3.codec.core.ZstdCodec;
9+
1510
import java.util.ArrayList;
1611
import java.util.Collections;
1712
import java.util.List;
@@ -65,6 +60,19 @@ public CodecBuilder withBlosc() {
6560
return withBlosc("zstd");
6661
}
6762

63+
public CodecBuilder withZlib(int level) {
64+
try {
65+
codecs.add(new ZlibCodec(new ZlibCodec.Configuration(level)));
66+
} catch (ZarrException e) {
67+
throw new RuntimeException(e);
68+
}
69+
return this;
70+
}
71+
72+
public CodecBuilder withZlib() {
73+
return withZlib(5);
74+
}
75+
6876
public CodecBuilder withTranspose(int[] order) {
6977
codecs.add(new TransposeCodec(new TransposeCodec.Configuration(order)));
7078
return this;

‎src/main/java/dev/zarr/zarrjava/v3/codec/CodecRegistry.java‎

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,8 @@
11
package dev.zarr.zarrjava.v3.codec;
22

33
import com.fasterxml.jackson.databind.jsontype.NamedType;
4-
import dev.zarr.zarrjava.v3.codec.core.BloscCodec;
5-
import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
6-
import dev.zarr.zarrjava.v3.codec.core.Crc32cCodec;
7-
import dev.zarr.zarrjava.v3.codec.core.GzipCodec;
8-
import dev.zarr.zarrjava.v3.codec.core.ShardingIndexedCodec;
9-
import dev.zarr.zarrjava.v3.codec.core.TransposeCodec;
10-
import dev.zarr.zarrjava.v3.codec.core.ZstdCodec;
4+
import dev.zarr.zarrjava.v3.codec.core.*;
5+
116
import java.util.HashMap;
127
import java.util.Map;
138

@@ -20,6 +15,7 @@ public class CodecRegistry {
2015
addType("bytes", BytesCodec.class);
2116
addType("blosc", BloscCodec.class);
2217
addType("gzip", GzipCodec.class);
18+
addType("zlib", ZlibCodec.class);
2319
addType("zstd", ZstdCodec.class);
2420
addType("crc32c", Crc32cCodec.class);
2521
addType("sharding_indexed", ShardingIndexedCodec.class);

‎src/main/java/dev/zarr/zarrjava/v3/codec/core/GzipCodec.java‎

Lines changed: 1 addition & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,6 @@
99
import java.io.ByteArrayInputStream;
1010
import java.io.ByteArrayOutputStream;
1111
import java.io.IOException;
12-
import java.io.InputStream;
13-
import java.io.OutputStream;
1412
import java.nio.ByteBuffer;
1513
import java.util.zip.GZIPInputStream;
1614
import java.util.zip.GZIPOutputStream;
@@ -28,13 +26,7 @@ public GzipCodec(
2826
this.configuration = configuration;
2927
}
3028

31-
private void copy(InputStream inputStream, OutputStream outputStream) throws IOException {
32-
byte[] buffer = new byte[4096];
33-
int len;
34-
while ((len = inputStream.read(buffer)) > 0) {
35-
outputStream.write(buffer, 0, len);
36-
}
37-
}
29+
3830

3931
@Override
4032
public ByteBuffer decode(ByteBuffer chunkBytes)

‎src/main/java/dev/zarr/zarrjava/v3/codec/core/ZlibCodec.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
package dev.zarr.zarrjava.v3.codec.core;
2+
3+
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonProperty;
5+
import dev.zarr.zarrjava.ZarrException;
6+
import dev.zarr.zarrjava.utils.Utils;
7+
import dev.zarr.zarrjava.v3.ArrayMetadata;
8+
import dev.zarr.zarrjava.v3.codec.BytesBytesCodec;
9+
10+
import java.io.ByteArrayInputStream;
11+
import java.io.ByteArrayOutputStream;
12+
import java.io.IOException;
13+
import java.nio.ByteBuffer;
14+
import java.util.zip.*;
15+
import javax.annotation.Nonnull;
16+
17+
public class ZlibCodec extends BytesBytesCodec {
18+
19+
public final String name = "zlib";
20+
@Nonnull
21+
public final Configuration configuration;
22+
23+
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
24+
public ZlibCodec(
25+
@Nonnull @JsonProperty(value = "configuration", required = true) Configuration configuration) {
26+
this.configuration = configuration;
27+
}
28+
29+
30+
@Override
31+
public ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException {
32+
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream(); InflaterInputStream inputStream = new InflaterInputStream(
33+
new ByteArrayInputStream(Utils.toArray(chunkBytes)))) {
34+
copy(inputStream, outputStream);
35+
inputStream.close();
36+
return ByteBuffer.wrap(outputStream.toByteArray());
37+
} catch (IOException ex) {
38+
throw new ZarrException("Error in decoding gzip.", ex);
39+
}
40+
}
41+
42+
@Override
43+
public ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException {
44+
try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
45+
DeflaterOutputStream dos = new DeflaterOutputStream(outputStream, new Deflater(this.configuration.level))) {
46+
dos.write(Utils.toArray(chunkBytes));
47+
dos.close();
48+
return ByteBuffer.wrap(outputStream.toByteArray());
49+
} catch (IOException ex) {
50+
throw new ZarrException("Error in encoding zlib.", ex);
51+
}
52+
}
53+
54+
@Override
55+
public long computeEncodedSize(long inputByteLength,
56+
ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException {
57+
throw new ZarrException("Not implemented for Zlib codec.");
58+
}
59+
60+
61+
public static final class Configuration {
62+
63+
public final int level;
64+
65+
@JsonCreator(mode = JsonCreator.Mode.PROPERTIES)
66+
public Configuration(@JsonProperty(value = "level", defaultValue = "1") int level)
67+
throws ZarrException {
68+
if (level < 0 || level > 9) {
69+
throw new ZarrException("'level' needs to be between 0 and 9.");
70+
}
71+
this.level = level;
72+
}
73+
}
74+
}

‎src/test/java/dev/zarr/zarrjava/ZarrTest.java‎

Lines changed: 63 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -662,7 +662,7 @@ public void testMetadataAcceptsEmptyStorageTransformer() throws ZarrException, I
662662

663663
@ParameterizedTest
664664
@CsvSource({"blosclz,noshuffle,0", "lz4,shuffle,6", "lz4hc,bitshuffle,3", "zlib,shuffle,5", "zstd,bitshuffle,9"})
665-
public void testV2create(String cname, String shuffle, int clevel) throws IOException, ZarrException {
665+
public void testV2createBlosc(String cname, String shuffle, int clevel) throws IOException, ZarrException {
666666
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(
667667
new FilesystemStore(TESTOUTPUT).resolve("v2_create", cname + "_" + shuffle + "_" + clevel),
668668
dev.zarr.zarrjava.v2.Array.metadataBuilder()
@@ -679,4 +679,66 @@ public void testV2create(String cname, String shuffle, int clevel) throws IOExce
679679
Assertions.assertEquals(outArray.getSize(), 8 * 8);
680680
Assertions.assertEquals(outArray.getByte(0), 0);
681681
}
682+
683+
@Test
684+
public void testV2create() throws IOException, ZarrException {
685+
DataType dataType = DataType.UINT32;
686+
687+
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(
688+
new FilesystemStore(TESTOUTPUT).resolve("v2_create"),
689+
dev.zarr.zarrjava.v2.Array.metadataBuilder()
690+
.withShape(10, 10)
691+
.withDataType(dataType)
692+
.withChunks(5, 5)
693+
.withFillValue(2)
694+
.build()
695+
);
696+
array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8}));
697+
698+
ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8});
699+
Assertions.assertEquals(outArray.getSize(), 8 * 8);
700+
Assertions.assertEquals(outArray.getByte(0), 0);
701+
}
702+
703+
@Test
704+
public void testV2Filters() throws IOException, ZarrException {
705+
DataType dataType = DataType.UINT32;
706+
707+
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(
708+
new FilesystemStore(TESTOUTPUT).resolve("v2_create"),
709+
dev.zarr.zarrjava.v2.Array.metadataBuilder()
710+
.withShape(10, 10)
711+
.withDataType(dataType)
712+
.withChunks(5, 5)
713+
.withFillValue(2)
714+
.withFilters(f -> f.withTranspose(new int[]{1, 0}))
715+
.build()
716+
);
717+
array.write(new long[]{2, 2}, ucar.ma2.Array.factory(dataType.getMA2DataType(), new int[]{8, 8}));
718+
719+
ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{8, 8});
720+
Assertions.assertEquals(outArray.getSize(), 8 * 8);
721+
Assertions.assertEquals(outArray.getByte(0), 0);
722+
}
723+
724+
725+
@ParameterizedTest
726+
@ValueSource(ints = {0, 1, 5, 9})
727+
public void testV2createZlib(int level) throws IOException, ZarrException {
728+
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.create(
729+
new FilesystemStore(TESTOUTPUT).resolve("v2_create_zlib", String.valueOf(level)),
730+
dev.zarr.zarrjava.v2.Array.metadataBuilder()
731+
.withShape(15, 10)
732+
.withDataType(DataType.UINT8)
733+
.withChunks(4, 5)
734+
.withFillValue(5)
735+
.withZlibCompressor(level)
736+
.build()
737+
);
738+
array.write(new long[]{2, 2}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{7, 6}));
739+
740+
ucar.ma2.Array outArray = array.read(new long[]{2, 2}, new int[]{7, 6});
741+
Assertions.assertEquals(outArray.getSize(), 7 * 6);
742+
Assertions.assertEquals(outArray.getByte(0), 0);
743+
}
682744
}

0 commit comments

Comments
 (0)