Skip to content

Commit a8c6347

Browse files
brokkoli71brokkoli71
committed
split v2 and v3 codecs to fix serialization
1 parent f3a8097 commit a8c6347

27 files changed

Lines changed: 389 additions & 111 deletions

‎src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package dev.zarr.zarrjava.codec;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import ucar.ma2.Array;
5+
6+
public interface ArrayArrayCodec {
7+
8+
Array encode(Array chunkArray)
9+
throws ZarrException;
10+
11+
Array decode(Array chunkArray)
12+
throws ZarrException;
13+
14+
}

src/main/java/dev/zarr/zarrjava/v3/codec/ArrayBytesCodec.java renamed to src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,24 @@
1-
package dev.zarr.zarrjava.v3.codec;
1+
package dev.zarr.zarrjava.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.store.StoreHandle;
55
import java.nio.ByteBuffer;
66
import ucar.ma2.Array;
77

8-
public abstract class ArrayBytesCodec extends Codec {
8+
public interface ArrayBytesCodec {
99

10-
protected abstract ByteBuffer encode(Array chunkArray)
10+
ByteBuffer encode(Array chunkArray)
1111
throws ZarrException;
1212

13-
protected abstract Array decode(ByteBuffer chunkBytes)
13+
Array decode(ByteBuffer chunkBytes)
1414
throws ZarrException;
1515

16-
public abstract static class WithPartialDecode extends ArrayBytesCodec {
16+
interface WithPartialDecode extends ArrayBytesCodec {
1717

1818
public abstract Array decode(ByteBuffer shardBytes) throws ZarrException;
1919
public abstract ByteBuffer encode(Array shardArray) throws ZarrException;
2020

21-
protected abstract Array decodePartial(
21+
Array decodePartial(
2222
StoreHandle handle, long[] offset, int[] shape
2323
) throws ZarrException;
2424
}

‎src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
package dev.zarr.zarrjava.codec;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
5+
import java.io.IOException;
6+
import java.io.InputStream;
7+
import java.io.OutputStream;
8+
import java.nio.ByteBuffer;
9+
10+
public interface BytesBytesCodec {
11+
12+
ByteBuffer encode(ByteBuffer chunkBytes) throws ZarrException;
13+
14+
ByteBuffer decode(ByteBuffer chunkBytes) throws ZarrException;
15+
16+
default void copy(InputStream inputStream, OutputStream outputStream) throws IOException {
17+
byte[] buffer = new byte[4096];
18+
int len;
19+
while ((len = inputStream.read(buffer)) > 0) {
20+
outputStream.write(buffer, 0, len);
21+
}
22+
}
23+
}

‎src/main/java/dev/zarr/zarrjava/codec/Codec.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package dev.zarr.zarrjava.codec;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.v3.ArrayMetadata;
5+
6+
public interface Codec {
7+
ArrayMetadata.CoreArrayMetadata resolveArrayMetadata() throws ZarrException;
8+
9+
long computeEncodedSize(long inputByteLength, ArrayMetadata.CoreArrayMetadata arrayMetadata)
10+
throws ZarrException;
11+
12+
void setCoreArrayMetadata(ArrayMetadata.CoreArrayMetadata arrayMetadata) throws ZarrException;
13+
}
14+

‎src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
package dev.zarr.zarrjava.codec;
2+
3+
import dev.zarr.zarrjava.v3.DataType;
4+
5+
public abstract class CodecBuilder {
6+
7+
final protected DataType dataType;
8+
9+
public CodecBuilder(DataType dataType) {
10+
this.dataType = dataType;
11+
}
12+
13+
public abstract Codec[] build();
14+
}

src/main/java/dev/zarr/zarrjava/v3/codec/CodecPipeline.java renamed to src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.v3.codec;
1+
package dev.zarr.zarrjava.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.store.StoreHandle;

‎src/main/java/dev/zarr/zarrjava/interfaces/Array.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
import dev.zarr.zarrjava.utils.IndexingUtils;
66
import dev.zarr.zarrjava.utils.MultiArrayUtils;
77
import dev.zarr.zarrjava.utils.Utils;
8-
import dev.zarr.zarrjava.v3.codec.CodecPipeline;
8+
import dev.zarr.zarrjava.codec.CodecPipeline;
99
import ucar.ma2.InvalidRangeException;
1010

1111
import javax.annotation.Nonnull;

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,13 @@
11
package dev.zarr.zarrjava.v2;
22

33
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import com.fasterxml.jackson.datatype.jdk8.Jdk8Module;
45
import dev.zarr.zarrjava.ZarrException;
56
import dev.zarr.zarrjava.store.StoreHandle;
67
import dev.zarr.zarrjava.utils.Utils;
7-
import dev.zarr.zarrjava.v3.codec.CodecPipeline;
8-
import dev.zarr.zarrjava.v3.codec.Codec;
8+
import dev.zarr.zarrjava.codec.CodecPipeline;
9+
import dev.zarr.zarrjava.v2.codec.Codec;
10+
import dev.zarr.zarrjava.v2.codec.CodecRegistry;
911
import dev.zarr.zarrjava.v3.codec.core.BytesCodec;
1012

1113
import javax.annotation.Nonnull;
@@ -15,8 +17,6 @@
1517
import java.util.function.Function;
1618
import java.util.stream.Collectors;
1719

18-
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
19-
2020
public class Array implements dev.zarr.zarrjava.interfaces.Array {
2121

2222
static final String ZARRAY = ".zarray";
@@ -28,8 +28,9 @@ protected Array(StoreHandle storeHandle, ArrayMetadata arrayMetadata) throws IOE
2828
this.storeHandle = storeHandle;
2929
this.metadata = arrayMetadata;
3030
this.codecPipeline = new CodecPipeline(Utils.concatArrays(
31-
metadata.filters,
32-
new Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())},
31+
new dev.zarr.zarrjava.codec.Codec[]{},
32+
metadata.filters == null ? new Codec[]{} : metadata.filters,
33+
new dev.zarr.zarrjava.codec.Codec[]{new BytesCodec(arrayMetadata.endianness.toEndian())},
3334
metadata.compressor == null ? new Codec[]{} : new Codec[]{metadata.compressor}
3435
), metadata.coreArrayMetadata);
3536
}
@@ -52,6 +53,14 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
5253
);
5354
}
5455

56+
public static ObjectMapper makeObjectMapper() {
57+
ObjectMapper objectMapper = new ObjectMapper();
58+
objectMapper.registerModule(new Jdk8Module());
59+
objectMapper.registerSubtypes(CodecRegistry.getNamedTypes());
60+
return objectMapper;
61+
}
62+
63+
5564
/**
5665
* Creates a new Zarr array with the provided metadata at a specified storage location. This
5766
* method will raise an exception if a Zarr array already exists at the specified storage

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadata.java‎

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding;
1010
import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator;
1111
import dev.zarr.zarrjava.v3.chunkkeyencoding.V2ChunkKeyEncoding;
12-
import dev.zarr.zarrjava.v3.codec.Codec;
12+
import dev.zarr.zarrjava.v2.codec.Codec;
1313
import ucar.ma2.Array;
1414

1515
import javax.annotation.Nullable;
@@ -58,18 +58,17 @@ public ArrayMetadata(
5858
@JsonProperty(value = "shape", required = true) long[] shape,
5959
@JsonProperty(value = "chunks", required = true) int[] chunks,
6060
@JsonProperty(value = "dtype", required = true) DataTypeV2 dataTypeV2,
61-
@JsonProperty(value = "fill_value", required = true) Object fillValue, //todo can be "null"
61+
@Nullable @JsonProperty(value = "fill_value", required = true) Object fillValue, //todo test when null
6262
@JsonProperty(value = "order", required = true) Order order,
6363
@Nullable @JsonProperty(value = "dimension_separator") Separator dimensionSeparator,
64-
@Nullable @JsonProperty(value = "filters") Codec[] filters, //todo can be "null"
65-
@Nullable @JsonProperty(value = "compressor") Codec compressor //todo can be "null"
64+
@Nullable @JsonProperty(value = "filters") Codec[] filters,
65+
@Nullable @JsonProperty(value = "compressor") Codec compressor
6666
) throws ZarrException {
6767
super();
6868
if (zarrFormat != this.zarrFormat) {
6969
throw new ZarrException(
7070
"Expected zarr format '" + this.zarrFormat + "', got '" + zarrFormat + "'.");
7171
}
72-
7372
this.shape = shape;
7473
this.chunks = chunks;
7574
this.dataTypeV2 = dataTypeV2;

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.v3.DataType;
55
import dev.zarr.zarrjava.v3.chunkkeyencoding.Separator;
6-
import dev.zarr.zarrjava.v3.codec.Codec;
7-
import dev.zarr.zarrjava.v3.codec.CodecBuilder;
6+
import dev.zarr.zarrjava.v2.codec.Codec;
7+
import dev.zarr.zarrjava.v2.codec.CodecBuilder;
88

99
import java.util.function.Function;
1010

@@ -15,7 +15,7 @@ public class ArrayMetadataBuilder {
1515
Order order = Order.C;
1616
Separator dimensionSeparator = Separator.DOT;
1717
Object fillValue = 0;
18-
Codec[] filters = new Codec[]{};
18+
Codec[] filters = null;
1919
Codec compressor = null;
2020

2121

@@ -81,7 +81,7 @@ public ArrayMetadataBuilder withFilters(Function<CodecBuilder, CodecBuilder> cod
8181
}
8282
CodecBuilder nestedCodecBuilder = new CodecBuilder(dataTypeV2.toV3());
8383
this.filters = codecBuilder.apply(nestedCodecBuilder)
84-
.build(false);
84+
.build();
8585
return this;
8686
}
8787

@@ -94,7 +94,7 @@ public ArrayMetadataBuilder withBloscCompressor(String cname, String shuffle, i
9494
try {
9595
this.compressor = new CodecBuilder(dataTypeV2.toV3())
9696
.withBlosc(cname, shuffle, clevel)
97-
.build(false)[0];
97+
.build()[0];
9898
} catch (ZarrException e) {
9999
throw new RuntimeException(e);
100100
}
@@ -105,7 +105,7 @@ public ArrayMetadataBuilder withZlibCompressor(int level) {
105105
try {
106106
this.compressor = new CodecBuilder(dataTypeV2.toV3())
107107
.withZlib(level)
108-
.build(false)[0];
108+
.build()[0];
109109
} catch (ZarrException e) {
110110
throw new RuntimeException(e);
111111
}

0 commit comments

Comments
 (0)