Skip to content

Commit 8400124

Browse files
brokkoli71brokkoli71
committed
isolate v2 and v3 with common core
1 parent b93de23 commit 8400124

33 files changed

Lines changed: 322 additions & 308 deletions

src/main/java/dev/zarr/zarrjava/interfaces/Array.java renamed to src/main/java/dev/zarr/zarrjava/core/Array.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,11 @@
1-
package dev.zarr.zarrjava.interfaces;
1+
package dev.zarr.zarrjava.core;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.store.StoreHandle;
55
import dev.zarr.zarrjava.utils.IndexingUtils;
66
import dev.zarr.zarrjava.utils.MultiArrayUtils;
77
import dev.zarr.zarrjava.utils.Utils;
8-
import dev.zarr.zarrjava.codec.CodecPipeline;
8+
import dev.zarr.zarrjava.core.codec.CodecPipeline;
99
import ucar.ma2.InvalidRangeException;
1010

1111
import javax.annotation.Nonnull;

‎src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java‎

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
package dev.zarr.zarrjava.core;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.utils.MultiArrayUtils;
5+
import dev.zarr.zarrjava.utils.Utils;
6+
import dev.zarr.zarrjava.v3.chunkkeyencoding.ChunkKeyEncoding;
7+
import ucar.ma2.Array;
8+
9+
import javax.annotation.Nonnull;
10+
import java.nio.ByteBuffer;
11+
import java.util.Arrays;
12+
13+
public interface ArrayMetadata {
14+
int ndim();
15+
16+
int[] chunkShape();
17+
18+
long[] shape();
19+
20+
DataType dataType();
21+
22+
Array allocateFillValueChunk();
23+
24+
ChunkKeyEncoding chunkKeyEncoding();
25+
26+
Object parsedFillValue();
27+
28+
static Object parseFillValue(Object fillValue, @Nonnull DataType dataType)
29+
throws ZarrException {
30+
boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL;
31+
boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8;
32+
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16;
33+
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32;
34+
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64;
35+
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32;
36+
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64;
37+
38+
if (fillValue instanceof Boolean) {
39+
Boolean fillValueBool = (Boolean) fillValue;
40+
if (dataTypeIsBool) {
41+
return fillValueBool;
42+
}
43+
}
44+
if (fillValue instanceof Number) {
45+
Number fillValueNumber = (Number) fillValue;
46+
if (dataTypeIsBool) {
47+
return fillValueNumber.byteValue() != 0;
48+
} else if (dataTypeIsByte) {
49+
return fillValueNumber.byteValue();
50+
} else if (dataTypeIsShort) {
51+
return fillValueNumber.shortValue();
52+
} else if (dataTypeIsInt) {
53+
return fillValueNumber.intValue();
54+
} else if (dataTypeIsLong) {
55+
return fillValueNumber.longValue();
56+
} else if (dataTypeIsFloat) {
57+
return fillValueNumber.floatValue();
58+
} else if (dataTypeIsDouble) {
59+
return fillValueNumber.doubleValue();
60+
}
61+
// Fallback to throwing below
62+
} else if (fillValue instanceof String) {
63+
String fillValueString = (String) fillValue;
64+
if (fillValueString.equals("NaN")) {
65+
if (dataTypeIsFloat) {
66+
return Float.NaN;
67+
} else if (dataTypeIsDouble) {
68+
return Double.NaN;
69+
}
70+
throw new ZarrException(
71+
"Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'.");
72+
} else if (fillValueString.equals("+Infinity")) {
73+
if (dataTypeIsFloat) {
74+
return Float.POSITIVE_INFINITY;
75+
} else if (dataTypeIsDouble) {
76+
return Double.POSITIVE_INFINITY;
77+
}
78+
throw new ZarrException(
79+
"Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'.");
80+
} else if (fillValueString.equals("-Infinity")) {
81+
if (dataTypeIsFloat) {
82+
return Float.NEGATIVE_INFINITY;
83+
} else if (dataTypeIsDouble) {
84+
return Double.NEGATIVE_INFINITY;
85+
}
86+
throw new ZarrException(
87+
"Invalid fill value '" + fillValueString + "' for data type '" + dataType + "'.");
88+
}
89+
else if (fillValueString.startsWith("0b") || fillValueString.startsWith("0x")) {
90+
ByteBuffer buf = null;
91+
if (fillValueString.startsWith("0b")) {
92+
buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> {
93+
for (int i = 0; i < dataType.getByteCount(); i++) {
94+
b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 8, 2 + (i + 1) * 8),
95+
2));
96+
}
97+
return b;
98+
});
99+
} else if (fillValueString.startsWith("0x")) {
100+
buf = Utils.makeByteBuffer(dataType.getByteCount(), b -> {
101+
for (int i = 0; i < dataType.getByteCount(); i++) {
102+
b.put((byte) Integer.parseInt(fillValueString.substring(2 + i * 2, 2 + (i + 1) * 2),
103+
16));
104+
}
105+
return b;
106+
});
107+
}
108+
if (buf != null) {
109+
if (dataTypeIsBool) {
110+
return buf.get() != 0;
111+
} else if (dataTypeIsByte) {
112+
return buf.get();
113+
} else if (dataTypeIsShort) {
114+
return buf.getShort();
115+
} else if (dataTypeIsInt) {
116+
return buf.getInt();
117+
} else if (dataTypeIsLong) {
118+
return buf.getLong();
119+
} else if (dataTypeIsFloat) {
120+
return buf.getFloat();
121+
} else if (dataTypeIsDouble) {
122+
return buf.getDouble();
123+
// Fallback to throwing below
124+
}
125+
}
126+
}
127+
}
128+
throw new ZarrException("Invalid fill value '" + fillValue + "'.");
129+
}
130+
131+
final class CoreArrayMetadata {
132+
133+
public final long[] shape;
134+
public final int[] chunkShape;
135+
public final dev.zarr.zarrjava.v3.DataType dataType;
136+
public final Object parsedFillValue;
137+
138+
public CoreArrayMetadata(long[] shape, int[] chunkShape, dev.zarr.zarrjava.v3.DataType dataType,
139+
Object parsedFillValue) {
140+
this.shape = shape;
141+
this.chunkShape = chunkShape;
142+
this.dataType = dataType;
143+
this.parsedFillValue = parsedFillValue;
144+
}
145+
146+
public int ndim() {
147+
return shape.length;
148+
}
149+
150+
public int chunkSize() {
151+
return Arrays.stream(chunkShape)
152+
.reduce(1, (acc, a) -> acc * a);
153+
}
154+
155+
public int chunkByteLength() {
156+
return this.dataType.getByteCount() * chunkSize();
157+
}
158+
159+
public ucar.ma2.Array allocateFillValueChunk() {
160+
ucar.ma2.Array outputArray = ucar.ma2.Array.factory(dataType.getMA2DataType(), chunkShape);
161+
MultiArrayUtils.fill(outputArray, parsedFillValue);
162+
return outputArray;
163+
}
164+
}
165+
166+
167+
}

src/main/java/dev/zarr/zarrjava/interfaces/DataType.java renamed to src/main/java/dev/zarr/zarrjava/core/DataType.java

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
1-
package dev.zarr.zarrjava.interfaces;
1+
package dev.zarr.zarrjava.core;
22

33
public interface DataType {
44
ucar.ma2.DataType getMA2DataType();
5+
6+
int getByteCount();
57
}

src/main/java/dev/zarr/zarrjava/codec/ArrayArrayCodec.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/ArrayArrayCodec.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import ucar.ma2.Array;

src/main/java/dev/zarr/zarrjava/codec/ArrayBytesCodec.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/ArrayBytesCodec.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.store.StoreHandle;

src/main/java/dev/zarr/zarrjava/codec/BytesBytesCodec.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/BytesBytesCodec.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44

src/main/java/dev/zarr/zarrjava/codec/Codec.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/Codec.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.v3.ArrayMetadata;

src/main/java/dev/zarr/zarrjava/codec/CodecBuilder.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/CodecBuilder.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.v3.DataType;
44

src/main/java/dev/zarr/zarrjava/codec/CodecPipeline.java renamed to src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,8 @@
1-
package dev.zarr.zarrjava.codec;
1+
package dev.zarr.zarrjava.core.codec;
22

33
import dev.zarr.zarrjava.ZarrException;
44
import dev.zarr.zarrjava.store.StoreHandle;
5-
import dev.zarr.zarrjava.v3.ArrayMetadata.CoreArrayMetadata;
5+
import dev.zarr.zarrjava.core.ArrayMetadata.CoreArrayMetadata;
66
import java.nio.ByteBuffer;
77
import java.util.Arrays;
88
import javax.annotation.Nonnull;

‎src/main/java/dev/zarr/zarrjava/core/codec/core/BloscCodec.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package dev.zarr.zarrjava.core.codec.core;
2+
3+
import com.fasterxml.jackson.core.JsonGenerator;
4+
import com.fasterxml.jackson.core.JsonParseException;
5+
import com.fasterxml.jackson.core.JsonParser;
6+
import com.fasterxml.jackson.databind.DeserializationContext;
7+
import com.fasterxml.jackson.databind.SerializerProvider;
8+
import com.fasterxml.jackson.databind.deser.std.StdDeserializer;
9+
import com.fasterxml.jackson.databind.ser.std.StdSerializer;
10+
import com.scalableminds.bloscjava.Blosc;
11+
import dev.zarr.zarrjava.ZarrException;
12+
import dev.zarr.zarrjava.core.codec.BytesBytesCodec;
13+
import dev.zarr.zarrjava.utils.Utils;
14+
15+
import java.io.IOException;
16+
import java.nio.ByteBuffer;
17+
18+
public interface BloscCodec extends BytesBytesCodec {
19+
20+
@Override
21+
default ByteBuffer decode(ByteBuffer chunkBytes)
22+
throws ZarrException {
23+
try {
24+
return ByteBuffer.wrap(Blosc.decompress(Utils.toArray(chunkBytes)));
25+
} catch (Exception ex) {
26+
throw new ZarrException("Error in decoding blosc.", ex);
27+
}
28+
}
29+
30+
final class CustomCompressorDeserializer extends StdDeserializer<Blosc.Compressor> {
31+
32+
public CustomCompressorDeserializer() {
33+
this(null);
34+
}
35+
36+
public CustomCompressorDeserializer(Class<?> vc) {
37+
super(vc);
38+
}
39+
40+
@Override
41+
public Blosc.Compressor deserialize(JsonParser jsonParser, DeserializationContext ctxt)
42+
throws IOException {
43+
String cname = jsonParser.getCodec()
44+
.readValue(jsonParser, String.class);
45+
Blosc.Compressor compressor = Blosc.Compressor.fromString(cname);
46+
if (compressor == null) {
47+
throw new JsonParseException(
48+
jsonParser,
49+
String.format("Could not parse the Blosc.Compressor. Got '%s'", cname)
50+
);
51+
}
52+
return compressor;
53+
}
54+
}
55+
56+
final class CustomCompressorSerializer extends StdSerializer<Blosc.Compressor> {
57+
58+
public CustomCompressorSerializer() {
59+
super(Blosc.Compressor.class);
60+
}
61+
62+
public CustomCompressorSerializer(Class t) {
63+
super(t);
64+
}
65+
66+
@Override
67+
public void serialize(Blosc.Compressor compressor, JsonGenerator generator,
68+
SerializerProvider provider)
69+
throws IOException {
70+
generator.writeString(compressor.getValue());
71+
}
72+
}
73+
}

0 commit comments

Comments
 (0)