Skip to content

Commit 4cb4250

Browse files
authored
add big endian dtypes to v2 (#60)
* add big endian dtypes to v2 * add endian tests
1 parent f1c6ad0 commit 4cb4250

9 files changed

Lines changed: 210 additions & 119 deletions

File tree

src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType
3636
}
3737
boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL;
3838
boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8;
39-
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16;
40-
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32;
41-
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64;
42-
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32;
43-
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64;
39+
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16|| dataType == dev.zarr.zarrjava.v2.DataType.UINT16_BE;
40+
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32_BE;
41+
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE;
42+
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE;
43+
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE;
4444

4545
if (fillValue instanceof Boolean) {
4646
Boolean fillValueBool = (Boolean) fillValue;

src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,12 @@ public ArrayMetadata build() throws ZarrException {
150150
if (dataType == null) {
151151
throw new IllegalStateException("Please call `withDataType` first.");
152152
}
153-
153+
154154
// If chunks are not specified, calculate default chunks
155155
if (chunks == null) {
156156
chunks = Utils.calculateDefaultChunks(shape);
157157
}
158-
158+
159159
return new ArrayMetadata(
160160
2,
161161
shape,

src/main/java/dev/zarr/zarrjava/v2/DataType.java

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,15 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType {
1313
UINT32("u4", Endianness.LITTLE),
1414
UINT64("u8", Endianness.LITTLE),
1515
FLOAT32("f4", Endianness.LITTLE),
16-
FLOAT64("f8", Endianness.LITTLE);
16+
FLOAT64("f8", Endianness.LITTLE),
17+
INT16_BE("i2", Endianness.BIG),
18+
INT32_BE("i4", Endianness.BIG),
19+
INT64_BE("i8", Endianness.BIG),
20+
UINT16_BE("u2", Endianness.BIG),
21+
UINT32_BE("u4", Endianness.BIG),
22+
UINT64_BE("u8", Endianness.BIG),
23+
FLOAT32_BE("f4", Endianness.BIG),
24+
FLOAT64_BE("f8", Endianness.BIG);
1725

1826
private final String dtype;
1927
private final Endianness endianness;
@@ -41,22 +49,30 @@ public ucar.ma2.DataType getMA2DataType() {
4149
case INT8:
4250
return ucar.ma2.DataType.BYTE;
4351
case INT16:
52+
case INT16_BE:
4453
return ucar.ma2.DataType.SHORT;
4554
case INT32:
55+
case INT32_BE:
4656
return ucar.ma2.DataType.INT;
4757
case INT64:
58+
case INT64_BE:
4859
return ucar.ma2.DataType.LONG;
4960
case UINT8:
5061
return ucar.ma2.DataType.UBYTE;
5162
case UINT16:
63+
case UINT16_BE:
5264
return ucar.ma2.DataType.USHORT;
5365
case UINT32:
66+
case UINT32_BE:
5467
return ucar.ma2.DataType.UINT;
5568
case UINT64:
69+
case UINT64_BE:
5670
return ucar.ma2.DataType.ULONG;
5771
case FLOAT32:
72+
case FLOAT32_BE:
5873
return ucar.ma2.DataType.FLOAT;
5974
case FLOAT64:
75+
case FLOAT64_BE:
6076
return ucar.ma2.DataType.DOUBLE;
6177
default:
6278
throw new RuntimeException("Unreachable");

src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java

Lines changed: 4 additions & 105 deletions
Original file line numberDiff line numberDiff line change
@@ -66,97 +66,8 @@ public static void setupUV() {
6666
}
6767
}
6868

69-
static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
70-
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
71-
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
72-
for (int i = 0; i < array.getSize(); i++) {
73-
switch (ma2Type) {
74-
case BOOLEAN:
75-
array.setBoolean(i, i % 2 == 0);
76-
break;
77-
case BYTE:
78-
case UBYTE:
79-
array.setByte(i, (byte) i);
80-
break;
81-
case SHORT:
82-
case USHORT:
83-
array.setShort(i, (short) i);
84-
break;
85-
case INT:
86-
array.setInt(i, i);
87-
break;
88-
case UINT:
89-
array.setLong(i, i & 0xFFFFFFFFL);
90-
break;
91-
case LONG:
92-
case ULONG:
93-
array.setLong(i, i);
94-
break;
95-
case FLOAT:
96-
array.setFloat(i, (float) i);
97-
break;
98-
case DOUBLE:
99-
array.setDouble(i, i);
100-
break;
101-
default:
102-
throw new IllegalArgumentException("Invalid DataType: " + dt);
103-
}
104-
}
105-
return array;
106-
}
107-
108-
static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
109-
// expected values are i for index i
110-
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
111-
for (int i = 0; i < result.getSize(); i++) {
112-
switch (ma2Type) {
113-
case BOOLEAN:
114-
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
115-
break;
116-
case BYTE:
117-
case UBYTE:
118-
Assertions.assertEquals((byte) i, result.getByte(i));
119-
break;
120-
case SHORT:
121-
case USHORT:
122-
Assertions.assertEquals((short) i, result.getShort(i));
123-
break;
124-
case INT:
125-
Assertions.assertEquals(i, result.getInt(i));
126-
break;
127-
case UINT:
128-
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
129-
break;
130-
case LONG:
131-
case ULONG:
132-
Assertions.assertEquals(i, result.getLong(i));
133-
break;
134-
case FLOAT:
135-
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
136-
break;
137-
case DOUBLE:
138-
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
139-
break;
140-
default:
141-
throw new IllegalArgumentException("Invalid DataType: " + dt);
142-
}
143-
}
144-
}
145-
14669
static Stream<Object[]> compressorAndDataTypeProviderV3() {
147-
Stream<Object[]> datatypeTests = Stream.of(
148-
DataType.BOOL,
149-
DataType.INT8,
150-
DataType.UINT8,
151-
DataType.INT16,
152-
DataType.UINT16,
153-
DataType.INT32,
154-
DataType.UINT32,
155-
DataType.INT64,
156-
DataType.UINT64,
157-
DataType.FLOAT32,
158-
DataType.FLOAT64
159-
).flatMap(dt -> Stream.of(
70+
Stream<Object[]> datatypeTests = dataTypeProviderV3().flatMap(dt -> Stream.of(
16071
new Object[]{"sharding", "end", dt},
16172
new Object[]{"blosc", "blosclz_shuffle_3", dt}
16273
));
@@ -185,19 +96,7 @@ static Stream<Object[]> compressorAndDataTypeProviderV3() {
18596
}
18697

18798
static Stream<Object[]> compressorAndDataTypeProviderV2() {
188-
Stream<Object[]> datatypeTests = Stream.of(
189-
dev.zarr.zarrjava.v2.DataType.BOOL,
190-
dev.zarr.zarrjava.v2.DataType.INT8,
191-
dev.zarr.zarrjava.v2.DataType.UINT8,
192-
dev.zarr.zarrjava.v2.DataType.INT16,
193-
dev.zarr.zarrjava.v2.DataType.UINT16,
194-
dev.zarr.zarrjava.v2.DataType.INT32,
195-
dev.zarr.zarrjava.v2.DataType.UINT32,
196-
dev.zarr.zarrjava.v2.DataType.INT64,
197-
dev.zarr.zarrjava.v2.DataType.UINT64,
198-
dev.zarr.zarrjava.v2.DataType.FLOAT32,
199-
dev.zarr.zarrjava.v2.DataType.FLOAT64
200-
).flatMap(dt -> Stream.of(
99+
Stream<Object[]> datatypeTests = dataTypeProviderV2().flatMap(dt -> Stream.of(
201100
new Object[]{"zlib", "0", dt},
202101
new Object[]{"blosc", "blosclz_shuffle_3", dt}
203102
));
@@ -305,7 +204,7 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro
305204
@MethodSource("compressorAndDataTypeProviderV2")
306205
public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException {
307206
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name());
308-
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
207+
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());
309208

310209
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle);
311210
ucar.ma2.Array result = array.read();
@@ -361,7 +260,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr
361260
assertIsTestdata(result, dt);
362261

363262
//read in zarr_python
364-
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
263+
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());
365264
}
366265

367266
@CsvSource({"0,true", "0,false", "5, true", "10, false"})

src/test/java/dev/zarr/zarrjava/ZarrTest.java

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava;
22

33
import dev.zarr.zarrjava.core.Attributes;
4+
import dev.zarr.zarrjava.v3.DataType;
45
import org.junit.jupiter.api.Assertions;
56
import org.junit.jupiter.api.BeforeAll;
67

@@ -34,6 +35,46 @@ public static void clearTestoutputFolder() throws IOException {
3435
Files.createDirectory(TESTOUTPUT);
3536
}
3637

38+
static Stream<DataType> dataTypeProviderV3() {
39+
return Stream.of(
40+
DataType.BOOL,
41+
DataType.INT8,
42+
DataType.UINT8,
43+
DataType.INT16,
44+
DataType.UINT16,
45+
DataType.INT32,
46+
DataType.UINT32,
47+
DataType.INT64,
48+
DataType.UINT64,
49+
DataType.FLOAT32,
50+
DataType.FLOAT64
51+
);
52+
}
53+
54+
static Stream<dev.zarr.zarrjava.v2.DataType> dataTypeProviderV2() {
55+
return Stream.of(
56+
dev.zarr.zarrjava.v2.DataType.BOOL,
57+
dev.zarr.zarrjava.v2.DataType.INT8,
58+
dev.zarr.zarrjava.v2.DataType.UINT8,
59+
dev.zarr.zarrjava.v2.DataType.INT16,
60+
dev.zarr.zarrjava.v2.DataType.UINT16,
61+
dev.zarr.zarrjava.v2.DataType.INT32,
62+
dev.zarr.zarrjava.v2.DataType.UINT32,
63+
dev.zarr.zarrjava.v2.DataType.INT64,
64+
dev.zarr.zarrjava.v2.DataType.UINT64,
65+
dev.zarr.zarrjava.v2.DataType.FLOAT32,
66+
dev.zarr.zarrjava.v2.DataType.FLOAT64,
67+
dev.zarr.zarrjava.v2.DataType.UINT16_BE,
68+
dev.zarr.zarrjava.v2.DataType.UINT32_BE,
69+
dev.zarr.zarrjava.v2.DataType.UINT64_BE,
70+
dev.zarr.zarrjava.v2.DataType.INT16_BE,
71+
dev.zarr.zarrjava.v2.DataType.INT32_BE,
72+
dev.zarr.zarrjava.v2.DataType.INT64_BE,
73+
dev.zarr.zarrjava.v2.DataType.FLOAT32_BE,
74+
dev.zarr.zarrjava.v2.DataType.FLOAT64_BE
75+
);
76+
}
77+
3778
protected void assertListEquals(List<Object> a, List<Object> b) {
3879
Assertions.assertEquals(a.size(), b.size());
3980
for (int i = 0; i < a.size(); i++) {
@@ -112,4 +153,83 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx
112153
);
113154
}
114155

156+
157+
protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
158+
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
159+
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
160+
for (int i = 0; i < array.getSize(); i++) {
161+
switch (ma2Type) {
162+
case BOOLEAN:
163+
array.setBoolean(i, i % 2 == 0);
164+
break;
165+
case BYTE:
166+
case UBYTE:
167+
array.setByte(i, (byte) i);
168+
break;
169+
case SHORT:
170+
case USHORT:
171+
array.setShort(i, (short) i);
172+
break;
173+
case INT:
174+
array.setInt(i, i);
175+
break;
176+
case UINT:
177+
array.setLong(i, i & 0xFFFFFFFFL);
178+
break;
179+
case LONG:
180+
case ULONG:
181+
array.setLong(i, i);
182+
break;
183+
case FLOAT:
184+
array.setFloat(i, (float) i);
185+
break;
186+
case DOUBLE:
187+
array.setDouble(i, i);
188+
break;
189+
default:
190+
throw new IllegalArgumentException("Invalid DataType: " + dt);
191+
}
192+
}
193+
return array;
194+
}
195+
196+
protected void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
197+
// expected values are i for index i
198+
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
199+
for (int i = 0; i < result.getSize(); i++) {
200+
switch (ma2Type) {
201+
case BOOLEAN:
202+
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
203+
break;
204+
case BYTE:
205+
case UBYTE:
206+
Assertions.assertEquals((byte) i, result.getByte(i));
207+
break;
208+
case SHORT:
209+
case USHORT:
210+
Assertions.assertEquals((short) i, result.getShort(i));
211+
break;
212+
case INT:
213+
Assertions.assertEquals(i, result.getInt(i));
214+
break;
215+
case UINT:
216+
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
217+
break;
218+
case LONG:
219+
case ULONG:
220+
Assertions.assertEquals(i, result.getLong(i));
221+
break;
222+
case FLOAT:
223+
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
224+
break;
225+
case DOUBLE:
226+
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
227+
break;
228+
default:
229+
throw new IllegalArgumentException("Invalid DataType: " + dt);
230+
}
231+
}
232+
}
233+
234+
115235
}

0 commit comments

Comments
 (0)