Skip to content

Commit aa8931a

Browse files
brokkoli71brokkoli71
committed
add endian tests
1 parent 663cd58 commit aa8931a

5 files changed

Lines changed: 181 additions & 114 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -150,12 +150,12 @@ public ArrayMetadata build() throws ZarrException {
150150
if (dataType == null) {
151151
throw new IllegalStateException("Please call `withDataType` first.");
152152
}
153-
153+
154154
// If chunks are not specified, calculate default chunks
155155
if (chunks == null) {
156156
chunks = Utils.calculateDefaultChunks(shape);
157157
}
158-
158+
159159
return new ArrayMetadata(
160160
2,
161161
shape,

‎src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java‎

Lines changed: 2 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -66,97 +66,8 @@ public static void setupUV() {
6666
}
6767
}
6868

69-
static ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
70-
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
71-
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
72-
for (int i = 0; i < array.getSize(); i++) {
73-
switch (ma2Type) {
74-
case BOOLEAN:
75-
array.setBoolean(i, i % 2 == 0);
76-
break;
77-
case BYTE:
78-
case UBYTE:
79-
array.setByte(i, (byte) i);
80-
break;
81-
case SHORT:
82-
case USHORT:
83-
array.setShort(i, (short) i);
84-
break;
85-
case INT:
86-
array.setInt(i, i);
87-
break;
88-
case UINT:
89-
array.setLong(i, i & 0xFFFFFFFFL);
90-
break;
91-
case LONG:
92-
case ULONG:
93-
array.setLong(i, i);
94-
break;
95-
case FLOAT:
96-
array.setFloat(i, (float) i);
97-
break;
98-
case DOUBLE:
99-
array.setDouble(i, i);
100-
break;
101-
default:
102-
throw new IllegalArgumentException("Invalid DataType: " + dt);
103-
}
104-
}
105-
return array;
106-
}
107-
108-
static void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
109-
// expected values are i for index i
110-
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
111-
for (int i = 0; i < result.getSize(); i++) {
112-
switch (ma2Type) {
113-
case BOOLEAN:
114-
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
115-
break;
116-
case BYTE:
117-
case UBYTE:
118-
Assertions.assertEquals((byte) i, result.getByte(i));
119-
break;
120-
case SHORT:
121-
case USHORT:
122-
Assertions.assertEquals((short) i, result.getShort(i));
123-
break;
124-
case INT:
125-
Assertions.assertEquals(i, result.getInt(i));
126-
break;
127-
case UINT:
128-
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
129-
break;
130-
case LONG:
131-
case ULONG:
132-
Assertions.assertEquals(i, result.getLong(i));
133-
break;
134-
case FLOAT:
135-
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
136-
break;
137-
case DOUBLE:
138-
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
139-
break;
140-
default:
141-
throw new IllegalArgumentException("Invalid DataType: " + dt);
142-
}
143-
}
144-
}
145-
14669
static Stream<Object[]> compressorAndDataTypeProviderV3() {
147-
Stream<Object[]> datatypeTests = Stream.of(
148-
DataType.BOOL,
149-
DataType.INT8,
150-
DataType.UINT8,
151-
DataType.INT16,
152-
DataType.UINT16,
153-
DataType.INT32,
154-
DataType.UINT32,
155-
DataType.INT64,
156-
DataType.UINT64,
157-
DataType.FLOAT32,
158-
DataType.FLOAT64
159-
).flatMap(dt -> Stream.of(
70+
Stream<Object[]> datatypeTests = dataTypeProviderV3().flatMap(dt -> Stream.of(
16071
new Object[]{"sharding", "end", dt},
16172
new Object[]{"blosc", "blosclz_shuffle_3", dt}
16273
));
@@ -185,27 +96,7 @@ static Stream<Object[]> compressorAndDataTypeProviderV3() {
18596
}
18697

18798
static Stream<Object[]> compressorAndDataTypeProviderV2() {
188-
Stream<Object[]> datatypeTests = Stream.of(
189-
dev.zarr.zarrjava.v2.DataType.BOOL,
190-
dev.zarr.zarrjava.v2.DataType.INT8,
191-
dev.zarr.zarrjava.v2.DataType.UINT8,
192-
dev.zarr.zarrjava.v2.DataType.INT16,
193-
dev.zarr.zarrjava.v2.DataType.UINT16,
194-
dev.zarr.zarrjava.v2.DataType.INT32,
195-
dev.zarr.zarrjava.v2.DataType.UINT32,
196-
dev.zarr.zarrjava.v2.DataType.INT64,
197-
dev.zarr.zarrjava.v2.DataType.UINT64,
198-
dev.zarr.zarrjava.v2.DataType.FLOAT32,
199-
dev.zarr.zarrjava.v2.DataType.FLOAT64,
200-
dev.zarr.zarrjava.v2.DataType.UINT16_BE,
201-
dev.zarr.zarrjava.v2.DataType.UINT32_BE,
202-
dev.zarr.zarrjava.v2.DataType.UINT64_BE,
203-
dev.zarr.zarrjava.v2.DataType.INT16_BE,
204-
dev.zarr.zarrjava.v2.DataType.INT32_BE,
205-
dev.zarr.zarrjava.v2.DataType.INT64_BE,
206-
dev.zarr.zarrjava.v2.DataType.FLOAT32_BE,
207-
dev.zarr.zarrjava.v2.DataType.FLOAT64_BE
208-
).flatMap(dt -> Stream.of(
99+
Stream<Object[]> datatypeTests = dataTypeProviderV2().flatMap(dt -> Stream.of(
209100
new Object[]{"zlib", "0", dt},
210101
new Object[]{"blosc", "blosclz_shuffle_3", dt}
211102
));

‎src/test/java/dev/zarr/zarrjava/ZarrTest.java‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package dev.zarr.zarrjava;
22

33
import dev.zarr.zarrjava.core.Attributes;
4+
import dev.zarr.zarrjava.v3.DataType;
45
import org.junit.jupiter.api.Assertions;
56
import org.junit.jupiter.api.BeforeAll;
67

@@ -34,6 +35,46 @@ public static void clearTestoutputFolder() throws IOException {
3435
Files.createDirectory(TESTOUTPUT);
3536
}
3637

38+
static Stream<DataType> dataTypeProviderV3() {
39+
return Stream.of(
40+
DataType.BOOL,
41+
DataType.INT8,
42+
DataType.UINT8,
43+
DataType.INT16,
44+
DataType.UINT16,
45+
DataType.INT32,
46+
DataType.UINT32,
47+
DataType.INT64,
48+
DataType.UINT64,
49+
DataType.FLOAT32,
50+
DataType.FLOAT64
51+
);
52+
}
53+
54+
static Stream<dev.zarr.zarrjava.v2.DataType> dataTypeProviderV2() {
55+
return Stream.of(
56+
dev.zarr.zarrjava.v2.DataType.BOOL,
57+
dev.zarr.zarrjava.v2.DataType.INT8,
58+
dev.zarr.zarrjava.v2.DataType.UINT8,
59+
dev.zarr.zarrjava.v2.DataType.INT16,
60+
dev.zarr.zarrjava.v2.DataType.UINT16,
61+
dev.zarr.zarrjava.v2.DataType.INT32,
62+
dev.zarr.zarrjava.v2.DataType.UINT32,
63+
dev.zarr.zarrjava.v2.DataType.INT64,
64+
dev.zarr.zarrjava.v2.DataType.UINT64,
65+
dev.zarr.zarrjava.v2.DataType.FLOAT32,
66+
dev.zarr.zarrjava.v2.DataType.FLOAT64,
67+
dev.zarr.zarrjava.v2.DataType.UINT16_BE,
68+
dev.zarr.zarrjava.v2.DataType.UINT32_BE,
69+
dev.zarr.zarrjava.v2.DataType.UINT64_BE,
70+
dev.zarr.zarrjava.v2.DataType.INT16_BE,
71+
dev.zarr.zarrjava.v2.DataType.INT32_BE,
72+
dev.zarr.zarrjava.v2.DataType.INT64_BE,
73+
dev.zarr.zarrjava.v2.DataType.FLOAT32_BE,
74+
dev.zarr.zarrjava.v2.DataType.FLOAT64_BE
75+
);
76+
}
77+
3778
protected void assertListEquals(List<Object> a, List<Object> b) {
3879
Assertions.assertEquals(a.size(), b.size());
3980
for (int i = 0; i < a.size(); i++) {
@@ -112,4 +153,83 @@ protected void assertContainsTestAttributes(Attributes attributes) throws ZarrEx
112153
);
113154
}
114155

156+
157+
protected ucar.ma2.Array testdata(dev.zarr.zarrjava.core.DataType dt) {
158+
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
159+
ucar.ma2.Array array = ucar.ma2.Array.factory(ma2Type, new int[]{16, 16, 16});
160+
for (int i = 0; i < array.getSize(); i++) {
161+
switch (ma2Type) {
162+
case BOOLEAN:
163+
array.setBoolean(i, i % 2 == 0);
164+
break;
165+
case BYTE:
166+
case UBYTE:
167+
array.setByte(i, (byte) i);
168+
break;
169+
case SHORT:
170+
case USHORT:
171+
array.setShort(i, (short) i);
172+
break;
173+
case INT:
174+
array.setInt(i, i);
175+
break;
176+
case UINT:
177+
array.setLong(i, i & 0xFFFFFFFFL);
178+
break;
179+
case LONG:
180+
case ULONG:
181+
array.setLong(i, i);
182+
break;
183+
case FLOAT:
184+
array.setFloat(i, (float) i);
185+
break;
186+
case DOUBLE:
187+
array.setDouble(i, i);
188+
break;
189+
default:
190+
throw new IllegalArgumentException("Invalid DataType: " + dt);
191+
}
192+
}
193+
return array;
194+
}
195+
196+
protected void assertIsTestdata(ucar.ma2.Array result, dev.zarr.zarrjava.core.DataType dt) {
197+
// expected values are i for index i
198+
ucar.ma2.DataType ma2Type = dt.getMA2DataType();
199+
for (int i = 0; i < result.getSize(); i++) {
200+
switch (ma2Type) {
201+
case BOOLEAN:
202+
Assertions.assertEquals(i % 2 == 0, result.getBoolean(i));
203+
break;
204+
case BYTE:
205+
case UBYTE:
206+
Assertions.assertEquals((byte) i, result.getByte(i));
207+
break;
208+
case SHORT:
209+
case USHORT:
210+
Assertions.assertEquals((short) i, result.getShort(i));
211+
break;
212+
case INT:
213+
Assertions.assertEquals(i, result.getInt(i));
214+
break;
215+
case UINT:
216+
Assertions.assertEquals(i & 0xFFFFFFFFL, result.getLong(i));
217+
break;
218+
case LONG:
219+
case ULONG:
220+
Assertions.assertEquals(i, result.getLong(i));
221+
break;
222+
case FLOAT:
223+
Assertions.assertEquals((float) i, result.getFloat(i), 1e-6);
224+
break;
225+
case DOUBLE:
226+
Assertions.assertEquals(i, result.getDouble(i), 1e-12);
227+
break;
228+
default:
229+
throw new IllegalArgumentException("Invalid DataType: " + dt);
230+
}
231+
}
232+
}
233+
234+
115235
}

‎src/test/java/dev/zarr/zarrjava/ZarrV2Test.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
import java.util.stream.Stream;
2727

2828
import static dev.zarr.zarrjava.core.Node.ZARRAY;
29+
import static dev.zarr.zarrjava.utils.Utils.toLongArray;
2930

3031
public class ZarrV2Test extends ZarrTest {
3132
static Stream<Function<ArrayMetadataBuilder, ArrayMetadataBuilder>> compressorBuilder() {
@@ -581,4 +582,23 @@ public void testDefaultChunkShape() throws IOException, ZarrException {
581582
Assertions.assertTrue(mixedArray.metadata().chunks[2] > 0);
582583
Assertions.assertTrue(mixedArray.metadata().chunks[2] <= 2048);
583584
}
585+
586+
@ParameterizedTest
587+
@MethodSource("dataTypeProviderV2")
588+
public void testEndianness(DataType dataType) throws IOException, ZarrException {
589+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testEndiannessV2").resolve(dataType.name());
590+
ucar.ma2.Array testData = testdata(dataType);
591+
592+
Array array = Array.create(
593+
storeHandle,
594+
Array.metadataBuilder()
595+
.withShape(toLongArray(testData.getShape()))
596+
.withDataType(dataType)
597+
.build()
598+
);
599+
array.write(testData);
600+
Array reopenedArray = Array.open(storeHandle);
601+
ucar.ma2.Array readData = reopenedArray.read();
602+
assertIsTestdata(readData, dataType);
603+
}
584604
}

‎src/test/java/dev/zarr/zarrjava/ZarrV3Test.java‎

Lines changed: 37 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040

4141
import static dev.zarr.zarrjava.core.ArrayMetadata.parseFillValue;
4242
import static dev.zarr.zarrjava.core.Node.ZARR_JSON;
43+
import static dev.zarr.zarrjava.utils.Utils.toLongArray;
4344
import static org.junit.Assert.assertThrows;
4445

4546
public class ZarrV3Test extends ZarrTest {
@@ -118,6 +119,23 @@ static Stream<Arguments> unalignedArrayAccessProvider() {
118119
return builder.build();
119120
}
120121

122+
static Stream<Arguments> dataTypeAndEndianProvider() {
123+
return Stream.of(
124+
Arguments.of(DataType.INT16, BytesCodec.Endian.LITTLE),
125+
Arguments.of(DataType.INT16, BytesCodec.Endian.BIG),
126+
Arguments.of(DataType.UINT16, BytesCodec.Endian.LITTLE),
127+
Arguments.of(DataType.UINT16, BytesCodec.Endian.BIG),
128+
Arguments.of(DataType.INT32, BytesCodec.Endian.LITTLE),
129+
Arguments.of(DataType.INT32, BytesCodec.Endian.BIG),
130+
Arguments.of(DataType.UINT32, BytesCodec.Endian.LITTLE),
131+
Arguments.of(DataType.UINT32, BytesCodec.Endian.BIG),
132+
Arguments.of(DataType.FLOAT32, BytesCodec.Endian.LITTLE),
133+
Arguments.of(DataType.FLOAT32, BytesCodec.Endian.BIG),
134+
Arguments.of(DataType.FLOAT64, BytesCodec.Endian.LITTLE),
135+
Arguments.of(DataType.FLOAT64, BytesCodec.Endian.BIG)
136+
);
137+
}
138+
121139
@ParameterizedTest
122140
@MethodSource("invalidCodecBuilder")
123141
public void testCheckInvalidCodecConfiguration(Function<CodecBuilder, CodecBuilder> codecBuilder) {
@@ -997,7 +1015,7 @@ public void testLargeArrayWithOffsetBeyondMaxInt() throws IOException, ZarrExcep
9971015
array.write(new long[]{0, 0}, smallChunk);
9981016

9991017
// Write a small chunk at position [1, Integer.MAX_VALUE + 1]
1000-
long beyondIntMax = (long)(Integer.MAX_VALUE) + 1;
1018+
long beyondIntMax = (long) (Integer.MAX_VALUE) + 1;
10011019
long[] offset = new long[]{1, beyondIntMax};
10021020
Arrays.fill(testData, 200);
10031021
smallChunk = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{1, 1000}, testData);
@@ -1016,4 +1034,22 @@ public void testLargeArrayWithOffsetBeyondMaxInt() throws IOException, ZarrExcep
10161034
Assertions.assertEquals(largeSize, array.metadata().shape[0]);
10171035
Assertions.assertEquals(largeSize, array.metadata().shape[1]);
10181036
}
1037+
1038+
@ParameterizedTest
1039+
@MethodSource("dataTypeAndEndianProvider")
1040+
public void testEndianness(DataType dataType, BytesCodec.Endian endian) throws IOException, ZarrException {
1041+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testEndiannessV3").resolve(dataType.name()).resolve(endian.name());
1042+
ucar.ma2.Array testData = testdata(dataType);
1043+
1044+
ArrayMetadata metadata = Array.metadataBuilder()
1045+
.withShape(toLongArray(testData.getShape()))
1046+
.withDataType(dataType)
1047+
.withCodecs(c -> c.withBytes(endian))
1048+
.build();
1049+
Array array = Array.create(storeHandle, metadata);
1050+
array.write(testData);
1051+
Array reopenedArray = Array.open(storeHandle);
1052+
ucar.ma2.Array readData = reopenedArray.read();
1053+
assertIsTestdata(readData, dataType);
1054+
}
10191055
}

0 commit comments

Comments
 (0)