Skip to content

Commit 663cd58

Browse files
brokkoli71brokkoli71
committed
add big endian dtypes to v2
1 parent 4612451 commit 663cd58

6 files changed

Lines changed: 38 additions & 14 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/core/ArrayMetadata.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -36,11 +36,11 @@ public static Object parseFillValue(Object fillValue, @Nonnull DataType dataType
3636
}
3737
boolean dataTypeIsBool = dataType == dev.zarr.zarrjava.v3.DataType.BOOL || dataType == dev.zarr.zarrjava.v2.DataType.BOOL;
3838
boolean dataTypeIsByte = dataType == dev.zarr.zarrjava.v3.DataType.INT8 || dataType == dev.zarr.zarrjava.v2.DataType.INT8 || dataType == dev.zarr.zarrjava.v3.DataType.UINT8 || dataType == dev.zarr.zarrjava.v2.DataType.UINT8;
39-
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16;
40-
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32;
41-
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64;
42-
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32;
43-
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64;
39+
boolean dataTypeIsShort = dataType == dev.zarr.zarrjava.v3.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16 || dataType == dev.zarr.zarrjava.v2.DataType.INT16_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT16 || dataType == dev.zarr.zarrjava.v2.DataType.UINT16|| dataType == dev.zarr.zarrjava.v2.DataType.UINT16_BE;
40+
boolean dataTypeIsInt = dataType == dev.zarr.zarrjava.v3.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32 || dataType == dev.zarr.zarrjava.v2.DataType.INT32_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32 || dataType == dev.zarr.zarrjava.v2.DataType.UINT32_BE;
41+
boolean dataTypeIsLong = dataType == dev.zarr.zarrjava.v3.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64 || dataType == dev.zarr.zarrjava.v2.DataType.INT64_BE || dataType == dev.zarr.zarrjava.v3.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64 || dataType == dev.zarr.zarrjava.v2.DataType.UINT64_BE;
42+
boolean dataTypeIsFloat = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT32_BE;
43+
boolean dataTypeIsDouble = dataType == dev.zarr.zarrjava.v3.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64 || dataType == dev.zarr.zarrjava.v2.DataType.FLOAT64_BE;
4444

4545
if (fillValue instanceof Boolean) {
4646
Boolean fillValueBool = (Boolean) fillValue;

‎src/main/java/dev/zarr/zarrjava/v2/DataType.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,15 @@ public enum DataType implements dev.zarr.zarrjava.core.DataType {
1313
UINT32("u4", Endianness.LITTLE),
1414
UINT64("u8", Endianness.LITTLE),
1515
FLOAT32("f4", Endianness.LITTLE),
16-
FLOAT64("f8", Endianness.LITTLE);
16+
FLOAT64("f8", Endianness.LITTLE),
17+
INT16_BE("i2", Endianness.BIG),
18+
INT32_BE("i4", Endianness.BIG),
19+
INT64_BE("i8", Endianness.BIG),
20+
UINT16_BE("u2", Endianness.BIG),
21+
UINT32_BE("u4", Endianness.BIG),
22+
UINT64_BE("u8", Endianness.BIG),
23+
FLOAT32_BE("f4", Endianness.BIG),
24+
FLOAT64_BE("f8", Endianness.BIG);
1725

1826
private final String dtype;
1927
private final Endianness endianness;
@@ -41,22 +49,30 @@ public ucar.ma2.DataType getMA2DataType() {
4149
case INT8:
4250
return ucar.ma2.DataType.BYTE;
4351
case INT16:
52+
case INT16_BE:
4453
return ucar.ma2.DataType.SHORT;
4554
case INT32:
55+
case INT32_BE:
4656
return ucar.ma2.DataType.INT;
4757
case INT64:
58+
case INT64_BE:
4859
return ucar.ma2.DataType.LONG;
4960
case UINT8:
5061
return ucar.ma2.DataType.UBYTE;
5162
case UINT16:
63+
case UINT16_BE:
5264
return ucar.ma2.DataType.USHORT;
5365
case UINT32:
66+
case UINT32_BE:
5467
return ucar.ma2.DataType.UINT;
5568
case UINT64:
69+
case UINT64_BE:
5670
return ucar.ma2.DataType.ULONG;
5771
case FLOAT32:
72+
case FLOAT32_BE:
5873
return ucar.ma2.DataType.FLOAT;
5974
case FLOAT64:
75+
case FLOAT64_BE:
6076
return ucar.ma2.DataType.DOUBLE;
6177
default:
6278
throw new RuntimeException("Unreachable");

‎src/test/java/dev/zarr/zarrjava/ZarrPythonTests.java‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -196,7 +196,15 @@ static Stream<Object[]> compressorAndDataTypeProviderV2() {
196196
dev.zarr.zarrjava.v2.DataType.INT64,
197197
dev.zarr.zarrjava.v2.DataType.UINT64,
198198
dev.zarr.zarrjava.v2.DataType.FLOAT32,
199-
dev.zarr.zarrjava.v2.DataType.FLOAT64
199+
dev.zarr.zarrjava.v2.DataType.FLOAT64,
200+
dev.zarr.zarrjava.v2.DataType.UINT16_BE,
201+
dev.zarr.zarrjava.v2.DataType.UINT32_BE,
202+
dev.zarr.zarrjava.v2.DataType.UINT64_BE,
203+
dev.zarr.zarrjava.v2.DataType.INT16_BE,
204+
dev.zarr.zarrjava.v2.DataType.INT32_BE,
205+
dev.zarr.zarrjava.v2.DataType.INT64_BE,
206+
dev.zarr.zarrjava.v2.DataType.FLOAT32_BE,
207+
dev.zarr.zarrjava.v2.DataType.FLOAT64_BE
200208
).flatMap(dt -> Stream.of(
201209
new Object[]{"zlib", "0", dt},
202210
new Object[]{"blosc", "blosclz_shuffle_3", dt}
@@ -305,7 +313,7 @@ public void testWriteV3(String codec, String codecParam, DataType dataType) thro
305313
@MethodSource("compressorAndDataTypeProviderV2")
306314
public void testReadV2(String compressor, String compressorParam, dev.zarr.zarrjava.v2.DataType dt) throws IOException, ZarrException, InterruptedException {
307315
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testReadV2", compressor, compressorParam, dt.name());
308-
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
316+
run_python_script("zarr_python_write_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());
309317

310318
dev.zarr.zarrjava.v2.Array array = dev.zarr.zarrjava.v2.Array.open(storeHandle);
311319
ucar.ma2.Array result = array.read();
@@ -361,7 +369,7 @@ public void testWriteV2(String compressor, String compressorParam, dev.zarr.zarr
361369
assertIsTestdata(result, dt);
362370

363371
//read in zarr_python
364-
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.name().toLowerCase(), storeHandle.toPath().toString());
372+
run_python_script("zarr_python_read_v2.py", compressor, compressorParam, dt.getValue(), storeHandle.toPath().toString());
365373
}
366374

367375
@CsvSource({"0,true", "0,false", "5, true", "10, false"})

‎src/test/java/dev/zarr/zarrjava/ZarrV2Test.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ public void testCreateZlib(int level) throws IOException, ZarrException {
109109
}
110110

111111
@ParameterizedTest
112-
@ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64"})
112+
@ValueSource(strings = {"BOOL", "INT8", "UINT8", "INT16", "UINT16", "INT32", "UINT32", "INT64", "UINT64", "FLOAT32", "FLOAT64", "INT16_BE", "UINT16_BE", "INT32_BE", "UINT32_BE", "INT64_BE", "UINT64_BE", "FLOAT32_BE", "FLOAT64_BE"})
113113
public void testNoFillValue(DataType dataType) throws IOException, ZarrException {
114114
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_no_fillvalue", dataType.name());
115115

@@ -409,7 +409,7 @@ public void testResizeArrayShrink() throws IOException, ZarrException {
409409
ucar.ma2.Array data = array.read();
410410
int[] expectedData = new int[5 * 5];
411411
for (int i = 0; i < 5; i++) {
412-
System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5);
412+
System.arraycopy(testData, i * 10, expectedData, i * 5, 5);
413413
}
414414
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
415415
}
@@ -449,7 +449,7 @@ public void testResizeArrayShrinkWithChunkCleanup() throws IOException, ZarrExce
449449
ucar.ma2.Array data = array.read();
450450
int[] expectedData = new int[5 * 5];
451451
for (int i = 0; i < 5; i++) {
452-
System.arraycopy(testData, i * 10 + 0, expectedData, i * 5 + 0, 5);
452+
System.arraycopy(testData, i * 10, expectedData, i * 5, 5);
453453
}
454454
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
455455
}

‎src/test/python-scripts/zarr_python_read_v2.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
dtype = sys.argv[3]
1313
store_path = Path(sys.argv[4])
1414

15-
if dtype == 'bool':
15+
if 'b1' in dtype:
1616
expected_data = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0
1717
else:
1818
expected_data = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16)

‎src/test/python-scripts/zarr_python_write_v2.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
dtype = sys.argv[3]
1313
store_path = Path(sys.argv[4])
1414

15-
if dtype == 'bool':
15+
if 'b1' in dtype:
1616
testdata = np.arange(16 * 16 * 16, dtype='uint8').reshape(16, 16, 16) % 2 == 0
1717
else:
1818
testdata = np.arange(16 * 16 * 16, dtype=dtype).reshape(16, 16, 16)

0 commit comments

Comments
 (0)