Skip to content

Commit a57b094

Browse files
brokkoli71brokkoli71
committed
add tests
1 parent 93b10bb commit a57b094

9 files changed

Lines changed: 269 additions & 6 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/utils/IndexingUtils.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,7 +87,7 @@ public static ChunkProjection computeProjection(
8787

8888
if (selOffset[dimIdx] + selShape[dimIdx] > dimLimit) {
8989
// selection ends after current chunk
90-
shape[dimIdx] = (int) (chunkShape[dimIdx] - (selOffset[dimIdx] % chunkShape[dimIdx]));
90+
shape[dimIdx] = chunkShape[dimIdx] - chunkOffset[dimIdx];
9191
} else {
9292
// selection ends within current chunk
9393
shape[dimIdx] = (int) (selOffset[dimIdx] + selShape[dimIdx] - dimOffset

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,8 @@ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOExc
245245
* @throws IOException throws IOException if the new metadata cannot be serialized
246246
*/
247247
public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
248-
return setAttributes(attributeMapper.apply(metadata.attributes));
248+
Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
249+
return setAttributes(attributeMapper.apply(currentAttributes));
249250
}
250251

251252
@Override

‎src/main/java/dev/zarr/zarrjava/v2/Group.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -260,7 +260,8 @@ public Group setAttributes(Attributes newAttributes) throws ZarrException, IOExc
260260
*/
261261
public Group updateAttributes(Function<Attributes, Attributes> attributeMapper)
262262
throws ZarrException, IOException {
263-
return setAttributes(attributeMapper.apply(metadata.attributes));
263+
Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
264+
return setAttributes(attributeMapper.apply(currentAttributes));
264265
}
265266

266267

‎src/main/java/dev/zarr/zarrjava/v3/Array.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,8 @@ public Array setAttributes(Attributes newAttributes) throws ZarrException, IOExc
248248
* @throws IOException throws IOException if the new metadata cannot be serialized
249249
*/
250250
public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
251-
return setAttributes(attributeMapper.apply(metadata.attributes));
251+
Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
252+
return setAttributes(attributeMapper.apply(currentAttributes));
252253
}
253254

254255
@Override

‎src/main/java/dev/zarr/zarrjava/v3/Group.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -278,7 +278,8 @@ private Group writeMetadata(GroupMetadata newGroupMetadata) throws IOException {
278278
*/
279279
public Group updateAttributes(Function<Attributes, Attributes> attributeMapper)
280280
throws ZarrException, IOException {
281-
return setAttributes(attributeMapper.apply(metadata.attributes));
281+
Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
282+
return setAttributes(attributeMapper.apply(currentAttributes));
282283
}
283284

284285
/**

‎src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java‎

Lines changed: 154 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,154 @@
1+
package dev.zarr.zarrjava;
2+
3+
import dev.zarr.zarrjava.store.FilesystemStore;
4+
import dev.zarr.zarrjava.store.StoreHandle;
5+
import dev.zarr.zarrjava.v3.Array;
6+
import dev.zarr.zarrjava.v3.DataType;
7+
import org.junit.jupiter.api.Assertions;
8+
import org.junit.jupiter.api.Test;
9+
10+
import java.io.IOException;
11+
import java.nio.file.Path;
12+
import java.util.ArrayList;
13+
import java.util.List;
14+
import java.util.concurrent.*;
15+
16+
public class ParallelWriteTest extends ZarrTest {
17+
18+
@Test
19+
public void testParallelWriteDataSafety() throws IOException, ZarrException {
20+
// Test internal parallelism of write method (using parallel=true)
21+
Path path = TESTOUTPUT.resolve("parallel_write_safety");
22+
StoreHandle storeHandle = new FilesystemStore(path).resolve();
23+
24+
int shape = 1000;
25+
int chunk = 100;
26+
27+
Array array = Array.create(storeHandle, Array.metadataBuilder()
28+
.withShape(shape, shape)
29+
.withDataType(DataType.INT32)
30+
.withChunkShape(chunk, chunk)
31+
.withFillValue(0)
32+
.build());
33+
34+
int[] data = new int[shape * shape];
35+
// Fill with some deterministic pattern
36+
for (int i = 0; i < shape * shape; i++) {
37+
data[i] = i;
38+
}
39+
40+
ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data);
41+
42+
// Write in parallel
43+
array.write(outputData, true);
44+
45+
// Read back
46+
ucar.ma2.Array readData = array.read();
47+
int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT);
48+
49+
Assertions.assertArrayEquals(data, readArr, "Data read back should match data written in parallel");
50+
}
51+
52+
@Test
53+
public void testParallelWriteWithSharding() throws IOException, ZarrException {
54+
// Test internal parallelism with Sharding (nested chunks + shared codec state potential)
55+
Path path = TESTOUTPUT.resolve("parallel_write_sharding");
56+
StoreHandle storeHandle = new FilesystemStore(path).resolve();
57+
58+
int shape = 128; // 128x128
59+
int shardSize = 64; // Shards are 64x64
60+
int innerChunk = 32; // Inner chunks 32x32
61+
62+
// Metadata with sharding
63+
// With shape 128 and shardSize 64, we have 2x2 = 4 shards.
64+
// Array.write(parallel=true) will likely process these shards concurrently.
65+
dev.zarr.zarrjava.v3.ArrayMetadata metadata = Array.metadataBuilder()
66+
.withShape(shape, shape)
67+
.withDataType(DataType.INT32)
68+
.withChunkShape(shardSize, shardSize) // This sets the shard shape (outer chunks)
69+
.withCodecs(c -> c.withSharding(new int[]{innerChunk, innerChunk}, c2 -> c2.withBytes("LITTLE")))
70+
.withFillValue(0)
71+
.build();
72+
73+
Array array = Array.create(storeHandle, metadata);
74+
75+
int[] data = new int[shape * shape];
76+
for (int i = 0; i < shape * shape; i++) {
77+
data[i] = i;
78+
}
79+
80+
ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data);
81+
82+
// Write in parallel
83+
array.write(outputData, true);
84+
85+
ucar.ma2.Array readData = array.read();
86+
int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT);
87+
88+
Assertions.assertArrayEquals(data, readArr, "Sharded data written in parallel should match");
89+
}
90+
91+
@Test
92+
public void testConcurrentWritesDifferentChunks() throws IOException, ZarrException, InterruptedException, ExecutionException {
93+
// Test external parallelism (multiple threads calling write on same Array instance)
94+
Path path = TESTOUTPUT.resolve("concurrent_write_safety");
95+
StoreHandle storeHandle = new FilesystemStore(path).resolve();
96+
97+
int chunksX = 10;
98+
int chunksY = 10;
99+
int chunkSize = 50;
100+
int shapeX = chunksX * chunkSize;
101+
int shapeY = chunksY * chunkSize;
102+
103+
Array array = Array.create(storeHandle, Array.metadataBuilder()
104+
.withShape(shapeX, shapeY)
105+
.withDataType(DataType.INT32)
106+
.withChunkShape(chunkSize, chunkSize)
107+
.withFillValue(-1)
108+
.build());
109+
110+
ExecutorService executor = Executors.newFixedThreadPool(8);
111+
List<Callable<Void>> tasks = new ArrayList<>();
112+
113+
for (int i = 0; i < chunksX; i++) {
114+
for (int j = 0; j < chunksY; j++) {
115+
final int cx = i;
116+
final int cy = j;
117+
tasks.add(() -> {
118+
int[] chunkData = new int[chunkSize * chunkSize];
119+
int val = cx * chunksY + cy; // Unique value per chunk
120+
java.util.Arrays.fill(chunkData, val);
121+
122+
ucar.ma2.Array ucarArray = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{chunkSize, chunkSize}, chunkData);
123+
124+
// Write to specific chunk offset
125+
long[] offset = new long[]{cx * chunkSize, cy * chunkSize};
126+
// Use internal parallelism false to isolate external concurrency test mechanism
127+
array.write(offset, ucarArray, false);
128+
return null;
129+
});
130+
}
131+
}
132+
133+
List<Future<Void>> futures = executor.invokeAll(tasks);
134+
135+
for (Future<Void> f : futures) {
136+
f.get(); // Check for exceptions
137+
}
138+
executor.shutdown();
139+
140+
// Verification
141+
ucar.ma2.Array readData = array.read();
142+
for (int i = 0; i < chunksX; i++) {
143+
for (int j = 0; j < chunksY; j++) {
144+
int expectedVal = i * chunksY + j;
145+
int originX = i * chunkSize;
146+
int originY = j * chunkSize;
147+
148+
// Verify a pixel in the chunk
149+
int val = readData.getInt(readData.getIndex().set(originX, originY));
150+
Assertions.assertEquals(expectedVal, val, "Value at chunk " + i + "," + j + " mismatch");
151+
}
152+
}
153+
}
154+
}

‎src/test/java/dev/zarr/zarrjava/TestUtils.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,5 @@ public void testInversePermutation() {
2626
Assertions.assertArrayEquals(new int[]{0, 3, 2, 4, 1}, inversePermutation(new int[]{0, 4, 2, 1, 3}));
2727
Assertions.assertFalse(Arrays.equals(new int[]{2, 0, 1}, inversePermutation(new int[]{2, 0, 1})));
2828
}
29-
3029
}
3130

‎src/test/java/dev/zarr/zarrjava/ZarrV2Test.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,31 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException {
331331
assertContainsTestAttributes(array.metadata().attributes());
332332
}
333333

334+
@Test
335+
public void testUpdateAttributesBehavior() throws IOException, ZarrException {
336+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV2");
337+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
338+
.withShape(10, 10)
339+
.withDataType(DataType.UINT8)
340+
.withChunks(5, 5)
341+
.withAttributes(new Attributes(b -> b.set("key1", "val1")))
342+
.build();
343+
344+
Array array1 = Array.create(storeHandle, arrayMetadata);
345+
Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2"));
346+
347+
Assertions.assertNotSame(array1, array2);
348+
Assertions.assertEquals("val1", array1.metadata().attributes().get("key1"));
349+
Assertions.assertNull(array1.metadata().attributes().get("key2"));
350+
351+
Assertions.assertEquals("val1", array2.metadata().attributes().get("key1"));
352+
Assertions.assertEquals("val2", array2.metadata().attributes().get("key2"));
353+
354+
// Re-opening should show the updated attributes
355+
Array array3 = Array.open(storeHandle);
356+
Assertions.assertEquals("val2", array3.metadata().attributes().get("key2"));
357+
}
358+
334359
@Test
335360
public void testResizeArray() throws IOException, ZarrException {
336361
int[] testData = new int[10 * 10];
@@ -359,6 +384,34 @@ public void testResizeArray() throws IOException, ZarrException {
359384
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
360385
}
361386

387+
@Test
388+
public void testResizeArrayShrink() throws IOException, ZarrException {
389+
int[] testData = new int[10 * 10];
390+
Arrays.setAll(testData, p -> p);
391+
392+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV2");
393+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
394+
.withShape(10, 10)
395+
.withDataType(DataType.UINT32)
396+
.withChunks(5, 5)
397+
.build();
398+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
399+
Array array = Array.create(storeHandle, arrayMetadata);
400+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
401+
402+
array = array.resize(new long[]{5, 5});
403+
Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
404+
405+
ucar.ma2.Array data = array.read();
406+
int[] expectedData = new int[5 * 5];
407+
for (int i = 0; i < 5; i++) {
408+
for (int j = 0; j < 5; j++) {
409+
expectedData[i * 5 + j] = testData[i * 10 + j];
410+
}
411+
}
412+
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
413+
}
414+
362415
@Test
363416
public void testGroupAttributes() throws IOException, ZarrException {
364417
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2");

‎src/test/java/dev/zarr/zarrjava/ZarrV3Test.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -694,6 +694,31 @@ public void testSetAndUpdateAttributes() throws IOException, ZarrException {
694694
assertContainsTestAttributes(array.metadata().attributes());
695695
}
696696

697+
@Test
698+
public void testUpdateAttributesBehavior() throws IOException, ZarrException {
699+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV3");
700+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
701+
.withShape(10, 10)
702+
.withDataType(DataType.UINT8)
703+
.withChunkShape(5, 5)
704+
.withAttributes(new Attributes(b -> b.set("key1", "val1")))
705+
.build();
706+
707+
Array array1 = Array.create(storeHandle, arrayMetadata);
708+
Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2"));
709+
710+
Assertions.assertNotSame(array1, array2);
711+
Assertions.assertEquals("val1", array1.metadata().attributes().get("key1"));
712+
Assertions.assertNull(array1.metadata().attributes().get("key2"));
713+
714+
Assertions.assertEquals("val1", array2.metadata().attributes().get("key1"));
715+
Assertions.assertEquals("val2", array2.metadata().attributes().get("key2"));
716+
717+
// Re-opening should show the updated attributes
718+
Array array3 = Array.open(storeHandle);
719+
Assertions.assertEquals("val2", array3.metadata().attributes().get("key2"));
720+
}
721+
697722
@Test
698723
public void testResizeArray() throws IOException, ZarrException {
699724
int[] testData = new int[10 * 10];
@@ -722,6 +747,34 @@ public void testResizeArray() throws IOException, ZarrException {
722747
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
723748
}
724749

750+
@Test
751+
public void testResizeArrayShrink() throws IOException, ZarrException {
752+
int[] testData = new int[10 * 10];
753+
Arrays.setAll(testData, p -> p);
754+
755+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV3");
756+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
757+
.withShape(10, 10)
758+
.withDataType(DataType.UINT32)
759+
.withChunkShape(5, 5)
760+
.build();
761+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
762+
Array array = Array.create(storeHandle, arrayMetadata);
763+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
764+
765+
array = array.resize(new long[]{5, 5});
766+
Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
767+
768+
ucar.ma2.Array data = array.read();
769+
int[] expectedData = new int[5 * 5];
770+
for (int i = 0; i < 5; i++) {
771+
for (int j = 0; j < 5; j++) {
772+
expectedData[i * 5 + j] = testData[i * 10 + j];
773+
}
774+
}
775+
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
776+
}
777+
725778
@Test
726779
public void testGroupAttributes() throws IOException, ZarrException {
727780
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3");

0 commit comments

Comments
 (0)