Commit 71c36ec
authored
Documentation (#55)
* first draft
* add tests
diff --git c/src/main/java/dev/zarr/zarrjava/v2/Array.java i/src/main/java/dev/zarr/zarrjava/v2/Array.java
index a34c85a..c5ed44b 100644
--- c/src/main/java/dev/zarr/zarrjava/v2/Array.java
+++ i/src/main/java/dev/zarr/zarrjava/v2/Array.java
@@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node {
* @throws IOException throws IOException if the new metadata cannot be serialized
*/
public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
- return setAttributes(attributeMapper.apply(metadata.attributes));
+ Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
+ return setAttributes(attributeMapper.apply(currentAttributes));
}
@OverRide
diff --git c/src/main/java/dev/zarr/zarrjava/v2/Group.java i/src/main/java/dev/zarr/zarrjava/v2/Group.java
index d3229e4..73b8cdf 100644
--- c/src/main/java/dev/zarr/zarrjava/v2/Group.java
+++ i/src/main/java/dev/zarr/zarrjava/v2/Group.java
@@ -283,7 +283,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node {
*/
public Group updateAttributes(Function<Attributes, Attributes> attributeMapper)
throws ZarrException, IOException {
- return setAttributes(attributeMapper.apply(metadata.attributes));
+ Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
+ return setAttributes(attributeMapper.apply(currentAttributes));
}
diff --git c/src/main/java/dev/zarr/zarrjava/v3/Array.java i/src/main/java/dev/zarr/zarrjava/v3/Array.java
index ea029db..45060e9 100644
--- c/src/main/java/dev/zarr/zarrjava/v3/Array.java
+++ i/src/main/java/dev/zarr/zarrjava/v3/Array.java
@@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node {
* @throws IOException throws IOException if the new metadata cannot be serialized
*/
public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
- return setAttributes(attributeMapper.apply(metadata.attributes));
+ Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
+ return setAttributes(attributeMapper.apply(currentAttributes));
}
@OverRide
diff --git c/src/main/java/dev/zarr/zarrjava/v3/Group.java i/src/main/java/dev/zarr/zarrjava/v3/Group.java
index 1d3aa6b..8b1a81b 100644
--- c/src/main/java/dev/zarr/zarrjava/v3/Group.java
+++ i/src/main/java/dev/zarr/zarrjava/v3/Group.java
@@ -289,7 +289,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node {
* @throws IOException if the metadata cannot be serialized
*/
public Group updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException {
- return setAttributes(attributeMapper.apply(metadata.attributes));
+ Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes();
+ return setAttributes(attributeMapper.apply(currentAttributes));
}
/**
diff --git c/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java
new file mode 100644
index 0000000..60db2ee
--- /dev/null
+++ i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java
@@ -0,0 +1,154 @@
+package dev.zarr.zarrjava;
+
+import dev.zarr.zarrjava.store.FilesystemStore;
+import dev.zarr.zarrjava.store.StoreHandle;
+import dev.zarr.zarrjava.v3.Array;
+import dev.zarr.zarrjava.v3.DataType;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.*;
+
+public class ParallelWriteTest extends ZarrTest {
+
+ @test
+ public void testParallelWriteDataSafety() throws IOException, ZarrException {
+ // Test internal parallelism of write method (using parallel=true)
+ Path path = TESTOUTPUT.resolve("parallel_write_safety");
+ StoreHandle storeHandle = new FilesystemStore(path).resolve();
+
+ int shape = 1000;
+ int chunk = 100;
+
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(shape, shape)
+ .withDataType(DataType.INT32)
+ .withChunkShape(chunk, chunk)
+ .withFillValue(0)
+ .build());
+
+ int[] data = new int[shape * shape];
+ // Fill with some deterministic pattern
+ for (int i = 0; i < shape * shape; i++) {
+ data[i] = i;
+ }
+
+ ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data);
+
+ // Write in parallel
+ array.write(outputData, true);
+
+ // Read back
+ ucar.ma2.Array readData = array.read();
+ int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT);
+
+ Assertions.assertArrayEquals(data, readArr, "Data read back should match data written in parallel");
+ }
+
+ @test
+ public void testParallelWriteWithSharding() throws IOException, ZarrException {
+ // Test internal parallelism with Sharding (nested chunks + shared codec state potential)
+ Path path = TESTOUTPUT.resolve("parallel_write_sharding");
+ StoreHandle storeHandle = new FilesystemStore(path).resolve();
+
+ int shape = 128; // 128x128
+ int shardSize = 64; // Shards are 64x64
+ int innerChunk = 32; // Inner chunks 32x32
+
+ // Metadata with sharding
+ // With shape 128 and shardSize 64, we have 2x2 = 4 shards.
+ // Array.write(parallel=true) will likely process these shards concurrently.
+ dev.zarr.zarrjava.v3.ArrayMetadata metadata = Array.metadataBuilder()
+ .withShape(shape, shape)
+ .withDataType(DataType.INT32)
+ .withChunkShape(shardSize, shardSize) // This sets the shard shape (outer chunks)
+ .withCodecs(c -> c.withSharding(new int[]{innerChunk, innerChunk}, c2 -> c2.withBytes("LITTLE")))
+ .withFillValue(0)
+ .build();
+
+ Array array = Array.create(storeHandle, metadata);
+
+ int[] data = new int[shape * shape];
+ for (int i = 0; i < shape * shape; i++) {
+ data[i] = i;
+ }
+
+ ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data);
+
+ // Write in parallel
+ array.write(outputData, true);
+
+ ucar.ma2.Array readData = array.read();
+ int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT);
+
+ Assertions.assertArrayEquals(data, readArr, "Sharded data written in parallel should match");
+ }
+
+ @test
+ public void testConcurrentWritesDifferentChunks() throws IOException, ZarrException, InterruptedException, ExecutionException {
+ // Test external parallelism (multiple threads calling write on same Array instance)
+ Path path = TESTOUTPUT.resolve("concurrent_write_safety");
+ StoreHandle storeHandle = new FilesystemStore(path).resolve();
+
+ int chunksX = 10;
+ int chunksY = 10;
+ int chunkSize = 50;
+ int shapeX = chunksX * chunkSize;
+ int shapeY = chunksY * chunkSize;
+
+ Array array = Array.create(storeHandle, Array.metadataBuilder()
+ .withShape(shapeX, shapeY)
+ .withDataType(DataType.INT32)
+ .withChunkShape(chunkSize, chunkSize)
+ .withFillValue(-1)
+ .build());
+
+ ExecutorService executor = Executors.newFixedThreadPool(8);
+ List<Callable<Void>> tasks = new ArrayList<>();
+
+ for (int i = 0; i < chunksX; i++) {
+ for (int j = 0; j < chunksY; j++) {
+ final int cx = i;
+ final int cy = j;
+ tasks.add(() -> {
+ int[] chunkData = new int[chunkSize * chunkSize];
+ int val = cx * chunksY + cy; // Unique value per chunk
+ java.util.Arrays.fill(chunkData, val);
+
+ ucar.ma2.Array ucarArray = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{chunkSize, chunkSize}, chunkData);
+
+ // Write to specific chunk offset
+ long[] offset = new long[]{cx * chunkSize, cy * chunkSize};
+ // Use internal parallelism false to isolate external concurrency test mechanism
+ array.write(offset, ucarArray, false);
+ return null;
+ });
+ }
+ }
+
+ List<Future<Void>> futures = executor.invokeAll(tasks);
+
+ for (Future<Void> f : futures) {
+ f.get(); // Check for exceptions
+ }
+ executor.shutdown();
+
+ // Verification
+ ucar.ma2.Array readData = array.read();
+ for (int i = 0; i < chunksX; i++) {
+ for (int j = 0; j < chunksY; j++) {
+ int expectedVal = i * chunksY + j;
+ int originX = i * chunkSize;
+ int originY = j * chunkSize;
+
+ // Verify a pixel in the chunk
+ int val = readData.getInt(readData.getIndex().set(originX, originY));
+ Assertions.assertEquals(expectedVal, val, "Value at chunk " + i + "," + j + " mismatch");
+ }
+ }
+ }
+}
diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java
index c3e498a..779824c 100644
--- c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java
+++ i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java
@@ -332,6 +332,31 @@ public class ZarrV2Test extends ZarrTest {
assertContainsTestAttributes(array.metadata().attributes());
}
+ @test
+ public void testUpdateAttributesBehavior() throws IOException, ZarrException {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV2");
+ ArrayMetadata arrayMetadata = Array.metadataBuilder()
+ .withShape(10, 10)
+ .withDataType(DataType.UINT8)
+ .withChunks(5, 5)
+ .withAttributes(new Attributes(b -> b.set("key1", "val1")))
+ .build();
+
+ Array array1 = Array.create(storeHandle, arrayMetadata);
+ Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2"));
+
+ Assertions.assertNotSame(array1, array2);
+ Assertions.assertEquals("val1", array1.metadata().attributes().get("key1"));
+ Assertions.assertNull(array1.metadata().attributes().get("key2"));
+
+ Assertions.assertEquals("val1", array2.metadata().attributes().get("key1"));
+ Assertions.assertEquals("val2", array2.metadata().attributes().get("key2"));
+
+ // Re-opening should show the updated attributes
+ Array array3 = Array.open(storeHandle);
+ Assertions.assertEquals("val2", array3.metadata().attributes().get("key2"));
+ }
+
@test
public void testResizeArray() throws IOException, ZarrException {
int[] testData = new int[10 * 10];
@@ -360,6 +385,34 @@ public class ZarrV2Test extends ZarrTest {
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
}
+ @test
+ public void testResizeArrayShrink() throws IOException, ZarrException {
+ int[] testData = new int[10 * 10];
+ Arrays.setAll(testData, p -> p);
+
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV2");
+ ArrayMetadata arrayMetadata = Array.metadataBuilder()
+ .withShape(10, 10)
+ .withDataType(DataType.UINT32)
+ .withChunks(5, 5)
+ .build();
+ ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
+ Array array = Array.create(storeHandle, arrayMetadata);
+ array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
+
+ array = array.resize(new long[]{5, 5});
+ Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
+
+ ucar.ma2.Array data = array.read();
+ int[] expectedData = new int[5 * 5];
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 5; j++) {
+ expectedData[i * 5 + j] = testData[i * 10 + j];
+ }
+ }
+ Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
+ }
+
@test
public void testGroupAttributes() throws IOException, ZarrException {
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2");
diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java
index 3ed3c10..4e67204 100644
--- c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java
+++ i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java
@@ -708,6 +708,31 @@ public class ZarrV3Test extends ZarrTest {
assertContainsTestAttributes(array.metadata().attributes());
}
+ @test
+ public void testUpdateAttributesBehavior() throws IOException, ZarrException {
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV3");
+ ArrayMetadata arrayMetadata = Array.metadataBuilder()
+ .withShape(10, 10)
+ .withDataType(DataType.UINT8)
+ .withChunkShape(5, 5)
+ .withAttributes(new Attributes(b -> b.set("key1", "val1")))
+ .build();
+
+ Array array1 = Array.create(storeHandle, arrayMetadata);
+ Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2"));
+
+ Assertions.assertNotSame(array1, array2);
+ Assertions.assertEquals("val1", array1.metadata().attributes().get("key1"));
+ Assertions.assertNull(array1.metadata().attributes().get("key2"));
+
+ Assertions.assertEquals("val1", array2.metadata().attributes().get("key1"));
+ Assertions.assertEquals("val2", array2.metadata().attributes().get("key2"));
+
+ // Re-opening should show the updated attributes
+ Array array3 = Array.open(storeHandle);
+ Assertions.assertEquals("val2", array3.metadata().attributes().get("key2"));
+ }
+
@test
public void testResizeArray() throws IOException, ZarrException {
int[] testData = new int[10 * 10];
@@ -736,6 +761,34 @@ public class ZarrV3Test extends ZarrTest {
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
}
+ @test
+ public void testResizeArrayShrink() throws IOException, ZarrException {
+ int[] testData = new int[10 * 10];
+ Arrays.setAll(testData, p -> p);
+
+ StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV3");
+ ArrayMetadata arrayMetadata = Array.metadataBuilder()
+ .withShape(10, 10)
+ .withDataType(DataType.UINT32)
+ .withChunkShape(5, 5)
+ .build();
+ ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
+ Array array = Array.create(storeHandle, arrayMetadata);
+ array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
+
+ array = array.resize(new long[]{5, 5});
+ Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
+
+ ucar.ma2.Array data = array.read();
+ int[] expectedData = new int[5 * 5];
+ for (int i = 0; i < 5; i++) {
+ for (int j = 0; j < 5; j++) {
+ expectedData[i * 5 + j] = testData[i * 10 + j];
+ }
+ }
+ Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
+ }
+
@test
public void testGroupAttributes() throws IOException, ZarrException {
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3");
* test resize and reopen array
* add resizeMetadataOnly argument for resize
* reformat
* default to parallel read, write
diff --git c/src/main/java/dev/zarr/zarrjava/core/Array.java i/src/main/java/dev/zarr/zarrjava/core/Array.java
index c450cee..7e41e7c 100644
--- c/src/main/java/dev/zarr/zarrjava/core/Array.java
+++ i/src/main/java/dev/zarr/zarrjava/core/Array.java
@@ -21,6 +21,7 @@ import java.util.stream.Stream;
public abstract class Array extends AbstractNode {
protected CodecPipeline codecPipeline;
+ public static final boolean DEFAULT_PARALLELISM = true;
protected Array(StoreHandle storeHandle) throws ZarrException {
super(storeHandle);
@@ -299,7 +300,7 @@ public abstract class Array extends AbstractNode {
* @param array the data to write
*/
public void write(long[] offset, ucar.ma2.Array array) {
- write(offset, array, false);
+ write(offset, array, DEFAULT_PARALLELISM);
}
/**
@@ -334,7 +335,7 @@ public abstract class Array extends AbstractNode {
*/
@nonnull
public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException {
- return read(offset, shape, false);
+ return read(offset, shape, DEFAULT_PARALLELISM);
}
/**
* resize default to parallel and resizeMetadataOnly
* fix USERGUIDE.md
* update tests with long
* small restructuring of userguide
* remove API reference1 parent 7fc2428 commit 71c36ec
2 files changed
Lines changed: 939 additions & 1 deletion
| Original file line number | Diff line number | Diff line change | |
|---|---|---|---|
| |||
2 | 2 | | |
3 | 3 | | |
4 | 4 | | |
5 | | - | |
| 5 | + | |
| 6 | + | |
| 7 | + | |
| 8 | + | |
| 9 | + | |
| 10 | + | |
| 11 | + | |
| 12 | + | |
| 13 | + | |
| 14 | + | |
| 15 | + | |
| 16 | + | |
| 17 | + | |
| 18 | + | |
6 | 19 | | |
7 | 20 | | |
8 | 21 | | |
| |||
0 commit comments