Skip to content

Commit 71c36ec

Browse files
authored
Documentation (#55)
* first draft * add tests diff --git c/src/main/java/dev/zarr/zarrjava/v2/Array.java i/src/main/java/dev/zarr/zarrjava/v2/Array.java index a34c85a..c5ed44b 100644 --- c/src/main/java/dev/zarr/zarrjava/v2/Array.java +++ i/src/main/java/dev/zarr/zarrjava/v2/Array.java @@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @OverRide diff --git c/src/main/java/dev/zarr/zarrjava/v2/Group.java i/src/main/java/dev/zarr/zarrjava/v2/Group.java index d3229e4..73b8cdf 100644 --- c/src/main/java/dev/zarr/zarrjava/v2/Group.java +++ i/src/main/java/dev/zarr/zarrjava/v2/Group.java @@ -283,7 +283,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node { */ public Group updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } diff --git c/src/main/java/dev/zarr/zarrjava/v3/Array.java i/src/main/java/dev/zarr/zarrjava/v3/Array.java index ea029db..45060e9 100644 --- c/src/main/java/dev/zarr/zarrjava/v3/Array.java +++ i/src/main/java/dev/zarr/zarrjava/v3/Array.java @@ -248,7 +248,8 @@ public class Array extends dev.zarr.zarrjava.core.Array implements Node { * @throws IOException throws IOException if the new metadata cannot be serialized */ public Array updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } @OverRide diff --git c/src/main/java/dev/zarr/zarrjava/v3/Group.java i/src/main/java/dev/zarr/zarrjava/v3/Group.java index 1d3aa6b..8b1a81b 100644 --- c/src/main/java/dev/zarr/zarrjava/v3/Group.java +++ i/src/main/java/dev/zarr/zarrjava/v3/Group.java @@ -289,7 +289,8 @@ public class Group extends dev.zarr.zarrjava.core.Group implements Node { * @throws IOException if the metadata cannot be serialized */ public Group updateAttributes(Function<Attributes, Attributes> attributeMapper) throws ZarrException, IOException { - return setAttributes(attributeMapper.apply(metadata.attributes)); + Attributes currentAttributes = metadata.attributes != null ? new Attributes(metadata.attributes) : new Attributes(); + return setAttributes(attributeMapper.apply(currentAttributes)); } /** diff --git c/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java new file mode 100644 index 0000000..60db2ee --- /dev/null +++ i/src/test/java/dev/zarr/zarrjava/ParallelWriteTest.java @@ -0,0 +1,154 @@ +package dev.zarr.zarrjava; + +import dev.zarr.zarrjava.store.FilesystemStore; +import dev.zarr.zarrjava.store.StoreHandle; +import dev.zarr.zarrjava.v3.Array; +import dev.zarr.zarrjava.v3.DataType; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.io.IOException; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.*; + +public class ParallelWriteTest extends ZarrTest { + + @test + public void testParallelWriteDataSafety() throws IOException, ZarrException { + // Test internal parallelism of write method (using parallel=true) + Path path = TESTOUTPUT.resolve("parallel_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 1000; + int chunk = 100; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(chunk, chunk) + .withFillValue(0) + .build()); + + int[] data = new int[shape * shape]; + // Fill with some deterministic pattern + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + // Read back + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Data read back should match data written in parallel"); + } + + @test + public void testParallelWriteWithSharding() throws IOException, ZarrException { + // Test internal parallelism with Sharding (nested chunks + shared codec state potential) + Path path = TESTOUTPUT.resolve("parallel_write_sharding"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int shape = 128; // 128x128 + int shardSize = 64; // Shards are 64x64 + int innerChunk = 32; // Inner chunks 32x32 + + // Metadata with sharding + // With shape 128 and shardSize 64, we have 2x2 = 4 shards. + // Array.write(parallel=true) will likely process these shards concurrently. + dev.zarr.zarrjava.v3.ArrayMetadata metadata = Array.metadataBuilder() + .withShape(shape, shape) + .withDataType(DataType.INT32) + .withChunkShape(shardSize, shardSize) // This sets the shard shape (outer chunks) + .withCodecs(c -> c.withSharding(new int[]{innerChunk, innerChunk}, c2 -> c2.withBytes("LITTLE"))) + .withFillValue(0) + .build(); + + Array array = Array.create(storeHandle, metadata); + + int[] data = new int[shape * shape]; + for (int i = 0; i < shape * shape; i++) { + data[i] = i; + } + + ucar.ma2.Array outputData = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{shape, shape}, data); + + // Write in parallel + array.write(outputData, true); + + ucar.ma2.Array readData = array.read(); + int[] readArr = (int[]) readData.get1DJavaArray(ucar.ma2.DataType.INT); + + Assertions.assertArrayEquals(data, readArr, "Sharded data written in parallel should match"); + } + + @test + public void testConcurrentWritesDifferentChunks() throws IOException, ZarrException, InterruptedException, ExecutionException { + // Test external parallelism (multiple threads calling write on same Array instance) + Path path = TESTOUTPUT.resolve("concurrent_write_safety"); + StoreHandle storeHandle = new FilesystemStore(path).resolve(); + + int chunksX = 10; + int chunksY = 10; + int chunkSize = 50; + int shapeX = chunksX * chunkSize; + int shapeY = chunksY * chunkSize; + + Array array = Array.create(storeHandle, Array.metadataBuilder() + .withShape(shapeX, shapeY) + .withDataType(DataType.INT32) + .withChunkShape(chunkSize, chunkSize) + .withFillValue(-1) + .build()); + + ExecutorService executor = Executors.newFixedThreadPool(8); + List<Callable<Void>> tasks = new ArrayList<>(); + + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + final int cx = i; + final int cy = j; + tasks.add(() -> { + int[] chunkData = new int[chunkSize * chunkSize]; + int val = cx * chunksY + cy; // Unique value per chunk + java.util.Arrays.fill(chunkData, val); + + ucar.ma2.Array ucarArray = ucar.ma2.Array.factory(ucar.ma2.DataType.INT, new int[]{chunkSize, chunkSize}, chunkData); + + // Write to specific chunk offset + long[] offset = new long[]{cx * chunkSize, cy * chunkSize}; + // Use internal parallelism false to isolate external concurrency test mechanism + array.write(offset, ucarArray, false); + return null; + }); + } + } + + List<Future<Void>> futures = executor.invokeAll(tasks); + + for (Future<Void> f : futures) { + f.get(); // Check for exceptions + } + executor.shutdown(); + + // Verification + ucar.ma2.Array readData = array.read(); + for (int i = 0; i < chunksX; i++) { + for (int j = 0; j < chunksY; j++) { + int expectedVal = i * chunksY + j; + int originX = i * chunkSize; + int originY = j * chunkSize; + + // Verify a pixel in the chunk + int val = readData.getInt(readData.getIndex().set(originX, originY)); + Assertions.assertEquals(expectedVal, val, "Value at chunk " + i + "," + j + " mismatch"); + } + } + } +} diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java index c3e498a..779824c 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrV2Test.java @@ -332,6 +332,31 @@ public class ZarrV2Test extends ZarrTest { assertContainsTestAttributes(array.metadata().attributes()); } + @test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunks(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -360,6 +385,34 @@ public class ZarrV2Test extends ZarrTest { Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + @test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV2"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunks(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + expectedData[i * 5 + j] = testData[i * 10 + j]; + } + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + @test public void testGroupAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2"); diff --git c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java index 3ed3c10..4e67204 100644 --- c/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java +++ i/src/test/java/dev/zarr/zarrjava/ZarrV3Test.java @@ -708,6 +708,31 @@ public class ZarrV3Test extends ZarrTest { assertContainsTestAttributes(array.metadata().attributes()); } + @test + public void testUpdateAttributesBehavior() throws IOException, ZarrException { + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testUpdateAttributesBehaviorV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT8) + .withChunkShape(5, 5) + .withAttributes(new Attributes(b -> b.set("key1", "val1"))) + .build(); + + Array array1 = Array.create(storeHandle, arrayMetadata); + Array array2 = array1.updateAttributes(attrs -> attrs.set("key2", "val2")); + + Assertions.assertNotSame(array1, array2); + Assertions.assertEquals("val1", array1.metadata().attributes().get("key1")); + Assertions.assertNull(array1.metadata().attributes().get("key2")); + + Assertions.assertEquals("val1", array2.metadata().attributes().get("key1")); + Assertions.assertEquals("val2", array2.metadata().attributes().get("key2")); + + // Re-opening should show the updated attributes + Array array3 = Array.open(storeHandle); + Assertions.assertEquals("val2", array3.metadata().attributes().get("key2")); + } + @test public void testResizeArray() throws IOException, ZarrException { int[] testData = new int[10 * 10]; @@ -736,6 +761,34 @@ public class ZarrV3Test extends ZarrTest { Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); } + @test + public void testResizeArrayShrink() throws IOException, ZarrException { + int[] testData = new int[10 * 10]; + Arrays.setAll(testData, p -> p); + + StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkV3"); + ArrayMetadata arrayMetadata = Array.metadataBuilder() + .withShape(10, 10) + .withDataType(DataType.UINT32) + .withChunkShape(5, 5) + .build(); + ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType(); + Array array = Array.create(storeHandle, arrayMetadata); + array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData)); + + array = array.resize(new long[]{5, 5}); + Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape()); + + ucar.ma2.Array data = array.read(); + int[] expectedData = new int[5 * 5]; + for (int i = 0; i < 5; i++) { + for (int j = 0; j < 5; j++) { + expectedData[i * 5 + j] = testData[i * 10 + j]; + } + } + Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType)); + } + @test public void testGroupAttributes() throws IOException, ZarrException { StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3"); * test resize and reopen array * add resizeMetadataOnly argument for resize * reformat * default to parallel read, write diff --git c/src/main/java/dev/zarr/zarrjava/core/Array.java i/src/main/java/dev/zarr/zarrjava/core/Array.java index c450cee..7e41e7c 100644 --- c/src/main/java/dev/zarr/zarrjava/core/Array.java +++ i/src/main/java/dev/zarr/zarrjava/core/Array.java @@ -21,6 +21,7 @@ import java.util.stream.Stream; public abstract class Array extends AbstractNode { protected CodecPipeline codecPipeline; + public static final boolean DEFAULT_PARALLELISM = true; protected Array(StoreHandle storeHandle) throws ZarrException { super(storeHandle); @@ -299,7 +300,7 @@ public abstract class Array extends AbstractNode { * @param array the data to write */ public void write(long[] offset, ucar.ma2.Array array) { - write(offset, array, false); + write(offset, array, DEFAULT_PARALLELISM); } /** @@ -334,7 +335,7 @@ public abstract class Array extends AbstractNode { */ @nonnull public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException { - return read(offset, shape, false); + return read(offset, shape, DEFAULT_PARALLELISM); } /** * resize default to parallel and resizeMetadataOnly * fix USERGUIDE.md * update tests with long * small restructuring of userguide * remove API reference
1 parent 7fc2428 commit 71c36ec

2 files changed

Lines changed: 939 additions & 1 deletion

File tree

README.md

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,20 @@
22

33
This repository contains a Java implementation of Zarr version 2 and 3.
44

5-
## Usage
5+
## Documentation
6+
7+
For comprehensive documentation, see the [**User Guide**](USERGUIDE.md), which includes:
8+
9+
- Installation instructions
10+
- Quick start examples
11+
- Core concepts and API reference
12+
- Working with arrays and groups
13+
- Storage backends (Filesystem, HTTP, S3, ZIP, Memory)
14+
- Compression and codecs
15+
- Best practices
16+
- Troubleshooting
17+
18+
## Quick Usage Example
619

720
```java
821
import dev.zarr.zarrjava.store.FilesystemStore;

0 commit comments

Comments
 (0)