Merge branch 'main' into cli-wrapper

# Conflicts:
#	pom.xml

Author: brokkoli71

.github/workflows/ci.yml

@@ -14,6 +14,14 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
       fail-fast: false
     runs-on: ${{ matrix.os }}
+    services:
+      s3mock:
+        # This service will only start if the runner is Linux
+        image: ${{ matrix.os == 'ubuntu-latest' && 'adobe/s3mock:3.11.0' || '' }}
+        ports:
+          - 9090:9090
+        env:
+          initialBuckets: zarr-test-bucket
     defaults:
       run:
         shell: bash
@@ -49,8 +57,12 @@ jobs:
       - name: Test
         env:
           MAVEN_OPTS: "-Xmx6g"
-        run: mvn --no-transfer-progress test -DargLine="-Xmx6g"
-
+        run: |
+          if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+            mvn --no-transfer-progress test -DargLine="-Xmx6g" -DrunS3Tests=true
+          else
+            mvn --no-transfer-progress test -DargLine="-Xmx6g"
+          fi
       - name: Assemble JAR
         run: mvn package -DskipTests
 

pom.xml

@@ -6,7 +6,7 @@
 
     <groupId>dev.zarr</groupId>
     <artifactId>zarr-java</artifactId>
-    <version>0.0.9</version>
+    <version>0.0.10</version>
 
     <name>zarr-java</name>
 
@@ -121,6 +121,11 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.28.0</version>
+        </dependency>
 
         <dependency>
             <groupId>info.picocli</groupId>
@@ -145,6 +150,10 @@
                 <version>3.2.5</version>
                 <configuration>
                     <useSystemClassLoader>false</useSystemClassLoader>
+                    <environmentVariables>
+                        <!-- Force Testcontainers to use a newer Docker API version supported by your local Daemon -->
+                        <DOCKER_API_VERSION>1.44</DOCKER_API_VERSION>
+                    </environmentVariables>
                 </configuration>
             </plugin>
             <plugin>

src/main/java/dev/zarr/zarrjava/core/Array.java

@@ -21,6 +21,7 @@
 public abstract class Array extends AbstractNode {
 
     protected CodecPipeline codecPipeline;
+    public static final boolean DEFAULT_PARALLELISM = true;
 
     protected Array(StoreHandle storeHandle) throws ZarrException {
         super(storeHandle);
@@ -88,7 +89,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
             throw new IllegalArgumentException("'array' needs to have rank '" + metadata.ndim() + "'.");
         }
 
-        int[] shape = array.getShape();
+        long[] shape = Utils.toLongArray(array.getShape());
 
         final int[] chunkShape = metadata.chunkShape();
         Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
@@ -118,8 +119,14 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
                             );
                         }
                         writeChunk(chunkCoords, chunkArray);
-                    } catch (ZarrException | InvalidRangeException e) {
-                        throw new RuntimeException(e);
+                    } catch (ZarrException e) {
+                        throw new RuntimeException(
+                                "Failed to write chunk at coordinates " + Arrays.toString(chunkCoords) +
+                                ": " + e.getMessage(), e);
+                    } catch (InvalidRangeException e) {
+                        throw new RuntimeException(
+                                "Invalid array range when writing chunk at coordinates " + Arrays.toString(chunkCoords) +
+                                ": " + e.getMessage(), e);
                     }
                 });
 
@@ -174,6 +181,110 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) throws ZarrException {
         return codecPipeline.decode(chunkBytes);
     }
 
+    /**
+     * Deletes chunks that are completely outside the new shape and trims boundary chunks.
+     *
+     * @param newShape the new shape of the array
+     * @param parallel utilizes parallelism if true
+     */
+    protected void cleanupChunksForResize(long[] newShape, boolean parallel) {
+        ArrayMetadata metadata = metadata();
+        final int[] chunkShape = metadata.chunkShape();
+        final int ndim = metadata.ndim();
+        final dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding chunkKeyEncoding = metadata.chunkKeyEncoding();
+
+        // Calculate max valid chunk coordinates for the new shape
+        long[] newMaxChunkCoords = new long[ndim];
+        for (int i = 0; i < ndim; i++) {
+            newMaxChunkCoords[i] = (newShape[i] + chunkShape[i] - 1) / chunkShape[i];
+        }
+
+        // Iterate over all possible chunk coordinates in the old shape
+        long[][] allOldChunkCoords = IndexingUtils.computeChunkCoords(metadata.shape, chunkShape);
+
+        Stream<long[]> chunkStream = Arrays.stream(allOldChunkCoords);
+        if (parallel) {
+            chunkStream = chunkStream.parallel();
+        }
+
+        chunkStream.forEach(chunkCoords -> {
+            boolean isOutsideBounds = false;
+            boolean isOnBoundary = false;
+
+            for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
+                if (chunkCoords[dimIdx] >= newMaxChunkCoords[dimIdx]) {
+                    isOutsideBounds = true;
+                    break;
+                }
+                // Check if this chunk is on the boundary (partially outside new shape)
+                long chunkEnd = (chunkCoords[dimIdx] + 1) * chunkShape[dimIdx];
+                if (chunkEnd > newShape[dimIdx]) {
+                    isOnBoundary = true;
+                }
+            }
+
+            String[] chunkKeys = chunkKeyEncoding.encodeChunkKey(chunkCoords);
+            StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
+
+            if (isOutsideBounds) {
+                // Delete chunk that is completely outside
+                chunkHandle.delete();
+            } else if (isOnBoundary) {
+                // Trim boundary chunk - read, clear out-of-bounds data, write back
+                try {
+                    trimBoundaryChunk(chunkCoords, newShape, chunkShape);
+                } catch (ZarrException e) {
+                    throw new RuntimeException(e);
+                }
+            }
+        });
+    }
+
+    /**
+     * Trims a boundary chunk by reading it, clearing the out-of-bounds portion, and writing it back.
+     *
+     * @param chunkCoords the coordinates of the chunk to trim
+     * @param newShape    the new shape of the array
+     * @param chunkShape  the shape of the chunks
+     * @throws ZarrException if reading or writing the chunk fails
+     */
+    protected void trimBoundaryChunk(long[] chunkCoords, long[] newShape, int[] chunkShape) throws ZarrException {
+        ArrayMetadata metadata = metadata();
+        final int ndim = metadata.ndim();
+
+        // Calculate the valid region within this chunk
+        int[] validShape = new int[ndim];
+        boolean needsTrimming = false;
+        for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
+            long chunkStart = chunkCoords[dimIdx] * chunkShape[dimIdx];
+            long chunkEnd = chunkStart + chunkShape[dimIdx];
+            if (chunkEnd > newShape[dimIdx]) {
+                validShape[dimIdx] = (int) (newShape[dimIdx] - chunkStart);
+                needsTrimming = true;
+            } else {
+                validShape[dimIdx] = chunkShape[dimIdx];
+            }
+        }
+
+        if (!needsTrimming) {
+            return;
+        }
+
+        // Read the existing chunk
+        ucar.ma2.Array chunkData = readChunk(chunkCoords);
+
+        // Create a new chunk filled with fill value
+        ucar.ma2.Array newChunkData = metadata.allocateFillValueChunk();
+
+        // Copy only the valid region
+        MultiArrayUtils.copyRegion(
+                chunkData, new int[ndim], newChunkData, new int[ndim], validShape
+        );
+
+        // Write the trimmed chunk back
+        writeChunk(chunkCoords, newChunkData);
+    }
+
 
     /**
      * Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of
@@ -195,7 +306,7 @@ public void write(ucar.ma2.Array array) {
      * @param array  the data to write
      */
     public void write(long[] offset, ucar.ma2.Array array) {
-        write(offset, array, false);
+        write(offset, array, DEFAULT_PARALLELISM);
     }
 
     /**
@@ -217,7 +328,7 @@ public void write(ucar.ma2.Array array, boolean parallel) {
      */
     @Nonnull
     public ucar.ma2.Array read() throws ZarrException {
-        return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape));
+        return read(new long[metadata().ndim()], metadata().shape);
     }
 
     /**
@@ -229,8 +340,8 @@ public ucar.ma2.Array read() throws ZarrException {
      * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails
      */
     @Nonnull
-    public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException {
-        return read(offset, shape, false);
+    public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException {
+        return read(offset, shape, DEFAULT_PARALLELISM);
     }
 
     /**
@@ -241,7 +352,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
      */
     @Nonnull
     public ucar.ma2.Array read(final boolean parallel) throws ZarrException {
-        return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel);
+        return read(new long[metadata().ndim()], metadata().shape, parallel);
     }
 
     boolean chunkIsInArray(long[] chunkCoords) {
@@ -264,7 +375,7 @@ boolean chunkIsInArray(long[] chunkCoords) {
      * @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails
      */
     @Nonnull
-    public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException {
+    public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolean parallel) throws ZarrException {
         ArrayMetadata metadata = metadata();
         if (offset.length != metadata.ndim()) {
             throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'.");
@@ -284,7 +395,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
         }
 
         final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
-                shape);
+                Utils.toIntArray(shape));
         Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
         if (parallel) {
             chunkStream = chunkStream.parallel();
@@ -306,19 +417,20 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
 
                         final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
                         final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
-                        if (!chunkHandle.exists()) {
-                            return;
-                        }
+
                         if (codecPipeline.supportsPartialDecode()) {
                             final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
                                     Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
                             MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
                                     chunkProjection.outOffset, chunkProjection.shape
                             );
                         } else {
-                            MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
-                                    outputArray, chunkProjection.outOffset, chunkProjection.shape
-                            );
+                            ByteBuffer chunkBytes = chunkHandle.read();
+                            if (chunkBytes != null) {
+                                MultiArrayUtils.copyRegion(codecPipeline.decode(chunkBytes), chunkProjection.chunkOffset,
+                                        outputArray, chunkProjection.outOffset, chunkProjection.shape
+                                );
+                            }
                         }
 
                     } catch (ZarrException e) {
@@ -328,6 +440,46 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
         return outputArray;
     }
 
+    /**
+     * Sets a new shape for the Zarr array. Only the metadata is updated by default.
+     * This method returns a new instance of the Zarr array class and the old instance
+     * becomes invalid.
+     *
+     * @param newShape the new shape of the Zarr array
+     * @throws ZarrException if the new metadata is invalid
+     * @throws IOException   throws IOException if the new metadata cannot be serialized
+     */
+    public Array resize(long[] newShape) throws ZarrException, IOException {
+        return resize(newShape, true);
+    }
+
+    /**
+     * Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class
+     * and the old instance becomes invalid.
+     *
+     * @param newShape           the new shape of the Zarr array
+     * @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new
+     *                           bounds are deleted and boundary chunks are trimmed
+     * @throws ZarrException if the new metadata is invalid
+     * @throws IOException   throws IOException if the new metadata cannot be serialized
+     */
+    public Array resize(long[] newShape, boolean resizeMetadataOnly) throws ZarrException, IOException {
+        return resize(newShape, resizeMetadataOnly, DEFAULT_PARALLELISM);
+    }
+
+    /**
+     * Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class
+     * and the old instance becomes invalid.
+     *
+     * @param newShape           the new shape of the Zarr array
+     * @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new
+     *                           bounds are deleted and boundary chunks are trimmed
+     * @param parallel           utilizes parallelism if true when cleaning up chunks
+     * @throws ZarrException if the new metadata is invalid
+     * @throws IOException   throws IOException if the new metadata cannot be serialized
+     */
+    public abstract Array resize(long[] newShape, boolean resizeMetadataOnly, boolean parallel) throws ZarrException, IOException;
+
     public ArrayAccessor access() {
         return new ArrayAccessor(this);
     }
@@ -336,7 +488,7 @@ public static final class ArrayAccessor {
         @Nullable
         long[] offset;
         @Nullable
-        int[] shape;
+        long[] shape;
         @Nonnull
         Array array;
 
@@ -353,13 +505,13 @@ public ArrayAccessor withOffset(@Nonnull long... offset) {
 
         @Nonnull
         public ArrayAccessor withShape(@Nonnull int... shape) {
-            this.shape = shape;
+            this.shape = Utils.toLongArray(shape);
             return this;
         }
 
         @Nonnull
         public ArrayAccessor withShape(@Nonnull long... shape) {
-            this.shape = Utils.toIntArray(shape);
+            this.shape = shape;
             return this;
         }
 

src/main/java/dev/zarr/zarrjava/core/Group.java

@@ -9,7 +9,6 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Objects;
 import java.util.stream.Stream;
 
 public abstract class Group extends AbstractNode {
@@ -64,20 +63,15 @@ public static Group open(String path) throws IOException, ZarrException {
     }
 
     @Nullable
-    public abstract Node get(String key) throws ZarrException, IOException;
+    public abstract Node get(String[] key) throws ZarrException, IOException;
 
-    public Stream<Node> list() {
-        return storeHandle.list()
-                .map(key -> {
-                    try {
-                        return get(key);
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
-                })
-                .filter(Objects::nonNull);
+    @Nullable
+    public Node get(String key) throws ZarrException, IOException {
+        return get(new String[]{key});
     }
 
+    public abstract Stream<Node> list();
+
     public Node[] listAsArray() {
         try (Stream<Node> nodeStream = list()) {
             return nodeStream.toArray(Node[]::new);