Zip Store (#37)

* add ZipStore tests

diff --git c/src/main/java/dev/zarr/zarrjava/store/ZipStore.java i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java
new file mode 100644
index 0000000..054917f
--- /dev/null
+++ i/src/main/java/dev/zarr/zarrjava/store/ZipStore.java
@@ -0,0 +1,72 @@
+package dev.zarr.zarrjava.store;
+
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+import java.nio.ByteBuffer;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.stream.Stream;
+
+public class ZipStore implements Store, Store.ListableStore {
+    @nonnull
+    private final Path path;
+
+    public ZipStore(@nonnull Path path) {
+        this.path = path;
+    }
+
+    public ZipStore(@nonnull String path) {
+        this.path = Paths.get(path);
+    }
+
+
+    @OverRide
+    public Stream<String> list(String[] keys) {
+        return Stream.empty();
+    }
+
+    @OverRide
+    public boolean exists(String[] keys) {
+        return false;
+    }
+
+    @nullable
+    @OverRide
+    public ByteBuffer get(String[] keys) {
+        return null;
+    }
+
+    @nullable
+    @OverRide
+    public ByteBuffer get(String[] keys, long start) {
+        return null;
+    }
+
+    @nullable
+    @OverRide
+    public ByteBuffer get(String[] keys, long start, long end) {
+        return null;
+    }
+
+    @OverRide
+    public void set(String[] keys, ByteBuffer bytes) {
+
+    }
+
+    @OverRide
+    public void delete(String[] keys) {
+
+    }
+
+    @nonnull
+    @OverRide
+    public StoreHandle resolve(String... keys) {
+        return new StoreHandle(this, keys);
+    }
+
+    @OverRide
+    public String toString() {
+        return this.path.toUri().toString().replaceAll("\\/$", "");
+    }
+
+}
diff --git c/src/test/java/dev/zarr/zarrjava/Utils.java i/src/test/java/dev/zarr/zarrjava/Utils.java
new file mode 100644
index 0000000..0026200
--- /dev/null
+++ i/src/test/java/dev/zarr/zarrjava/Utils.java
@@ -0,0 +1,40 @@
+package dev.zarr.zarrjava;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+public class Utils {
+
+    static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) throws IOException {
+        if (fileToZip.isHidden()) {
+            return;
+        }
+        if (fileToZip.isDirectory()) {
+            if (fileName.endsWith("/")) {
+                zipOut.putNextEntry(new ZipEntry(fileName));
+                zipOut.closeEntry();
+            } else {
+                zipOut.putNextEntry(new ZipEntry(fileName + "/"));
+                zipOut.closeEntry();
+            }
+            File[] children = fileToZip.listFiles();
+            for (File childFile : children) {
+                zipFile(childFile, fileName + "/" + childFile.getName(), zipOut);
+            }
+            return;
+        }
+        FileInputStream fis = new FileInputStream(fileToZip);
+        ZipEntry zipEntry = new ZipEntry(fileName);
+        zipOut.putNextEntry(zipEntry);
+        byte[] bytes = new byte[1024];
+        int length;
+        while ((length = fis.read(bytes)) >= 0) {
+            zipOut.write(bytes, 0, length);
+        }
+        fis.close();
+    }
+
+}
diff --git c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java
index 4a369c9..c7d2ab4 100644
--- c/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java
+++ i/src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java
@@ -7,16 +7,22 @@ import dev.zarr.zarrjava.v3.*;
 import org.junit.jupiter.api.Assertions;
 import org.junit.jupiter.api.Test;
 import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
 import org.junit.jupiter.params.provider.CsvSource;
 import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider;
 import software.amazon.awssdk.regions.Region;
 import software.amazon.awssdk.services.s3.S3Client;

+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
 import java.util.Arrays;
 import java.util.stream.Stream;
+import java.nio.file.Path;
+import java.util.zip.ZipOutputStream;

+import static dev.zarr.zarrjava.Utils.zipFile;
 import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;

 public class ZarrStoreTest extends ZarrTest {
@@ -132,4 +138,72 @@ public class ZarrStoreTest extends ZarrTest {
         Assertions.assertEquals("test group", attrs.getString("description"));

     }
+
+    @test
+    public void testZipStore() throws ZarrException, IOException {
+        Path sourceDir = TESTOUTPUT.resolve("testZipStore");
+        Path targetDir = TESTOUTPUT.resolve("testZipStore.zip");
+        FilesystemStore fsStore = new FilesystemStore(sourceDir);
+        writeTestGroupV3(fsStore, true);
+
+        FileOutputStream fos = new FileOutputStream(targetDir.toFile());
+        ZipOutputStream zipOut = new ZipOutputStream(fos);
+
+        File fileToZip = new File(sourceDir.toUri());
+        zipFile(fileToZip, fileToZip.getName(), zipOut);
+        zipOut.close();
+        fos.close();
+
+        ZipStore zipStore = new ZipStore(targetDir);
+        assertIsTestGroupV3(Group.open(zipStore.resolve()), true);
+    }
+
+    static Stream<Store> localStores() {
+        return Stream.of(
+//                new ConcurrentMemoryStore(),
+                new FilesystemStore(TESTOUTPUT.resolve("testLocalStoresFS")),
+                new ZipStore(TESTOUTPUT.resolve("testLocalStoresZIP.zip"))
+        );
+    }
+
+    @ParameterizedTest
+    @MethodSource("localStores")
+    public void testLocalStores(Store store) throws IOException, ZarrException {
+        boolean useParallel = true;
+        Group group = writeTestGroupV3(store, useParallel);
+        assertIsTestGroupV3(group, useParallel);
+    }
+
+    int[] testData(){
+        int[] testData = new int[1024 * 1024];
+        Arrays.setAll(testData, p -> p);
+        return testData;
+    }
+
+    Group writeTestGroupV3(Store store, boolean useParallel) throws ZarrException, IOException {
+        StoreHandle storeHandle = store.resolve();
+
+        Group group = Group.create(storeHandle);
+        Array array = group.createArray("array", b -> b
+                .withShape(1024, 1024)
+                .withDataType(DataType.UINT32)
+                .withChunkShape(5, 5)
+        );
+        array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData()), useParallel);
+        group.createGroup("subgroup");
+        group.setAttributes(new Attributes(b -> b.set("some", "value")));
+        return group;
+    }
+
+    void assertIsTestGroupV3(Group group, boolean useParallel) throws ZarrException {
+        Stream<dev.zarr.zarrjava.core.Node> nodes = group.list();
+        Assertions.assertEquals(2, nodes.count());
+        Array array = (Array) group.get("array");
+        Assertions.assertNotNull(array);
+        ucar.ma2.Array result = array.read(useParallel);
+        Assertions.assertArrayEquals(testData(), (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
+        Attributes attrs = group.metadata().attributes;
+        Assertions.assertNotNull(attrs);
+        Assertions.assertEquals("value", attrs.getString("some"));
+    }
 }

* refactor and unify outputs of Store.list

* read zip store

* Bump to 0.0.6 to trigger release

There are apparently cases where release: [created]
leads to the deploy not being triggered. Attempting
an expansion to [created, published].

diff --git c/pom.xml i/pom.xml
index f4c1091..9c9d45d 100644
--- c/pom.xml
+++ i/pom.xml
@@ -6,7 +6,7 @@

     <groupId>dev.zarr</groupId>
     <artifactId>zarr-java</artifactId>
-    <version>0.0.9</version>
+    <version>0.0.6</version>

     <name>zarr-java</name>

@@ -123,6 +123,17 @@
         </dependency>
     </dependencies>

+    <distributionManagement>
+        <snapshotRepository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
+        </snapshotRepository>
+        <repository>
+            <id>ossrh</id>
+            <url>https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
+        </repository>
+    </distributionManagement>
+
     <repositories>
         <repository>
             <id>unidata-all</id>
@@ -221,16 +232,6 @@
                     </execution>
                 </executions>
             </plugin>
-            <plugin>
-                <groupId>org.sonatype.central</groupId>
-                <artifactId>central-publishing-maven-plugin</artifactId>
-                <version>0.8.0</version>
-                <extensions>true</extensions>
-                <configuration>
-                    <publishingServerId>central</publishingServerId>
-                    <tokenAuth>true</tokenAuth>
-                </configuration>
-            </plugin>
         </plugins>
     </build>
 </project>

* Bump to 0.0.7

* Bump to 0.0.8

Use new sonatype plugin for upload

* Bump to 0.0.9

* write buffer of zip store

* use apache commons compress for zip file read and write

* set Zip64Mode.AsNeeded

* test Zipped OME-Zarr requirements

* Sort zarr.json files in breadth-first order within BufferedZipStore

* manually read zip comment

* refactor read zip comment

* test zip store with v2

* use com.fasterxml.jackson.databind.util.ByteBufferBackedInputStream instead of own implementation

* add ReadOnlyZipStore

* fix ReadOnlyZipStore for zips with
1. leading slashes in paths
2. no sizes in entry headers

* add BufferedZipStore parameter flushOnWrite

* fix testMemoryStore

* default flushOnWrite to false

* fix s3 store get range

* add store.getInputStream

* add store.getSize

* improve performance of ReadOnlyZipStore.getArchiveComment

* inherit zipstores from common parent and reduce buffers in memory in loadBuffer

* fix s3 test buckets

* reduce getArchiveComment store requests

* format

* fix merge

* improve Group::list performance

* fix ReadOnlyZipStore::list

* less requests to store on read

* fix FilesystemStore::list

* fix MemoryStore::list

* fix ReadOnlyZipStore::list

* test Store::list for local stores

* less store.exists() for v2.Group.list()

* add S3Mock for tests

* fallback to chunkHandle.exists() for non listable stores

* fix ZarrV2Test::testGroup

* downgrade s3mock version to match java-version 11

* downgrade s3mock version to match java-version 11

* run S3 Mock with docker

* S3Mock tests only on linux

* fix ZarrV3Test::testGroup for windows, mac

* refactor and add missing store tests

* fix s3Store::list

* fix ReadOnlyZipStore::list

* add tests

* fix tests with path collision

* reformat code

* improve httpStore testing

* change store::list to list no directories
add store::listChildren containing direct children including directories

* adjust test for store::list

* test store::exist, list, listChildren

* test store::get with start and end

* test store::delete

* fix ReadOnlyZipStoreTest.testReadFromBufferedZipStore and S3StoreTest>StoreTest.testExists

* fix S3StoreTest.testList

* test store.get with start argument

* BufferedZipStore auto closable

* s3 tests opt-in instead of opt-out

* cache key existence and sizes in ReadOnlyZipStore

* remove listing existingKeys from array read

---------

Co-authored-by: Josh Moore <josh@openmicroscopy.org>

Author: brokkoli71

.github/workflows/ci.yml

@@ -14,6 +14,14 @@ jobs:
         os: [ ubuntu-latest, windows-latest, macos-latest ]
       fail-fast: false
     runs-on: ${{ matrix.os }}
+    services:
+      s3mock:
+        # This service will only start if the runner is Linux
+        image: ${{ matrix.os == 'ubuntu-latest' && 'adobe/s3mock:3.11.0' || '' }}
+        ports:
+          - 9090:9090
+        env:
+          initialBuckets: zarr-test-bucket
     defaults:
       run:
         shell: bash
@@ -49,8 +57,12 @@ jobs:
       - name: Test
         env:
           MAVEN_OPTS: "-Xmx6g"
-        run: mvn --no-transfer-progress test -DargLine="-Xmx6g"
-
+        run: |
+          if [ "${{ matrix.os }}" == "ubuntu-latest" ]; then
+            mvn --no-transfer-progress test -DargLine="-Xmx6g" -DrunS3Tests=true
+          else
+            mvn --no-transfer-progress test -DargLine="-Xmx6g"
+          fi
       - name: Assemble JAR
         run: mvn package -DskipTests
 

.gitignore

@@ -45,4 +45,5 @@ build/
 /main.py
 /pyproject.toml
 /uv.lock
-**/__pycache__
+**/__pycache__
+/dependency-reduced-pom.xml

pom.xml

@@ -121,6 +121,11 @@
             <version>4.13.1</version>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-compress</artifactId>
+            <version>1.28.0</version>
+        </dependency>
     </dependencies>
 
     <repositories>
@@ -139,6 +144,10 @@
                 <version>3.2.5</version>
                 <configuration>
                     <useSystemClassLoader>false</useSystemClassLoader>
+                    <environmentVariables>
+                        <!-- Force Testcontainers to use a newer Docker API version supported by your local Daemon -->
+                        <DOCKER_API_VERSION>1.44</DOCKER_API_VERSION>
+                    </environmentVariables>
                 </configuration>
             </plugin>
             <plugin>

src/main/java/dev/zarr/zarrjava/core/Array.java

@@ -3,6 +3,7 @@
 import dev.zarr.zarrjava.ZarrException;
 import dev.zarr.zarrjava.core.codec.CodecPipeline;
 import dev.zarr.zarrjava.store.FilesystemStore;
+import dev.zarr.zarrjava.store.Store;
 import dev.zarr.zarrjava.store.StoreHandle;
 import dev.zarr.zarrjava.utils.IndexingUtils;
 import dev.zarr.zarrjava.utils.MultiArrayUtils;
@@ -16,6 +17,9 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.util.Arrays;
+import java.util.List;
+import java.util.Set;
+import java.util.stream.Collectors;
 import java.util.stream.Stream;
 
 public abstract class Array extends AbstractNode {
@@ -306,9 +310,9 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
 
                         final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
                         final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
-                        if (!chunkHandle.exists()) {
-                            return;
-                        }
+
+                        if (!chunkHandle.exists()) return;
+
                         if (codecPipeline.supportsPartialDecode()) {
                             final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
                                     Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);

src/main/java/dev/zarr/zarrjava/core/Group.java

@@ -9,7 +9,6 @@
 import java.io.IOException;
 import java.nio.file.Path;
 import java.nio.file.Paths;
-import java.util.Objects;
 import java.util.stream.Stream;
 
 public abstract class Group extends AbstractNode {
@@ -64,20 +63,15 @@ public static Group open(String path) throws IOException, ZarrException {
     }
 
     @Nullable
-    public abstract Node get(String key) throws ZarrException, IOException;
+    public abstract Node get(String[] key) throws ZarrException, IOException;
 
-    public Stream<Node> list() {
-        return storeHandle.list()
-                .map(key -> {
-                    try {
-                        return get(key);
-                    } catch (Exception e) {
-                        throw new RuntimeException(e);
-                    }
-                })
-                .filter(Objects::nonNull);
+    @Nullable
+    public Node get(String key) throws ZarrException, IOException {
+        return get(new String[]{key});
     }
 
+    public abstract Stream<Node> list();
+
     public Node[] listAsArray() {
         try (Stream<Node> nodeStream = list()) {
             return nodeStream.toArray(Node[]::new);