Skip to content

Commit 8c4e988

Browse files
brokkoli71brokkoli71
committed
refactor and add missing store tests
1 parent 5ae5ac2 commit 8c4e988

11 files changed

Lines changed: 751 additions & 530 deletions

‎src/test/java/dev/zarr/zarrjava/Utils.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99

1010
public class Utils {
1111

12-
static void zipFile(Path sourceDir, Path targetDir) throws IOException {
12+
public static void zipFile(Path sourceDir, Path targetDir) throws IOException {
1313
FileOutputStream fos = new FileOutputStream(targetDir.toFile());
1414
ZipOutputStream zipOut = new ZipOutputStream(fos);
1515

@@ -53,7 +53,7 @@ static void zipFile(File fileToZip, String fileName, ZipOutputStream zipOut) thr
5353
* Unzip sourceZip into targetDir.
5454
* Protects against Zip Slip by ensuring extracted paths remain inside targetDir.
5555
*/
56-
static void unzipFile(Path sourceZip, Path targetDir) throws IOException {
56+
public static void unzipFile(Path sourceZip, Path targetDir) throws IOException {
5757
Files.createDirectories(targetDir);
5858
try (FileInputStream fis = new FileInputStream(sourceZip.toFile());
5959
ZipInputStream zis = new ZipInputStream(fis)) {

‎src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java‎

Lines changed: 0 additions & 528 deletions
This file was deleted.

‎src/test/java/dev/zarr/zarrjava/store/BufferedZipStoreTest.java‎

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
package dev.zarr.zarrjava.store;
2+
3+
import dev.zarr.zarrjava.Utils;
4+
import dev.zarr.zarrjava.ZarrException;
5+
import dev.zarr.zarrjava.core.Array;
6+
import dev.zarr.zarrjava.core.Group;
7+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
8+
import org.apache.commons.compress.archivers.zip.ZipFile;
9+
import org.junit.jupiter.api.Assertions;
10+
import org.junit.jupiter.api.BeforeAll;
11+
import org.junit.jupiter.api.Test;
12+
import org.junit.jupiter.params.ParameterizedTest;
13+
import org.junit.jupiter.params.provider.CsvSource;
14+
15+
import java.io.IOException;
16+
import java.nio.file.Files;
17+
import java.nio.file.Path;
18+
import java.util.ArrayList;
19+
import java.util.Collections;
20+
import java.util.zip.ZipEntry;
21+
22+
import static dev.zarr.zarrjava.Utils.unzipFile;
23+
24+
public class BufferedZipStoreTest extends WritableStoreTest {
25+
26+
Path testGroupDir = TESTOUTPUT.resolve("testZipStore.zip");
27+
28+
@BeforeAll
29+
void writeTestGroup() throws ZarrException, IOException {
30+
Path sourceDir = TESTOUTPUT.resolve("testZipStore");
31+
FilesystemStore fsStore = new FilesystemStore(sourceDir);
32+
writeTestGroupV3(fsStore.resolve(), true);
33+
Utils.zipFile(sourceDir, testGroupDir);
34+
}
35+
36+
@Override
37+
StoreHandle storeHandleWithData() {
38+
return new BufferedZipStore(testGroupDir).resolve("zarr.json");
39+
}
40+
41+
@Test
42+
public void testOpenZipStore() throws ZarrException, IOException {
43+
BufferedZipStore zipStore = new BufferedZipStore(testGroupDir);
44+
assertIsTestGroupV3(Group.open(zipStore.resolve()), true);
45+
}
46+
47+
@ParameterizedTest
48+
@CsvSource({"false", "true",})
49+
public void testWriteZipStore(boolean flushOnWrite) throws ZarrException, IOException {
50+
Path path = TESTOUTPUT.resolve("testWriteZipStore" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip");
51+
BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite);
52+
writeTestGroupV3(zipStore.resolve(), true);
53+
if (!flushOnWrite) zipStore.flush();
54+
55+
BufferedZipStore zipStoreRead = new BufferedZipStore(path);
56+
assertIsTestGroupV3(Group.open(zipStoreRead.resolve()), true);
57+
58+
Path unzippedPath = TESTOUTPUT.resolve("testWriteZipStoreUnzipped" + (flushOnWrite ? "Flush" : "NoFlush"));
59+
60+
unzipFile(path, unzippedPath);
61+
FilesystemStore fsStore = new FilesystemStore(unzippedPath);
62+
assertIsTestGroupV3(Group.open(fsStore.resolve()), true);
63+
}
64+
65+
@ParameterizedTest
66+
@CsvSource({"false", "true",})
67+
public void testZipStoreWithComment(boolean flushOnWrite) throws ZarrException, IOException {
68+
Path path = TESTOUTPUT.resolve("testZipStoreWithComment" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip");
69+
String comment = "{\"ome\": { \"version\": \"XX.YY\" }}";
70+
BufferedZipStore zipStore = new BufferedZipStore(path, comment, flushOnWrite);
71+
writeTestGroupV3(zipStore.resolve(), true);
72+
if (!flushOnWrite) zipStore.flush();
73+
74+
try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) {
75+
String retrievedComment = zipFile.getComment();
76+
Assertions.assertEquals(comment, retrievedComment, "ZIP archive comment does not match expected value.");
77+
}
78+
79+
Assertions.assertEquals(comment, new BufferedZipStore(path).getArchiveComment(), "ZIP archive comment from store does not match expected value.");
80+
}
81+
82+
/**
83+
* Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr
84+
*
85+
* @see <a href="https://ngff.openmicroscopy.org/rfc/9/index.html">RFC-9: Zipped OME-Zarr</a>
86+
*/
87+
@Test
88+
public void testZipStoreRequirements() throws ZarrException, IOException {
89+
Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip");
90+
BufferedZipStore zipStore = new BufferedZipStore(path);
91+
92+
dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(zipStore.resolve());
93+
Array array = group.createArray("a1", b -> b
94+
.withShape(1024, 1024)
95+
.withDataType(dev.zarr.zarrjava.v3.DataType.UINT32)
96+
.withChunkShape(512, 512)
97+
);
98+
array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testDataInt()), true);
99+
100+
dev.zarr.zarrjava.v3.Group g1 = group.createGroup("g1");
101+
g1.createGroup("g1_1").createGroup("g1_1_1");
102+
g1.createGroup("g1_2");
103+
group.createGroup("g2").createGroup("g2_1");
104+
group.createGroup("g3");
105+
106+
zipStore.flush();
107+
108+
try (ZipFile zip = new ZipFile(path.toFile())) {
109+
ArrayList<ZipArchiveEntry> entries = Collections.list(zip.getEntries());
110+
111+
// no compression
112+
for (ZipArchiveEntry e : entries) {
113+
Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed");
114+
}
115+
116+
// correct order of zarr.json files
117+
String[] expectedFirstEntries = new String[]{
118+
"zarr.json",
119+
"a1/zarr.json",
120+
"g1/zarr.json",
121+
"g2/zarr.json",
122+
"g3/zarr.json",
123+
"g1/g1_1/zarr.json",
124+
"g1/g1_2/zarr.json",
125+
"g2/g2_1/zarr.json",
126+
"g1/g1_1/g1_1_1/zarr.json"
127+
};
128+
String[] actualFirstEntries = entries.stream()
129+
.map(ZipArchiveEntry::getName)
130+
.limit(expectedFirstEntries.length)
131+
.toArray(String[]::new);
132+
133+
Assertions.assertArrayEquals(expectedFirstEntries, actualFirstEntries, "zarr.json files are not in the expected breadth-first order");
134+
}
135+
}
136+
137+
@ParameterizedTest
138+
@CsvSource({"false", "true",})
139+
public void testZipStoreV2(boolean flushOnWrite) throws ZarrException, IOException {
140+
Path path = TESTOUTPUT.resolve("testZipStoreV2" + (flushOnWrite ? "Flush" : "NoFlush") + ".zip");
141+
BufferedZipStore zipStore = new BufferedZipStore(path, flushOnWrite);
142+
writeTestGroupV2(zipStore.resolve(), true);
143+
if (!flushOnWrite) zipStore.flush();
144+
145+
BufferedZipStore zipStoreRead = new BufferedZipStore(path);
146+
assertIsTestGroupV2(Group.open(zipStoreRead.resolve()), true);
147+
148+
Path unzippedPath = TESTOUTPUT.resolve("testZipStoreV2Unzipped");
149+
150+
unzipFile(path, unzippedPath);
151+
FilesystemStore fsStore = new FilesystemStore(unzippedPath);
152+
assertIsTestGroupV2(Group.open(fsStore.resolve()), true);
153+
}
154+
155+
156+
@Override
157+
Store writableStore() {
158+
Path path = TESTOUTPUT.resolve("writableStore.ZIP");
159+
if (Files.exists(path)) {
160+
try{
161+
Files.delete(path);
162+
}catch (IOException e) {
163+
throw new RuntimeException("Failed to delete existing test ZIP store at: " + path.toAbsolutePath(), e);
164+
}
165+
}
166+
return new BufferedZipStore(TESTOUTPUT.resolve("writableStore.ZIP"), true);
167+
}
168+
}

‎src/test/java/dev/zarr/zarrjava/store/FileSystemStoreTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
package dev.zarr.zarrjava.store;
2+
3+
import com.fasterxml.jackson.databind.ObjectMapper;
4+
import dev.zarr.zarrjava.ZarrException;
5+
import dev.zarr.zarrjava.core.*;
6+
import org.junit.jupiter.api.Assertions;
7+
import org.junit.jupiter.api.Test;
8+
9+
import java.io.IOException;
10+
import java.nio.file.Files;
11+
12+
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
13+
14+
public class FileSystemStoreTest extends WritableStoreTest{
15+
16+
@Override
17+
StoreHandle storeHandleWithData() {
18+
return new FilesystemStore(TESTDATA).resolve("l4_sample", "zarr.json");
19+
}
20+
21+
@Test
22+
public void testFileSystemStores() throws IOException, ZarrException {
23+
FilesystemStore fsStore = new FilesystemStore(TESTDATA);
24+
ObjectMapper objectMapper = makeObjectMapper();
25+
26+
GroupMetadata groupMetadata = objectMapper.readValue(
27+
Files.readAllBytes(TESTDATA.resolve("l4_sample").resolve("zarr.json")),
28+
dev.zarr.zarrjava.v3.GroupMetadata.class
29+
);
30+
31+
String groupMetadataString = objectMapper.writeValueAsString(groupMetadata);
32+
Assertions.assertTrue(groupMetadataString.contains("\"zarr_format\":3"));
33+
Assertions.assertTrue(groupMetadataString.contains("\"node_type\":\"group\""));
34+
35+
ArrayMetadata arrayMetadata = objectMapper.readValue(Files.readAllBytes(TESTDATA.resolve(
36+
"l4_sample").resolve("color").resolve("1").resolve("zarr.json")),
37+
dev.zarr.zarrjava.v3.ArrayMetadata.class);
38+
39+
String arrayMetadataString = objectMapper.writeValueAsString(arrayMetadata);
40+
Assertions.assertTrue(arrayMetadataString.contains("\"zarr_format\":3"));
41+
Assertions.assertTrue(arrayMetadataString.contains("\"node_type\":\"array\""));
42+
Assertions.assertTrue(arrayMetadataString.contains("\"shape\":[1,4096,4096,2048]"));
43+
44+
Assertions.assertInstanceOf(Array.class, Array.open(fsStore.resolve("l4_sample", "color", "1")));
45+
46+
Node[] subNodes = Group.open(fsStore.resolve("l4_sample")).list().toArray(Node[]::new);
47+
Assertions.assertEquals(12, subNodes.length);
48+
49+
Array[] colorSubNodes = ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).list().toArray(Array[]::new);
50+
51+
Assertions.assertEquals(5, colorSubNodes.length);
52+
Assertions.assertInstanceOf(Array.class, colorSubNodes[0]);
53+
54+
Array array = (Array) ((Group) Group.open(fsStore.resolve("l4_sample")).get("color")).get("1");
55+
Assertions.assertArrayEquals(new long[]{1, 4096, 4096, 2048}, array.metadata().shape);
56+
}
57+
58+
@Override
59+
Store writableStore() {
60+
return new FilesystemStore(TESTOUTPUT.resolve("writableFSStore"));
61+
}
62+
}

‎src/test/java/dev/zarr/zarrjava/store/HttpStoreTest.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package dev.zarr.zarrjava.store;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.core.Array;
5+
import org.junit.jupiter.api.Assertions;
6+
import org.junit.jupiter.api.Test;
7+
8+
import java.io.IOException;
9+
10+
public class HttpStoreTest extends StoreTest {
11+
12+
@Override
13+
StoreHandle storeHandleWithData() {
14+
HttpStore httpStore = new dev.zarr.zarrjava.store.HttpStore("https://uk1s3.embassy.ebi.ac.uk/idr/zarr/v0.5/idr0033A");
15+
return httpStore.resolve("BR00109990_C2.zarr", "0", "0");
16+
17+
}
18+
19+
@Test
20+
public void testHttpStore() throws IOException, ZarrException {
21+
Array array = Array.open(storeHandleWithData());
22+
Assertions.assertArrayEquals(new long[]{5, 1552, 2080}, array.metadata().shape);
23+
}
24+
25+
}

‎src/test/java/dev/zarr/zarrjava/store/MemoryStoreTest.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,77 @@
1+
package dev.zarr.zarrjava.store;
2+
3+
import dev.zarr.zarrjava.ZarrException;
4+
import dev.zarr.zarrjava.core.Array;
5+
import dev.zarr.zarrjava.core.Attributes;
6+
import dev.zarr.zarrjava.core.Node;
7+
import org.junit.jupiter.api.Assertions;
8+
import org.junit.jupiter.params.ParameterizedTest;
9+
import org.junit.jupiter.params.provider.CsvSource;
10+
11+
import java.io.IOException;
12+
import java.nio.ByteBuffer;
13+
import java.util.stream.Stream;
14+
15+
public class MemoryStoreTest extends WritableStoreTest {
16+
17+
@ParameterizedTest
18+
@CsvSource({"false", "true",})
19+
public void testMemoryStoreV3(boolean useParallel) throws ZarrException, IOException {
20+
int[] testData = testDataInt();
21+
22+
dev.zarr.zarrjava.v3.Group group = dev.zarr.zarrjava.v3.Group.create(new MemoryStore().resolve());
23+
Array array = group.createArray("array", b -> b
24+
.withShape(1024, 1024)
25+
.withDataType(dev.zarr.zarrjava.v3.DataType.UINT32)
26+
.withChunkShape(64, 64)
27+
);
28+
array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel);
29+
group.createGroup("subgroup");
30+
group.setAttributes(new Attributes(b -> b.set("some", "value")));
31+
Stream<Node> nodes = group.list();
32+
Assertions.assertEquals(2, nodes.count());
33+
34+
ucar.ma2.Array result = array.read(useParallel);
35+
Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
36+
Attributes attrs = group.metadata().attributes;
37+
Assertions.assertNotNull(attrs);
38+
Assertions.assertEquals("value", attrs.getString("some"));
39+
}
40+
41+
@ParameterizedTest
42+
@CsvSource({"false", "true",})
43+
public void testMemoryStoreV2(boolean useParallel) throws ZarrException, IOException {
44+
int[] testData = testDataInt();
45+
46+
dev.zarr.zarrjava.v2.Group group = dev.zarr.zarrjava.v2.Group.create(new MemoryStore().resolve());
47+
dev.zarr.zarrjava.v2.Array array = group.createArray("array", b -> b
48+
.withShape(1024, 1024)
49+
.withDataType(dev.zarr.zarrjava.v2.DataType.UINT32)
50+
.withChunks(512, 512)
51+
);
52+
array.write(ucar.ma2.Array.factory(ucar.ma2.DataType.UINT, new int[]{1024, 1024}, testData), useParallel);
53+
group.createGroup("subgroup");
54+
Stream<dev.zarr.zarrjava.core.Node> nodes = group.list();
55+
group.setAttributes(new Attributes().set("description", "test group"));
56+
Assertions.assertEquals(2, nodes.count());
57+
58+
ucar.ma2.Array result = array.read(useParallel);
59+
Assertions.assertArrayEquals(testData, (int[]) result.get1DJavaArray(ucar.ma2.DataType.UINT));
60+
Attributes attrs = group.metadata().attributes;
61+
Assertions.assertNotNull(attrs);
62+
Assertions.assertEquals("test group", attrs.getString("description"));
63+
64+
}
65+
66+
@Override
67+
Store writableStore() {
68+
return new MemoryStore();
69+
}
70+
71+
@Override
72+
StoreHandle storeHandleWithData() {
73+
StoreHandle memoryStoreHandle = new MemoryStore().resolve();
74+
memoryStoreHandle.set(ByteBuffer.wrap(testData()));
75+
return memoryStoreHandle;
76+
}
77+
}

0 commit comments

Comments
 (0)