Skip to content

Commit c525a63

Browse files
brokkoli71brokkoli71
committed
use apache commons compress for zip file read and write
1 parent f163b29 commit c525a63

3 files changed

Lines changed: 141 additions & 23 deletions

File tree

‎pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -121,6 +121,11 @@
121121
<version>4.13.1</version>
122122
<scope>test</scope>
123123
</dependency>
124+
<dependency>
125+
<groupId>org.apache.commons</groupId>
126+
<artifactId>commons-compress</artifactId>
127+
<version>1.28.0</version>
128+
</dependency>
124129
</dependencies>
125130

126131
<repositories>

‎src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java‎

Lines changed: 65 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,15 @@
99
import java.nio.file.Path;
1010
import java.nio.file.Paths;
1111
import java.util.stream.Stream;
12-
import java.util.zip.ZipEntry;
13-
import java.util.zip.ZipInputStream;
14-
import java.util.zip.ZipOutputStream;
12+
13+
import org.apache.commons.compress.archivers.ArchiveEntry;
14+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
15+
import org.apache.commons.compress.archivers.zip.ZipArchiveInputStream;
16+
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
17+
import org.apache.commons.compress.archivers.zip.Zip64Mode;
18+
19+
import java.util.zip.CRC32;
20+
import java.util.zip.ZipEntry; // for STORED constant
1521

1622

1723
/** A Store implementation that buffers reads and writes and flushes them to an underlying Store as a zip file.
@@ -20,11 +26,19 @@ public class BufferedZipStore implements Store, Store.ListableStore {
2026

2127
private final StoreHandle underlyingStore;
2228
private final Store.ListableStore bufferStore;
29+
private final String archiveComment;
2330

2431
private void writeBuffer() throws IOException{
2532
// create zip file bytes from buffer store and write to underlying store
2633
ByteArrayOutputStream baos = new ByteArrayOutputStream();
27-
try (ZipOutputStream zos = new ZipOutputStream(baos)) {
34+
try (ZipArchiveOutputStream zos = new ZipArchiveOutputStream(baos)) {
35+
// always use zip64
36+
zos.setUseZip64(Zip64Mode.Always);
37+
// set archive comment if provided
38+
if (archiveComment != null) {
39+
zos.setComment(archiveComment);
40+
}
41+
2842
// iterate all entries provided by bufferStore.list()
2943
bufferStore.list().forEach(keys -> {
3044
try {
@@ -39,17 +53,30 @@ private void writeBuffer() throws IOException{
3953
if (!entryName.endsWith("/")) {
4054
entryName = entryName + "/";
4155
}
42-
zos.putNextEntry(new ZipEntry(entryName));
43-
zos.closeEntry();
56+
ZipArchiveEntry dirEntry = new ZipArchiveEntry(entryName);
57+
dirEntry.setMethod(ZipEntry.STORED);
58+
dirEntry.setSize(0);
59+
dirEntry.setCrc(0);
60+
zos.putArchiveEntry(dirEntry);
61+
zos.closeArchiveEntry();
4462
} else {
4563
// read bytes from ByteBuffer without modifying original
4664
ByteBuffer dup = bb.duplicate();
4765
int len = dup.remaining();
4866
byte[] bytes = new byte[len];
4967
dup.get(bytes);
50-
zos.putNextEntry(new ZipEntry(entryName));
68+
69+
// compute CRC and set size for STORED (no compression)
70+
CRC32 crc = new CRC32();
71+
crc.update(bytes, 0, bytes.length);
72+
ZipArchiveEntry fileEntry = new ZipArchiveEntry(entryName);
73+
fileEntry.setMethod(ZipEntry.STORED);
74+
fileEntry.setSize(bytes.length);
75+
fileEntry.setCrc(crc.getValue());
76+
77+
zos.putArchiveEntry(fileEntry);
5178
zos.write(bytes);
52-
zos.closeEntry();
79+
zos.closeArchiveEntry();
5380
}
5481
} catch (IOException e) {
5582
// wrap checked exception so it can be rethrown from stream for handling below
@@ -76,11 +103,12 @@ private void loadBuffer() throws IOException{
76103
if (buffer == null) {
77104
return;
78105
}
79-
try (ZipInputStream zis = new ZipInputStream(new ByteBufferBackedInputStream(buffer))) {
80-
ZipEntry entry;
81-
while ((entry = zis.getNextEntry()) != null) {
106+
try (ZipArchiveInputStream zis = new ZipArchiveInputStream(new ByteBufferBackedInputStream(buffer))) {
107+
// this.archiveComment = zis.getComment();
108+
ArchiveEntry aentry;
109+
while ((aentry = zis.getNextEntry()) != null) {
110+
ZipArchiveEntry entry = (ZipArchiveEntry) aentry;
82111
if (entry.isDirectory()) {
83-
zis.closeEntry();
84112
continue;
85113
}
86114
ByteArrayOutputStream baos = new ByteArrayOutputStream();
@@ -89,49 +117,63 @@ private void loadBuffer() throws IOException{
89117
while ((read = zis.read(tmp)) != -1) {
90118
baos.write(tmp, 0, read);
91119
}
92-
93120
byte[] bytes = baos.toByteArray();
94-
System.out.println("Loading entry: " + entry.getName() + " (" + bytes.length + " bytes)");
95-
96121
bufferStore.set(new String[]{entry.getName()}, ByteBuffer.wrap(bytes));
97-
98-
zis.closeEntry();
99122
}
100123
}
101124

102125
}
103126

104-
public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) {
127+
public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore, @Nullable String archiveComment) {
105128
this.underlyingStore = underlyingStore;
106129
this.bufferStore = bufferStore;
130+
this.archiveComment = archiveComment;
107131
try {
108132
loadBuffer();
109133
} catch (IOException e) {
110134
throw new RuntimeException("Failed to load buffer from underlying store", e);
111135
}
112136
}
113137

138+
public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.ListableStore bufferStore) {
139+
this(underlyingStore, bufferStore, null);
140+
}
141+
142+
public BufferedZipStore(@Nonnull StoreHandle underlyingStore, String archiveComment) {
143+
this(underlyingStore, new MemoryStore(), archiveComment);
144+
}
145+
114146
public BufferedZipStore(@Nonnull StoreHandle underlyingStore) {
115-
this(underlyingStore, new MemoryStore());
147+
this(underlyingStore, (String) null);
148+
}
149+
150+
public BufferedZipStore(@Nonnull Path underlyingStore, String archiveComment) {
151+
this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()), archiveComment);
116152
}
117153

118154
public BufferedZipStore(@Nonnull Path underlyingStore) {
119-
this(new FilesystemStore(underlyingStore.getParent()).resolve(underlyingStore.getFileName().toString()));
120-
System.out.println("Created BufferedZipStore with underlying path: " + this.underlyingStore.toString());
155+
this(underlyingStore, null);
156+
}
121157

158+
public BufferedZipStore(@Nonnull String underlyingStorePath, String archiveComment) {
159+
this(Paths.get(underlyingStorePath), archiveComment);
122160
}
123161

124162
public BufferedZipStore(@Nonnull String underlyingStorePath) {
125-
this(Paths.get(underlyingStorePath));
163+
this(underlyingStorePath, null);
126164
}
127165

128166
/**
129-
* Flushes the buffer to the underlying store as a zip file.
167+
* Flushes the buffer and archiveComment to the underlying store as a zip file.
130168
*/
131169
public void flush() throws IOException {
132170
writeBuffer();
133171
}
134172

173+
public String getArchiveComment() {
174+
return archiveComment;
175+
}
176+
135177
@Override
136178
public Stream<String[]> list(String[] keys) {
137179
return bufferStore.list(keys);

‎src/test/java/dev/zarr/zarrjava/ZarrStoreTest.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,10 @@
44
import dev.zarr.zarrjava.core.Attributes;
55
import dev.zarr.zarrjava.store.*;
66
import dev.zarr.zarrjava.v3.*;
7+
import org.apache.commons.compress.archivers.zip.*;
8+
9+
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
10+
import org.apache.commons.compress.archivers.zip.ZipFile;
711
import org.junit.jupiter.api.Assertions;
812
import org.junit.jupiter.api.Test;
913
import org.junit.jupiter.params.ParameterizedTest;
@@ -16,11 +20,14 @@
1620
import java.io.IOException;
1721
import java.nio.file.Files;
1822
import java.util.Arrays;
23+
import java.util.Collections;
1924
import java.util.stream.Stream;
2025
import java.nio.file.Path;
26+
import java.util.zip.ZipEntry;
2127

2228
import static dev.zarr.zarrjava.Utils.unzipFile;
2329
import static dev.zarr.zarrjava.Utils.zipFile;
30+
2431
import static dev.zarr.zarrjava.v3.Node.makeObjectMapper;
2532

2633
public class ZarrStoreTest extends ZarrTest {
@@ -167,6 +174,70 @@ public void testWriteZipStore() throws ZarrException, IOException {
167174
assertIsTestGroupV3(Group.open(fsStore.resolve()), true);
168175
}
169176

177+
@Test
178+
public void testZipStoreWithComment() throws ZarrException, IOException {
179+
Path path = TESTOUTPUT.resolve("testZipStoreWithComment.zip");
180+
String comment = "{\"ome\": { \"version\": \"XX.YY\" }}";
181+
BufferedZipStore zipStore = new BufferedZipStore(path, comment);
182+
writeTestGroupV3(zipStore, true);
183+
zipStore.flush();
184+
185+
try (java.util.zip.ZipFile zipFile = new java.util.zip.ZipFile(path.toFile())) {
186+
String retrievedComment = zipFile.getComment();
187+
Assertions.assertEquals(comment, retrievedComment, "ZIP archive comment does not match expected value.");
188+
}
189+
190+
Assertions.assertEquals(comment, new BufferedZipStore(path).getArchiveComment(), "ZIP archive comment from store does not match expected value.");
191+
}
192+
193+
/**
194+
* Test that ZipStore meets requirements for underlying store of Zipped OME-Zarr
195+
* @see <a href="https://ngff.openmicroscopy.org/rfc/9/index.html">RFC-9: Zipped OME-Zarr</a>
196+
*
197+
* Features to test:
198+
* - ZIP64 format
199+
* - No ZIP-level compression
200+
* - Option to add archive comments in the ZIP file header
201+
* - Prohibit nested or multi-part ZIP archives
202+
* - "The root-level zarr.json file SHOULD be the first ZIP file entry and the first entry in the central directory header; other zarr.json files SHOULD follow immediately afterwards, in breadth-first order."
203+
*/
204+
@Test
205+
public void testZipStoreRequirements() throws ZarrException, IOException {
206+
Path path = TESTOUTPUT.resolve("testZipStoreRequirements.zip");
207+
BufferedZipStore zipStore = new BufferedZipStore(path);
208+
writeTestGroupV3(zipStore, true);
209+
zipStore.flush();
210+
211+
// test for ZIP64
212+
// List<FileHeader> fileHeaders = new ZipFile(path.toFile()).getFileHeaders();
213+
//
214+
// HeaderReader headerReader = new HeaderReader();
215+
// ZipModel zipModel = headerReader.readAllHeaders(new RandomAccessFile(generatedZipFile,
216+
// RandomAccessFileMode.READ.getValue()), buildDefaultConfig());
217+
// assertThat(zipModel.getZip64EndOfCentralDirectoryLocator()).isNotNull();
218+
// assertThat(zipModel.getZip64EndOfCentralDirectoryRecord()).isNotNull();
219+
// assertThat(zipModel.isZip64Format()).isTrue();
220+
221+
try (ZipFile zip = new ZipFile(path.toFile())) {
222+
for (ZipArchiveEntry e : Collections.list(zip.getEntries())) {
223+
System.out.println(e.getName());
224+
ZipExtraField[] extraFields = e.getExtraFields();
225+
System.out.println(extraFields.length);
226+
Assertions.assertNotNull(extraFields, "Entry " + e.getName() + " has no extra fields");
227+
Assertions.assertTrue(Arrays.stream(extraFields).anyMatch(xf -> xf instanceof Zip64ExtendedInformationExtraField),
228+
"Entry " + e.getName() + " is missing ZIP64 extra field");
229+
}
230+
}
231+
// no compression
232+
try (ZipFile zip = new ZipFile(path.toFile())) {
233+
for (ZipArchiveEntry e : Collections.list(zip.getEntries())) {
234+
Assertions.assertEquals(ZipEntry.STORED, e.getMethod(), "Entry " + e.getName() + " is compressed");
235+
}
236+
}
237+
238+
239+
}
240+
170241
static Stream<Store> localStores() {
171242
return Stream.of(
172243
new MemoryStore(),

0 commit comments

Comments
 (0)