Skip to content

Commit 8c6511f

Browse files
committed
Improve ZIP store performance and filesystem listing
- Replace stream-based ZIP access with random-access `ZipFile` for faster and more efficient reads - Optimize ZIP store initialization by reading the ZIP index only, avoiding full stream traversal - Introduce improved caching of directory structure and file sizes using synchronized maps - Simplify internal logic and remove unnecessary dependencies - Improve filesystem listing performance Details: - Removed dependency on Apache Commons ZipArchiveInputStream and related classes in ReadOnlyZipStore - Added efficient caching using maps for directories and file sizes, async friendly - Normalized entry names for consistent lookup - Reduced redundant computations (e.g. cached entry sizes) - Simplified stream handling and chunk calculation These changes significantly improve performance, reduce complexity, and make the ZIP store implementation more maintainable.
1 parent c9a5ee1 commit 8c6511f

2 files changed

Lines changed: 400 additions & 145 deletions

File tree

src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java

Lines changed: 46 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,16 @@
1111
import java.nio.channels.SeekableByteChannel;
1212
import java.nio.file.*;
1313
import java.util.stream.Stream;
14+
import java.io.File;
15+
import java.util.stream.Collectors;
16+
import java.util.stream.Stream.Builder;
17+
import java.nio.file.attribute.BasicFileAttributes;
18+
// Java logging
19+
import java.util.logging.Logger;
20+
import java.util.logging.Level;
1421

1522
public class FilesystemStore implements Store, Store.ListableStore {
23+
private static final Logger logger = Logger.getLogger(FilesystemStore.class.getName());
1624

1725
@Nonnull
1826
private final Path path;
@@ -142,28 +150,46 @@ public void delete(String[] keys) {
142150
}
143151
}
144152

145-
/**
146-
* Helper to convert a filesystem Path back into the full String[] key array
147-
* relative to the prefix
148-
*/
149-
private String[] pathToKeyArray(Path rootPath, Path currentPath, String[] prefix) {
150-
Path relativePath = rootPath.relativize(currentPath);
151-
int relativeCount = relativePath.getNameCount();
152-
153-
String[] result = new String[relativeCount];
154-
for (int i = 0; i < relativeCount; i++) {
155-
result[i] = relativePath.getName(i).toString();
156-
}
157-
return result;
158-
}
159-
160153
@Override
161154
public Stream<String[]> list(String[] prefix) {
162155
Path rootPath = resolveKeys(prefix);
163156
try {
164-
return Files.walk(rootPath)
165-
.filter(Files::isRegularFile)
166-
.map(path -> pathToKeyArray(rootPath, path, prefix));
157+
Builder<String[]> builder = Stream.builder(); // Create a Stream.Builder to collect results
158+
// Walk the directory tree using walkFileTree
159+
Files.walkFileTree(rootPath, new SimpleFileVisitor<Path>() {
160+
@Override
161+
public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException {
162+
// Only process regular files avoids Files::isRegularFile additional IO calls
163+
if (attrs.isRegularFile()) {
164+
String[] keys = rootPath.relativize(path).toString().split(File.separator);
165+
builder.add(keys); // Add the keys to the stream builder
166+
}
167+
return FileVisitResult.CONTINUE;
168+
}
169+
170+
@Override
171+
public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException {
172+
// Called when a file could not be visited
173+
String msg = "Failed to visit file: " + file + " due to: " + exc.getMessage();
174+
logger.log(Level.WARNING, msg, exc);
175+
return FileVisitResult.CONTINUE;
176+
}
177+
178+
@Override
179+
public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {
180+
return FileVisitResult.CONTINUE;
181+
}
182+
183+
@Override
184+
public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {
185+
return FileVisitResult.CONTINUE;
186+
}
187+
});
188+
189+
return builder.build(); // Build the stream and return it
190+
191+
//return Files.walk(rootPath).parallel().filter(Files::isRegularFile) // Filter only regular files (not directories)
192+
// .map(path -> rootPath.relativize(path).toString().split(File.separator)); // Get relative path and split into keys
167193
} catch (IOException e) {
168194
throw StoreException.listFailed(
169195
this.toString(),
@@ -207,8 +233,7 @@ public InputStream getInputStream(String[] keys, long start, long end) {
207233
if (start > 0) {
208234
long skipped = inputStream.skip(start);
209235
if (skipped < start) {
210-
throw new IOException("Unable to skip to position " + start +
211-
", only skipped " + skipped + " bytes in file: " + keyPath);
236+
throw new IOException("Unable to skip to position " + start + ", only skipped " + skipped + " bytes in file: " + keyPath);
212237
}
213238
}
214239
if (end != -1) {
@@ -223,8 +248,7 @@ public InputStream getInputStream(String[] keys, long start, long end) {
223248
throw StoreException.readFailed(
224249
this.toString(),
225250
keys,
226-
new IOException("Failed to open input stream for file: " + keyPath +
227-
" (start: " + start + ", end: " + end + ")", e));
251+
new IOException("Failed to open input stream for file: " + keyPath + " (start: " + start + ", end: " + end + ")", e));
228252
}
229253
}
230254

0 commit comments

Comments
 (0)