Skip to content

Commit a1e3601

Browse files
brokkoli71brokkoli71
committed
change store::list to list no directories
add store::listChildren containing direct children including directories
1 parent 2e048f7 commit a1e3601

6 files changed

Lines changed: 222 additions & 71 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -245,6 +245,11 @@ public Stream<String[]> list(String[] keys) {
245245
return bufferStore.list(keys);
246246
}
247247

248+
@Override
249+
public Stream<String[]> listChildren(String[] prefix) {
250+
return bufferStore.listChildren(prefix);
251+
}
252+
248253
@Override
249254
public boolean exists(String[] keys) {
250255
return bufferStore.exists(keys);

‎src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java‎

Lines changed: 36 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -120,21 +120,44 @@ public void delete(String[] keys) {
120120
}
121121
}
122122

123-
public Stream<String[]> list(String[] keys) {
124-
Path keyPath = resolveKeys(keys);
123+
/**
124+
* Helper to convert a filesystem Path back into the full String[] key array
125+
* relative to the prefix
126+
*/
127+
private String[] pathToKeyArray(Path rootPath, Path currentPath, String[] prefix) {
128+
Path relativePath = rootPath.relativize(currentPath);
129+
int relativeCount = relativePath.getNameCount();
130+
131+
String[] result = new String[relativeCount];
132+
for (int i = 0; i < relativeCount; i++) {
133+
result[i] = relativePath.getName(i).toString();
134+
}
135+
return result;
136+
}
137+
138+
@Override
139+
public Stream<String[]> list(String[] prefix) {
140+
Path rootPath = resolveKeys(prefix);
125141
try {
126-
return Files.walk(keyPath)
127-
.filter(path -> !path.equals(keyPath))
128-
.map(path -> {
129-
Path relativePath = keyPath.relativize(path);
130-
String[] parts = new String[relativePath.getNameCount()];
131-
for (int i = 0; i < relativePath.getNameCount(); i++) {
132-
parts[i] = relativePath.getName(i).toString();
133-
}
134-
return parts;
135-
});
142+
return Files.walk(rootPath)
143+
.filter(Files::isRegularFile)
144+
.map(path -> pathToKeyArray(rootPath, path, prefix));
136145
} catch (IOException e) {
137-
throw new RuntimeException(e);
146+
throw new RuntimeException("Failed to list store content", e);
147+
}
148+
}
149+
150+
@Override
151+
public Stream<String[]> listChildren(String[] prefix) {
152+
Path rootPath = resolveKeys(prefix);
153+
if (!Files.exists(rootPath) || !Files.isDirectory(rootPath)) {
154+
return Stream.empty();
155+
}
156+
try {
157+
return Files.list(rootPath) // note: Files.list is non-recursive
158+
.map(path -> pathToKeyArray(rootPath, path, prefix));
159+
} catch (IOException e) {
160+
throw new RuntimeException("Failed to list store children", e);
138161
}
139162
}
140163

‎src/main/java/dev/zarr/zarrjava/store/MemoryStore.java‎

Lines changed: 26 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -60,18 +60,32 @@ public void delete(String[] keys) {
6060
map.remove(resolveKeys(keys));
6161
}
6262

63-
public Stream<String[]> list(String[] keys) {
64-
List<String> prefix = resolveKeys(keys);
65-
Set<List<String>> allKeys = new HashSet<>();
66-
67-
for (List<String> k : map.keySet()) {
68-
if (k.size() <= prefix.size() || !k.subList(0, prefix.size()).equals(prefix))
69-
continue;
70-
for (int i = prefix.size(); i < k.size(); i++) {
71-
allKeys.add(k.subList(prefix.size(), i + 1));
72-
}
73-
}
74-
return allKeys.stream().map(k -> k.toArray(new String[0]));
63+
@Override
64+
public Stream<String[]> list(String[] prefix) {
65+
List<String> prefixList = resolveKeys(prefix);
66+
int prefixSize = prefixList.size();
67+
68+
return map.keySet().stream()
69+
.filter(key -> key.size() >= prefixSize && key.subList(0, prefixSize).equals(prefixList))
70+
.map(key -> key.subList(prefixSize, key.size()).toArray(new String[0]));
71+
}
72+
73+
@Override
74+
public Stream<String[]> listChildren(String[] prefix) {
75+
List<String> prefixList = resolveKeys(prefix);
76+
int prefixSize = prefixList.size();
77+
78+
return map.keySet().stream()
79+
.filter(key -> key.size() > prefixSize && key.subList(0, prefixSize).equals(prefixList))
80+
// Identify the immediate child segment
81+
// e.g. if prefix is [a], and key is [a, b, c], the child is [a, b]
82+
.map(key -> {
83+
List<String> childPath = new ArrayList<>(prefixList);
84+
childPath.add(key.get(prefixSize));
85+
return childPath;
86+
})
87+
.distinct()
88+
.map(list -> list.toArray(new String[0]));
7589
}
7690

7791
@Nonnull

‎src/main/java/dev/zarr/zarrjava/store/ReadOnlyZipStore.java‎

Lines changed: 52 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -125,39 +125,69 @@ public String toString() {
125125
}
126126

127127
@Override
128-
public Stream<String[]> list(String[] keys) {
128+
public Stream<String[]> list(String[] prefixKeys) {
129129
Stream.Builder<String[]> builder = Stream.builder();
130-
131130
InputStream inputStream = underlyingStore.getInputStream();
132-
if (inputStream == null) {
133-
return builder.build();
131+
if (inputStream == null) return builder.build();
132+
133+
String prefix = resolveKeys(prefixKeys);
134+
if (!prefix.isEmpty() && !prefix.endsWith("/")) {
135+
prefix += "/";
134136
}
137+
135138
try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) {
136139
ZipArchiveEntry entry;
137-
String prefix = resolveKeys(keys);
138140
while ((entry = zis.getNextEntry()) != null) {
139-
String entryName = entry.getName();
140-
if (entryName.startsWith("/")) {
141-
entryName = entryName.substring(1);
142-
}
143-
if (entryName.endsWith("/")) {
144-
entryName = entryName.substring(0, entryName.length() - 1);
141+
String name = normalizeEntryName(entry.getName());
142+
if (name.startsWith(prefix) && !entry.isDirectory()) {
143+
builder.add(resolveEntryKeys(name.substring(prefix.length())));
145144
}
146-
if (!entryName.startsWith(prefix) || entryName.equals(prefix)) {
147-
continue;
148-
}
149-
entryName = entryName.substring(prefix.length());
150-
if (entryName.startsWith("/")) {
151-
entryName = entryName.substring(1);
152-
}
153-
String[] entryKeys = resolveEntryKeys(entryName);
154-
builder.add(entryKeys);
155145
}
156-
} catch (IOException ignored) {
157-
}
146+
} catch (IOException ignored) {}
158147
return builder.build();
159148
}
160149

150+
@Override
151+
public Stream<String[]> listChildren(String[] prefixKeys) {
152+
java.util.Set<String> children = new java.util.LinkedHashSet<>();
153+
InputStream inputStream = underlyingStore.getInputStream();
154+
if (inputStream == null) return Stream.empty();
155+
156+
String prefix = resolveKeys(prefixKeys);
157+
if (!prefix.isEmpty() && !prefix.endsWith("/")) {
158+
prefix += "/";
159+
}
160+
161+
try (ZipArchiveInputStream zis = new ZipArchiveInputStream(inputStream)) {
162+
ZipArchiveEntry entry;
163+
while ((entry = zis.getNextEntry()) != null) {
164+
String name = normalizeEntryName(entry.getName());
165+
166+
if (name.startsWith(prefix) && !name.equals(prefix)) {
167+
String relative = name.substring(prefix.length());
168+
String[] parts = relative.split("/");
169+
// The child is the prefix + the very next segment
170+
String childSegment = parts[0];
171+
children.add(childSegment);
172+
}
173+
}
174+
} catch (IOException ignored) {}
175+
176+
return children.stream().map(segment -> {
177+
String[] result = new String[prefixKeys.length + 1];
178+
System.arraycopy(prefixKeys, 0, result, 0, prefixKeys.length);
179+
result[prefixKeys.length] = segment;
180+
return result;
181+
});
182+
}
183+
184+
private String normalizeEntryName(String name) {
185+
if (name.startsWith("/")) name = name.substring(1);
186+
if (name.endsWith("/")) name = name.substring(0, name.length() - 1);
187+
return name;
188+
}
189+
190+
161191
@Override
162192
public InputStream getInputStream(String[] keys, long start, long end) {
163193
InputStream baseStream = underlyingStore.getInputStream();

‎src/main/java/dev/zarr/zarrjava/store/S3Store.java‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,9 @@ ByteBuffer get(GetObjectRequest getObjectRequest) {
5252

5353
@Override
5454
public boolean exists(String[] keys) {
55+
if (keys == null || keys.length == 0) {
56+
return true;
57+
}
5558
HeadObjectRequest req = HeadObjectRequest.builder().bucket(bucketName).key(resolveKeys(keys)).build();
5659
try {
5760
return s3client.headObject(req).sdkHttpResponse().statusCode() == 200;
@@ -105,27 +108,89 @@ public void delete(String[] keys) {
105108
.build());
106109
}
107110

111+
// @Override
112+
// public Stream<String[]> list(String[] keys) {
113+
// final String fullKey = resolveKeys(keys);
114+
// ListObjectsRequest req = ListObjectsRequest.builder()
115+
// .bucket(bucketName).prefix(fullKey)
116+
// .build();
117+
// ListObjectsResponse res = s3client.listObjects(req);
118+
// return res.contents().stream()
119+
// .map(S3Object::key)
120+
// .flatMap(key -> {
121+
// List<String> pathSegments = new ArrayList<>();
122+
// int index = fullKey.length();
123+
// while ((index = key.indexOf('/', index + 1)) != -1) {
124+
// pathSegments.add(key.substring(fullKey.length() + 1, index));
125+
// }
126+
// pathSegments.add(key.substring(fullKey.length() + 1));
127+
// return pathSegments.stream();
128+
// })
129+
// .distinct()
130+
// .map(s -> s.split("/"))
131+
// .filter(arr -> arr.length > 0);
132+
// }
133+
108134
@Override
109135
public Stream<String[]> list(String[] keys) {
110-
final String fullKey = resolveKeys(keys);
111-
ListObjectsRequest req = ListObjectsRequest.builder()
112-
.bucket(bucketName).prefix(fullKey)
136+
String fullPrefix = resolveKeys(keys);
137+
// Ensure prefix ends with / for precise matching if not empty
138+
if (!fullPrefix.isEmpty() && !fullPrefix.endsWith("/")) {
139+
fullPrefix += "/";
140+
}
141+
142+
ListObjectsV2Request req = ListObjectsV2Request.builder()
143+
.bucket(bucketName)
144+
.prefix(fullPrefix)
113145
.build();
114-
ListObjectsResponse res = s3client.listObjects(req);
115-
return res.contents().stream()
146+
147+
final String finalFullPrefix = fullPrefix;
148+
return s3client.listObjectsV2Paginator(req).contents().stream()
116149
.map(S3Object::key)
117-
.flatMap(key -> {
118-
List<String> pathSegments = new ArrayList<>();
119-
int index = fullKey.length();
120-
while ((index = key.indexOf('/', index + 1)) != -1) {
121-
pathSegments.add(key.substring(fullKey.length() + 1, index));
122-
}
123-
pathSegments.add(key.substring(fullKey.length() + 1));
124-
return pathSegments.stream();
125-
})
126-
.distinct()
127-
.map(s -> s.split("/"))
128-
.filter(arr -> arr.length > 0);
150+
.filter(key -> !key.equals(finalFullPrefix) && !key.endsWith("/"))
151+
.map(k -> keyToRelativeArray(k, finalFullPrefix));
152+
}
153+
154+
@Override
155+
public Stream<String[]> listChildren(String[] keys) {
156+
String fullPrefix = resolveKeys(keys);
157+
if (!fullPrefix.isEmpty() && !fullPrefix.endsWith("/")) {
158+
fullPrefix += "/";
159+
}
160+
161+
ListObjectsV2Request req = ListObjectsV2Request.builder()
162+
.bucket(bucketName)
163+
.prefix(fullPrefix)
164+
.delimiter("/")
165+
.build();
166+
167+
ListObjectsV2Response res = s3client.listObjectsV2(req);
168+
169+
// Combine CommonPrefixes (folders) and Contents (files)
170+
Stream<String> folders = res.commonPrefixes().stream().map(CommonPrefix::prefix);
171+
final String finalFullPrefix = fullPrefix;
172+
Stream<String> files = res.contents().stream().map(S3Object::key)
173+
.filter(key -> !key.equals(finalFullPrefix));
174+
175+
return Stream.concat(folders, files)
176+
.map(k -> keyToRelativeArray(k, finalFullPrefix));
177+
}
178+
179+
/**
180+
* Helper to convert a full S3 key back into a String[] relative to the prefix.
181+
*/
182+
private String[] keyToRelativeArray(String fullS3Key, String prefix) {
183+
String relativePath = fullS3Key;
184+
if (prefix != null && fullS3Key.startsWith(prefix)) {
185+
relativePath = fullS3Key.substring(prefix.length());
186+
}
187+
if (relativePath.startsWith("/")) {
188+
relativePath = relativePath.substring(1);
189+
}
190+
if (relativePath.endsWith("/")) {
191+
relativePath = relativePath.substring(0, relativePath.length() - 1);
192+
}
193+
return relativePath.isEmpty() ? new String[0] : relativePath.split("/");
129194
}
130195

131196
@Nonnull

‎src/main/java/dev/zarr/zarrjava/store/Store.java‎

Lines changed: 21 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -40,19 +40,33 @@ default InputStream getInputStream(String[] keys) {
4040
*/
4141
long getSize(String[] keys);
4242

43+
/**
44+
* A store that supports discovery of keys.
45+
*/
4346
interface ListableStore extends Store {
4447

4548
/**
46-
* Lists all keys in the store that match the given prefix keys. Keys are represented as arrays of strings,
47-
* where each string is a segment of the key path.
48-
* Keys that are exactly equal to the prefix are not included in the results.
49-
* Keys that do not contain data (i.e. "directories") are included in the results.
49+
* Recursively lists all keys that contain data (leaf nodes) under the given prefix
50+
* relative to the prefix.
51+
* Directory-only entries are excluded.
52+
*
53+
* @param prefix The prefix keys to match.
54+
* @return A stream of full key arrays containing data.
55+
*/
56+
Stream<String[]> list(String[] prefix);
57+
58+
/**
59+
* Lists the immediate children (files and virtual directories) under the given prefix.
60+
* This is useful for UI navigation or browsing the store hierarchy.
5061
*
51-
* @param keys The prefix keys to match.
52-
* @return A stream of key arrays that match the given prefix. Prefixed keys are not included in the results.
62+
* @param prefix The prefix keys to explore.
63+
* @return A stream of key arrays representing one level deeper than the prefix.
5364
*/
54-
Stream<String[]> list(String[] keys);
65+
Stream<String[]> listChildren(String[] prefix);
5566

67+
/**
68+
* Lists all data-bearing keys in the entire store.
69+
*/
5670
default Stream<String[]> list() {
5771
return list(new String[]{});
5872
}

0 commit comments

Comments
 (0)