Skip to content

Commit 68c671a

Browse files
brokkoli71brokkoli71
committed
Merge branch 'main' into further-tests
# Conflicts: # src/main/java/dev/zarr/zarrjava/core/Array.java
2 parents 567ae11 + 4e4eea9 commit 68c671a

20 files changed

Lines changed: 423 additions & 158 deletions

‎src/main/java/dev/zarr/zarrjava/core/Array.java‎

Lines changed: 23 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
8989
throw new IllegalArgumentException("'array' needs to have rank '" + metadata.ndim() + "'.");
9090
}
9191

92-
int[] shape = array.getShape();
92+
long[] shape = Utils.toLongArray(array.getShape());
9393

9494
final int[] chunkShape = metadata.chunkShape();
9595
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
@@ -119,8 +119,14 @@ public void write(long[] offset, ucar.ma2.Array array, boolean parallel) {
119119
);
120120
}
121121
writeChunk(chunkCoords, chunkArray);
122-
} catch (ZarrException | InvalidRangeException e) {
123-
throw new RuntimeException(e);
122+
} catch (ZarrException e) {
123+
throw new RuntimeException(
124+
"Failed to write chunk at coordinates " + Arrays.toString(chunkCoords) +
125+
": " + e.getMessage(), e);
126+
} catch (InvalidRangeException e) {
127+
throw new RuntimeException(
128+
"Invalid array range when writing chunk at coordinates " + Arrays.toString(chunkCoords) +
129+
": " + e.getMessage(), e);
124130
}
125131
});
126132

@@ -322,7 +328,7 @@ public void write(ucar.ma2.Array array, boolean parallel) {
322328
*/
323329
@Nonnull
324330
public ucar.ma2.Array read() throws ZarrException {
325-
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape));
331+
return read(new long[metadata().ndim()], metadata().shape);
326332
}
327333

328334
/**
@@ -334,7 +340,7 @@ public ucar.ma2.Array read() throws ZarrException {
334340
* @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails
335341
*/
336342
@Nonnull
337-
public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrException {
343+
public ucar.ma2.Array read(final long[] offset, final long[] shape) throws ZarrException {
338344
return read(offset, shape, DEFAULT_PARALLELISM);
339345
}
340346

@@ -346,7 +352,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape) throws ZarrEx
346352
*/
347353
@Nonnull
348354
public ucar.ma2.Array read(final boolean parallel) throws ZarrException {
349-
return read(new long[metadata().ndim()], Utils.toIntArray(metadata().shape), parallel);
355+
return read(new long[metadata().ndim()], metadata().shape, parallel);
350356
}
351357

352358
boolean chunkIsInArray(long[] chunkCoords) {
@@ -369,7 +375,7 @@ boolean chunkIsInArray(long[] chunkCoords) {
369375
* @throws ZarrException throws ZarrException if the requested data is outside the array's domain or if the read fails
370376
*/
371377
@Nonnull
372-
public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean parallel) throws ZarrException {
378+
public ucar.ma2.Array read(final long[] offset, final long[] shape, final boolean parallel) throws ZarrException {
373379
ArrayMetadata metadata = metadata();
374380
if (offset.length != metadata.ndim()) {
375381
throw new IllegalArgumentException("'offset' needs to have rank '" + metadata.ndim() + "'.");
@@ -389,7 +395,7 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
389395
}
390396

391397
final ucar.ma2.Array outputArray = ucar.ma2.Array.factory(metadata.dataType().getMA2DataType(),
392-
shape);
398+
Utils.toIntArray(shape));
393399
Stream<long[]> chunkStream = Arrays.stream(IndexingUtils.computeChunkCoords(metadata.shape, chunkShape, offset, shape));
394400
if (parallel) {
395401
chunkStream = chunkStream.parallel();
@@ -412,18 +418,19 @@ public ucar.ma2.Array read(final long[] offset, final int[] shape, final boolean
412418
final String[] chunkKeys = metadata.chunkKeyEncoding().encodeChunkKey(chunkCoords);
413419
final StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
414420

415-
if (!chunkHandle.exists()) return;
416-
417421
if (codecPipeline.supportsPartialDecode()) {
418422
final ucar.ma2.Array chunkArray = codecPipeline.decodePartial(chunkHandle,
419423
Utils.toLongArray(chunkProjection.chunkOffset), chunkProjection.shape);
420424
MultiArrayUtils.copyRegion(chunkArray, new int[metadata.ndim()], outputArray,
421425
chunkProjection.outOffset, chunkProjection.shape
422426
);
423427
} else {
424-
MultiArrayUtils.copyRegion(readChunk(chunkCoords), chunkProjection.chunkOffset,
425-
outputArray, chunkProjection.outOffset, chunkProjection.shape
426-
);
428+
ByteBuffer chunkBytes = chunkHandle.read();
429+
if (chunkBytes != null) {
430+
MultiArrayUtils.copyRegion(codecPipeline.decode(chunkBytes), chunkProjection.chunkOffset,
431+
outputArray, chunkProjection.outOffset, chunkProjection.shape
432+
);
433+
}
427434
}
428435

429436
} catch (ZarrException e) {
@@ -481,7 +488,7 @@ public static final class ArrayAccessor {
481488
@Nullable
482489
long[] offset;
483490
@Nullable
484-
int[] shape;
491+
long[] shape;
485492
@Nonnull
486493
Array array;
487494

@@ -498,13 +505,13 @@ public ArrayAccessor withOffset(@Nonnull long... offset) {
498505

499506
@Nonnull
500507
public ArrayAccessor withShape(@Nonnull int... shape) {
501-
this.shape = shape;
508+
this.shape = Utils.toLongArray(shape);
502509
return this;
503510
}
504511

505512
@Nonnull
506513
public ArrayAccessor withShape(@Nonnull long... shape) {
507-
this.shape = Utils.toIntArray(shape);
514+
this.shape = shape;
508515
return this;
509516
}
510517

‎src/main/java/dev/zarr/zarrjava/core/codec/CodecPipeline.java‎

Lines changed: 3 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,9 @@ ArrayBytesCodec getArrayBytesCodec() {
6868
return (ArrayBytesCodec) codec;
6969
}
7070
}
71-
throw new RuntimeException(
72-
"Unreachable because the existence of exactly 1 ArrayBytes codec is asserted upon construction.");
71+
throw new IllegalStateException(
72+
"No ArrayBytesCodec found in codec pipeline. This should never happen as the existence " +
73+
"of exactly 1 ArrayBytesCodec is validated during construction.");
7374
}
7475

7576
BytesBytesCodec[] getBytesBytesCodecs() {
@@ -158,18 +159,4 @@ public long computeEncodedSize(long inputByteLength, CoreArrayMetadata arrayMeta
158159
}
159160
return inputByteLength;
160161
}
161-
162-
public Array partialDecode(
163-
StoreHandle valueHandle, long[] offset, int[] shape,
164-
CoreArrayMetadata arrayMetadata
165-
) {
166-
return null; // TODO
167-
}
168-
169-
public ByteBuffer partialEncode(
170-
StoreHandle oldValueHandle, Array array, long[] offset, int[] shape,
171-
CoreArrayMetadata arrayMetadata
172-
) {
173-
return null; // TODO
174-
}
175162
}

‎src/main/java/dev/zarr/zarrjava/core/codec/core/BytesCodec.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -102,7 +102,7 @@ public ByteOrder getByteOrder() {
102102
case BIG:
103103
return ByteOrder.BIG_ENDIAN;
104104
default:
105-
throw new RuntimeException("Unreachable");
105+
throw new IllegalStateException("Unknown endian type: " + this);
106106
}
107107
}
108108
}

‎src/main/java/dev/zarr/zarrjava/store/BufferedZipStore.java‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,10 @@ public BufferedZipStore(@Nonnull StoreHandle underlyingStore, @Nonnull Store.Lis
5656
try {
5757
loadBuffer();
5858
} catch (IOException e) {
59-
throw new RuntimeException("Failed to load buffer from underlying store", e);
59+
throw StoreException.readFailed(
60+
underlyingStore.toString(),
61+
new String[]{},
62+
new IOException("Failed to load ZIP buffer from underlying store: " + underlyingStore, e));
6063
}
6164
}
6265

@@ -285,7 +288,10 @@ public void set(String[] keys, ByteBuffer bytes) {
285288
try {
286289
writeBuffer();
287290
} catch (IOException e) {
288-
throw new RuntimeException("Failed to flush buffer to underlying store after set operation", e);
291+
throw StoreException.writeFailed(
292+
underlyingStore.toString(),
293+
keys,
294+
new IOException("Failed to flush ZIP buffer to underlying store after set operation", e));
289295
}
290296
}
291297
}
@@ -297,7 +303,10 @@ public void delete(String[] keys) {
297303
try {
298304
writeBuffer();
299305
} catch (IOException e) {
300-
throw new RuntimeException("Failed to flush buffer to underlying store after delete operation", e);
306+
throw StoreException.deleteFailed(
307+
underlyingStore.toString(),
308+
keys,
309+
new IOException("Failed to flush ZIP buffer to underlying store after delete operation", e));
301310
}
302311
}
303312
}

‎src/main/java/dev/zarr/zarrjava/store/FilesystemStore.java‎

Lines changed: 54 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -30,7 +30,13 @@ Path resolveKeys(String[] keys) {
3030
for (String key : keys) {
3131
newPath = newPath.resolve(key);
3232
}
33-
return newPath;
33+
Path absRoot = path.toAbsolutePath().normalize();
34+
Path absTarget = newPath.toAbsolutePath().normalize();
35+
36+
if (!absTarget.startsWith(absRoot)) {
37+
throw new IllegalArgumentException("Key resolves outside of store root: " + absTarget);
38+
}
39+
return newPath.normalize();
3440
}
3541

3642
@Override
@@ -43,8 +49,10 @@ public boolean exists(String[] keys) {
4349
public ByteBuffer get(String[] keys) {
4450
try {
4551
return ByteBuffer.wrap(Files.readAllBytes(resolveKeys(keys)));
46-
} catch (IOException e) {
52+
} catch (NoSuchFileException e) {
4753
return null;
54+
} catch (IOException e) {
55+
throw StoreException.readFailed(this.toString(), keys, e);
4856
}
4957
}
5058

@@ -64,8 +72,10 @@ public ByteBuffer get(String[] keys, long start) {
6472
byteChannel.read(bytes);
6573
bytes.rewind();
6674
return bytes;
67-
} catch (IOException e) {
75+
} catch (NoSuchFileException e) {
6876
return null;
77+
} catch (IOException e) {
78+
throw StoreException.readFailed(this.toString(), keys, e);
6979
}
7080
}
7181

@@ -84,8 +94,10 @@ public ByteBuffer get(String[] keys, long start, long end) {
8494
byteChannel.read(bytes);
8595
bytes.rewind();
8696
return bytes;
87-
} catch (IOException e) {
97+
} catch (NoSuchFileException e) {
8898
return null;
99+
} catch (IOException e) {
100+
throw StoreException.readFailed(this.toString(), keys, e);
89101
}
90102
}
91103

@@ -96,7 +108,10 @@ public void set(String[] keys, ByteBuffer bytes) {
96108
try {
97109
Files.createDirectories(keyPath.getParent());
98110
} catch (IOException e) {
99-
throw new RuntimeException(e);
111+
throw StoreException.writeFailed(
112+
this.toString(),
113+
keys,
114+
new IOException("Failed to create parent directories for path: " + keyPath.getParent(), e));
100115
}
101116
try (SeekableByteChannel channel = Files.newByteChannel(keyPath.toAbsolutePath(),
102117
StandardOpenOption.CREATE,
@@ -105,18 +120,25 @@ public void set(String[] keys, ByteBuffer bytes) {
105120
)) {
106121
channel.write(bytes);
107122
} catch (IOException e) {
108-
throw new RuntimeException(e);
123+
throw StoreException.writeFailed(
124+
this.toString(),
125+
keys,
126+
new IOException("Failed to write " + bytes.remaining() + " bytes to file: " + keyPath, e));
109127
}
110128
}
111129

112130
@Override
113131
public void delete(String[] keys) {
132+
Path keyPath = resolveKeys(keys);
114133
try {
115-
Files.delete(resolveKeys(keys));
134+
Files.delete(keyPath);
116135
} catch (NoSuchFileException e) {
117-
// ignore
136+
// ignore - file doesn't exist, which is the desired outcome
118137
} catch (IOException e) {
119-
throw new RuntimeException(e);
138+
throw StoreException.deleteFailed(
139+
this.toString(),
140+
keys,
141+
new IOException("Failed to delete file: " + keyPath, e));
120142
}
121143
}
122144

@@ -143,7 +165,10 @@ public Stream<String[]> list(String[] prefix) {
143165
.filter(Files::isRegularFile)
144166
.map(path -> pathToKeyArray(rootPath, path, prefix));
145167
} catch (IOException e) {
146-
throw new RuntimeException("Failed to list store content", e);
168+
throw StoreException.listFailed(
169+
this.toString(),
170+
prefix,
171+
new IOException("Failed to walk directory tree at: " + rootPath, e));
147172
}
148173
}
149174

@@ -156,7 +181,10 @@ public Stream<String> listChildren(String[] prefix) {
156181
try {
157182
return Files.list(rootPath).map(path -> path.getFileName().toString());
158183
} catch (IOException e) {
159-
throw new RuntimeException("Failed to list store children", e);
184+
throw StoreException.listFailed(
185+
this.toString(),
186+
prefix,
187+
new IOException("Failed to list directory contents at: " + rootPath, e));
160188
}
161189
}
162190

@@ -175,14 +203,12 @@ public String toString() {
175203
public InputStream getInputStream(String[] keys, long start, long end) {
176204
Path keyPath = resolveKeys(keys);
177205
try {
178-
if (!Files.exists(keyPath)) {
179-
return null;
180-
}
181206
InputStream inputStream = Files.newInputStream(keyPath);
182207
if (start > 0) {
183208
long skipped = inputStream.skip(start);
184209
if (skipped < start) {
185-
throw new IOException("Unable to skip to the desired start position.");
210+
throw new IOException("Unable to skip to position " + start +
211+
", only skipped " + skipped + " bytes in file: " + keyPath);
186212
}
187213
}
188214
if (end != -1) {
@@ -191,18 +217,28 @@ public InputStream getInputStream(String[] keys, long start, long end) {
191217
} else {
192218
return inputStream;
193219
}
220+
} catch (NoSuchFileException e) {
221+
return null;
194222
} catch (IOException e) {
195-
throw new RuntimeException(e);
223+
throw StoreException.readFailed(
224+
this.toString(),
225+
keys,
226+
new IOException("Failed to open input stream for file: " + keyPath +
227+
" (start: " + start + ", end: " + end + ")", e));
196228
}
197229
}
198230

199231
public long getSize(String[] keys) {
232+
Path keyPath = resolveKeys(keys);
200233
try {
201-
return Files.size(resolveKeys(keys));
234+
return Files.size(keyPath);
202235
} catch (NoSuchFileException e) {
203236
return -1;
204237
} catch (IOException e) {
205-
throw new RuntimeException(e);
238+
throw StoreException.readFailed(
239+
this.toString(),
240+
keys,
241+
new IOException("Failed to get file size for: " + keyPath, e));
206242
}
207243
}
208244
}

0 commit comments

Comments
 (0)