Skip to content

Commit f379ea1

Browse files
Copilotbrokkoli71
Copilot
and
brokkoli71
committed
Implement default chunk shape and dimension separator auto-detection
Co-authored-by: brokkoli71 <44113112+brokkoli71@users.noreply.github.com>
1 parent ec50083 commit f379ea1

3 files changed

Lines changed: 229 additions & 4 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,18 @@
44
import com.fasterxml.jackson.databind.ObjectWriter;
55
import dev.zarr.zarrjava.ZarrException;
66
import dev.zarr.zarrjava.core.Attributes;
7+
import dev.zarr.zarrjava.core.chunkkeyencoding.Separator;
78
import dev.zarr.zarrjava.core.codec.CodecPipeline;
89
import dev.zarr.zarrjava.store.FilesystemStore;
910
import dev.zarr.zarrjava.store.MemoryStore;
11+
import dev.zarr.zarrjava.store.Store;
1012
import dev.zarr.zarrjava.store.StoreHandle;
1113
import dev.zarr.zarrjava.utils.Utils;
1214
import dev.zarr.zarrjava.v2.codec.Codec;
1315
import dev.zarr.zarrjava.v2.codec.core.BytesCodec;
1416

1517
import javax.annotation.Nonnull;
18+
import javax.annotation.Nullable;
1619
import java.io.IOException;
1720
import java.nio.ByteBuffer;
1821
import java.nio.file.Path;
@@ -57,6 +60,27 @@ public static Array open(StoreHandle storeHandle) throws IOException, ZarrExcept
5760
Utils.toArray(storeHandle.resolve(ZATTRS).readNonNull()),
5861
Attributes.class
5962
);
63+
64+
// Auto-detect dimension separator if not specified and array has multiple dimensions
65+
if (metadata.dimensionSeparator == null && metadata.shape.length > 1) {
66+
Separator detectedSeparator = detectDimensionSeparator(storeHandle, metadata.chunks);
67+
if (detectedSeparator != null) {
68+
// Create a new metadata instance with the detected separator
69+
metadata = new ArrayMetadata(
70+
metadata.zarrFormat,
71+
metadata.shape,
72+
metadata.chunks,
73+
metadata.dataType,
74+
metadata.parsedFillValue,
75+
metadata.order,
76+
metadata.filters,
77+
metadata.compressor,
78+
detectedSeparator,
79+
metadata.attributes
80+
);
81+
}
82+
}
83+
6084
return new Array(
6185
storeHandle,
6286
metadata
@@ -248,6 +272,70 @@ public Array updateAttributes(Function<Attributes, Attributes> attributeMapper)
248272
return setAttributes(attributeMapper.apply(metadata.attributes));
249273
}
250274

275+
/**
276+
* Detects dimension separator from existing chunk files in the store.
277+
* Similar to JZarr's ZarrArray::findNestedChunks method.
278+
*
279+
* @param storeHandle the store containing the array
280+
* @param chunks the chunk shape
281+
* @return the detected separator (SLASH or DOT), or null if detection fails
282+
*/
283+
@Nullable
284+
private static Separator detectDimensionSeparator(StoreHandle storeHandle, int[] chunks) {
285+
// Only attempt detection if store supports listing
286+
if (!(storeHandle.store instanceof Store.ListableStore)) {
287+
return null;
288+
}
289+
290+
try {
291+
// List all files in the array directory
292+
String[] chunkFiles = storeHandle.list()
293+
.filter(key -> !key.startsWith(".z")) // Exclude metadata files
294+
.toArray(String[]::new);
295+
296+
if (chunkFiles.length == 0) {
297+
// No chunks exist yet, cannot detect
298+
return null;
299+
}
300+
301+
// Build regex pattern to match nested chunks (SLASH separator)
302+
// Pattern: \d+/\d+/... for multi-dimensional arrays
303+
StringBuilder nestedPattern = new StringBuilder("\\d+");
304+
for (int i = 1; i < chunks.length; i++) {
305+
nestedPattern.append("/\\d+");
306+
}
307+
String nestedRegex = nestedPattern.toString();
308+
309+
// Check if any chunk file matches the nested pattern
310+
for (String chunkFile : chunkFiles) {
311+
if (chunkFile.matches(nestedRegex)) {
312+
return Separator.SLASH;
313+
}
314+
}
315+
316+
// Build regex pattern to match flat chunks (DOT separator)
317+
// Pattern: \d+.\d+.... for multi-dimensional arrays
318+
StringBuilder flatPattern = new StringBuilder("\\d+");
319+
for (int i = 1; i < chunks.length; i++) {
320+
flatPattern.append("\\.\\d+");
321+
}
322+
String flatRegex = flatPattern.toString();
323+
324+
// Check if any chunk file matches the flat pattern
325+
for (String chunkFile : chunkFiles) {
326+
if (chunkFile.matches(flatRegex)) {
327+
return Separator.DOT;
328+
}
329+
}
330+
331+
// Could not detect separator from existing chunks
332+
return null;
333+
} catch (Exception e) {
334+
// If listing fails, return null
335+
return null;
336+
}
337+
}
338+
251339
@Override
252340
public String toString() {
253341
return String.format("<v2.Array {%s} (%s) %s>", storeHandle,

‎src/main/java/dev/zarr/zarrjava/v2/ArrayMetadataBuilder.java‎

Lines changed: 30 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,15 @@ public ArrayMetadata build() throws ZarrException {
146146
if (shape == null) {
147147
throw new IllegalStateException("Please call `withShape` first.");
148148
}
149-
if (chunks == null) {
150-
throw new IllegalStateException("Please call `withChunks` first.");
151-
}
152149
if (dataType == null) {
153150
throw new IllegalStateException("Please call `withDataType` first.");
154151
}
152+
153+
// If chunks are not specified, calculate default chunks
154+
if (chunks == null) {
155+
chunks = calculateDefaultChunks(shape);
156+
}
157+
155158
return new ArrayMetadata(
156159
2,
157160
shape,
@@ -165,4 +168,28 @@ public ArrayMetadata build() throws ZarrException {
165168
attributes
166169
);
167170
}
171+
172+
/**
173+
* Calculate default chunk shape when not specified.
174+
* Similar to JZarr's ArrayParams.build() logic, targeting chunks of about size 512.
175+
*/
176+
private int[] calculateDefaultChunks(long[] shape) {
177+
int[] chunks = new int[shape.length];
178+
for (int i = 0; i < shape.length; i++) {
179+
long shapeDim = shape[i];
180+
int numChunks = (int) (shapeDim / 512);
181+
if (numChunks > 0) {
182+
int chunkDim = (int) (shapeDim / (numChunks + 1));
183+
if (shapeDim % chunkDim == 0) {
184+
chunks[i] = chunkDim;
185+
} else {
186+
chunks[i] = chunkDim + 1;
187+
}
188+
} else {
189+
// If dimension is smaller than 512, use the full dimension
190+
chunks[i] = (int) shapeDim;
191+
}
192+
}
193+
return chunks;
194+
}
168195
}

‎src/test/java/dev/zarr/zarrjava/ZarrV2Test.java‎

Lines changed: 111 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -407,4 +407,114 @@ public void testMemoryStore() throws ZarrException, IOException {
407407
System.out.println(s);
408408
}
409409

410-
}
410+
@Test
411+
public void testDefaultChunkShape() throws IOException, ZarrException {
412+
// Test with a small array (< 512 elements per dimension)
413+
Array smallArray = Array.create(
414+
new FilesystemStore(TESTOUTPUT).resolve("v2_default_chunks_small"),
415+
Array.metadataBuilder()
416+
.withShape(100, 50)
417+
.withDataType(DataType.UINT8)
418+
.build()
419+
);
420+
Assertions.assertEquals(2, smallArray.metadata().chunks.length);
421+
// Both dimensions < 512, so chunks should equal shape
422+
Assertions.assertEquals(100, smallArray.metadata().chunks[0]);
423+
Assertions.assertEquals(50, smallArray.metadata().chunks[1]);
424+
425+
// Test with a larger array (> 512 elements per dimension)
426+
Array largeArray = Array.create(
427+
new FilesystemStore(TESTOUTPUT).resolve("v2_default_chunks_large"),
428+
Array.metadataBuilder()
429+
.withShape(2000, 1500)
430+
.withDataType(DataType.UINT8)
431+
.build()
432+
);
433+
Assertions.assertEquals(2, largeArray.metadata().chunks.length);
434+
// Chunks should be calculated based on division by 512
435+
Assertions.assertTrue(largeArray.metadata().chunks[0] > 0);
436+
Assertions.assertTrue(largeArray.metadata().chunks[0] < 2000);
437+
Assertions.assertTrue(largeArray.metadata().chunks[1] > 0);
438+
Assertions.assertTrue(largeArray.metadata().chunks[1] < 1500);
439+
440+
// Test with mixed dimensions
441+
Array mixedArray = Array.create(
442+
new FilesystemStore(TESTOUTPUT).resolve("v2_default_chunks_mixed"),
443+
Array.metadataBuilder()
444+
.withShape(1024, 100, 2048)
445+
.withDataType(DataType.UINT8)
446+
.build()
447+
);
448+
Assertions.assertEquals(3, mixedArray.metadata().chunks.length);
449+
// Verify chunks are reasonable
450+
Assertions.assertTrue(mixedArray.metadata().chunks[0] > 0);
451+
Assertions.assertTrue(mixedArray.metadata().chunks[0] <= 1024);
452+
Assertions.assertEquals(100, mixedArray.metadata().chunks[1]); // < 512, should equal shape
453+
Assertions.assertTrue(mixedArray.metadata().chunks[2] > 0);
454+
Assertions.assertTrue(mixedArray.metadata().chunks[2] <= 2048);
455+
}
456+
457+
@Test
458+
public void testDimensionSeparatorAutoDetection() throws IOException, ZarrException {
459+
// Test with SLASH separator
460+
StoreHandle slashStoreHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_separator_detection_slash");
461+
Array slashArray = Array.create(
462+
slashStoreHandle,
463+
Array.metadataBuilder()
464+
.withShape(10, 10)
465+
.withDataType(DataType.UINT8)
466+
.withChunks(5, 5)
467+
.withDimensionSeparator(dev.zarr.zarrjava.core.chunkkeyencoding.Separator.SLASH)
468+
.build()
469+
);
470+
471+
// Write some data to create chunk files
472+
slashArray.write(new long[]{0, 0}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{5, 5}));
473+
474+
// Now open without specifying separator - it should auto-detect SLASH
475+
Array reopenedSlashArray = Array.open(slashStoreHandle);
476+
Assertions.assertEquals(dev.zarr.zarrjava.core.chunkkeyencoding.Separator.SLASH,
477+
reopenedSlashArray.metadata().dimensionSeparator);
478+
479+
// Test with DOT separator
480+
StoreHandle dotStoreHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_separator_detection_dot");
481+
Array dotArray = Array.create(
482+
dotStoreHandle,
483+
Array.metadataBuilder()
484+
.withShape(10, 10)
485+
.withDataType(DataType.UINT8)
486+
.withChunks(5, 5)
487+
.withDimensionSeparator(dev.zarr.zarrjava.core.chunkkeyencoding.Separator.DOT)
488+
.build()
489+
);
490+
491+
// Write some data to create chunk files
492+
dotArray.write(new long[]{0, 0}, ucar.ma2.Array.factory(ucar.ma2.DataType.UBYTE, new int[]{5, 5}));
493+
494+
// Now open without specifying separator - it should auto-detect DOT
495+
Array reopenedDotArray = Array.open(dotStoreHandle);
496+
Assertions.assertEquals(dev.zarr.zarrjava.core.chunkkeyencoding.Separator.DOT,
497+
reopenedDotArray.metadata().dimensionSeparator);
498+
}
499+
500+
@Test
501+
public void testDimensionSeparatorDefaultFallback() throws IOException, ZarrException {
502+
// Test that when no chunks exist, the separator defaults to DOT (as per ArrayMetadata.chunkKeyEncoding())
503+
StoreHandle emptyStoreHandle = new FilesystemStore(TESTOUTPUT).resolve("v2_separator_detection_empty");
504+
Array emptyArray = Array.create(
505+
emptyStoreHandle,
506+
Array.metadataBuilder()
507+
.withShape(10, 10)
508+
.withDataType(DataType.UINT8)
509+
.withChunks(5, 5)
510+
.build()
511+
);
512+
513+
// Open without writing any chunks
514+
Array reopenedEmptyArray = Array.open(emptyStoreHandle);
515+
// Should use default DOT separator from ArrayMetadata.chunkKeyEncoding()
516+
// When dimensionSeparator is null, chunkKeyEncoding() defaults to DOT
517+
Assertions.assertEquals(dev.zarr.zarrjava.core.chunkkeyencoding.Separator.DOT,
518+
reopenedEmptyArray.metadata().chunkKeyEncoding().separator);
519+
}
520+
}

0 commit comments

Comments
 (0)