Skip to content

Commit f7086e6

Browse files
brokkoli71brokkoli71
committed
add resizeMetadataOnly argument for resize
1 parent f9d4fd5 commit f7086e6

5 files changed

Lines changed: 298 additions & 4 deletions

File tree

‎src/main/java/dev/zarr/zarrjava/core/Array.java‎

Lines changed: 98 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -174,6 +174,104 @@ public ucar.ma2.Array readChunk(long[] chunkCoords) throws ZarrException {
174174
return codecPipeline.decode(chunkBytes);
175175
}
176176

177+
/**
178+
* Deletes chunks that are completely outside the new shape and trims boundary chunks.
179+
*
180+
* @param newShape the new shape of the array
181+
*/
182+
protected void cleanupChunksForResize(long[] newShape) {
183+
ArrayMetadata metadata = metadata();
184+
final int[] chunkShape = metadata.chunkShape();
185+
final int ndim = metadata.ndim();
186+
final dev.zarr.zarrjava.core.chunkkeyencoding.ChunkKeyEncoding chunkKeyEncoding = metadata.chunkKeyEncoding();
187+
188+
// Calculate max valid chunk coordinates for the new shape
189+
long[] newMaxChunkCoords = new long[ndim];
190+
for (int i = 0; i < ndim; i++) {
191+
newMaxChunkCoords[i] = (newShape[i] + chunkShape[i] - 1) / chunkShape[i];
192+
}
193+
194+
// Iterate over all possible chunk coordinates in the old shape
195+
long[][] allOldChunkCoords = IndexingUtils.computeChunkCoords(metadata.shape, chunkShape);
196+
197+
for (long[] chunkCoords : allOldChunkCoords) {
198+
boolean isOutsideBounds = false;
199+
boolean isOnBoundary = false;
200+
201+
for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
202+
if (chunkCoords[dimIdx] >= newMaxChunkCoords[dimIdx]) {
203+
isOutsideBounds = true;
204+
break;
205+
}
206+
// Check if this chunk is on the boundary (partially outside new shape)
207+
long chunkEnd = (chunkCoords[dimIdx] + 1) * chunkShape[dimIdx];
208+
if (chunkEnd > newShape[dimIdx]) {
209+
isOnBoundary = true;
210+
}
211+
}
212+
213+
String[] chunkKeys = chunkKeyEncoding.encodeChunkKey(chunkCoords);
214+
StoreHandle chunkHandle = storeHandle.resolve(chunkKeys);
215+
216+
if (isOutsideBounds) {
217+
// Delete chunk that is completely outside
218+
chunkHandle.delete();
219+
} else if (isOnBoundary) {
220+
// Trim boundary chunk - read, clear out-of-bounds data, write back
221+
try {
222+
trimBoundaryChunk(chunkCoords, newShape, chunkShape);
223+
} catch (ZarrException e) {
224+
throw new RuntimeException(e);
225+
}
226+
}
227+
}
228+
}
229+
230+
/**
231+
* Trims a boundary chunk by reading it, clearing the out-of-bounds portion, and writing it back.
232+
*
233+
* @param chunkCoords the coordinates of the chunk to trim
234+
* @param newShape the new shape of the array
235+
* @param chunkShape the shape of the chunks
236+
* @throws ZarrException if reading or writing the chunk fails
237+
*/
238+
protected void trimBoundaryChunk(long[] chunkCoords, long[] newShape, int[] chunkShape) throws ZarrException {
239+
ArrayMetadata metadata = metadata();
240+
final int ndim = metadata.ndim();
241+
242+
// Calculate the valid region within this chunk
243+
int[] validShape = new int[ndim];
244+
boolean needsTrimming = false;
245+
for (int dimIdx = 0; dimIdx < ndim; dimIdx++) {
246+
long chunkStart = chunkCoords[dimIdx] * chunkShape[dimIdx];
247+
long chunkEnd = chunkStart + chunkShape[dimIdx];
248+
if (chunkEnd > newShape[dimIdx]) {
249+
validShape[dimIdx] = (int) (newShape[dimIdx] - chunkStart);
250+
needsTrimming = true;
251+
} else {
252+
validShape[dimIdx] = chunkShape[dimIdx];
253+
}
254+
}
255+
256+
if (!needsTrimming) {
257+
return;
258+
}
259+
260+
// Read the existing chunk
261+
ucar.ma2.Array chunkData = readChunk(chunkCoords);
262+
263+
// Create a new chunk filled with fill value
264+
ucar.ma2.Array newChunkData = metadata.allocateFillValueChunk();
265+
266+
// Copy only the valid region
267+
MultiArrayUtils.copyRegion(
268+
chunkData, new int[ndim], newChunkData, new int[ndim], validShape
269+
);
270+
271+
// Write the trimmed chunk back
272+
writeChunk(chunkCoords, newChunkData);
273+
}
274+
177275

178276
/**
179277
* Writes a ucar.ma2.Array into the Zarr array at the beginning of the Zarr array. The shape of

‎src/main/java/dev/zarr/zarrjava/v2/Array.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -198,26 +198,47 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException
198198
}
199199

200200
/**
201-
* Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or
202-
* deleted. This method returns a new instance of the Zarr array class and the old instance
201+
* Sets a new shape for the Zarr array. Old array data outside the new shape will be deleted.
202+
* If data deletion is not desired, use {@link #resize(long[], boolean)} with
203+
* `resizeMetadataOnly` set to true.
204+
* This method returns a new instance of the Zarr array class and the old instance
203205
* becomes invalid.
204206
*
205207
* @param newShape the new shape of the Zarr array
206208
* @throws ZarrException if the new metadata is invalid
207209
* @throws IOException throws IOException if the new metadata cannot be serialized
208210
*/
209211
public Array resize(long[] newShape) throws ZarrException, IOException {
212+
return resize(newShape, false);
213+
}
214+
215+
/**
216+
* Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class
217+
* and the old instance becomes invalid.
218+
*
219+
* @param newShape the new shape of the Zarr array
220+
* @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new
221+
* bounds are deleted and boundary chunks are trimmed
222+
* @throws ZarrException if the new metadata is invalid
223+
* @throws IOException throws IOException if the new metadata cannot be serialized
224+
*/
225+
public Array resize(long[] newShape, boolean resizeMetadataOnly) throws ZarrException, IOException {
210226
if (newShape.length != metadata.ndim()) {
211227
throw new IllegalArgumentException(
212228
"'newShape' needs to have rank '" + metadata.ndim() + "'.");
213229
}
214230

231+
if (!resizeMetadataOnly) {
232+
cleanupChunksForResize(newShape);
233+
}
234+
215235
ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata)
216236
.withShape(newShape)
217237
.build();
218238
return writeMetadata(newArrayMetadata);
219239
}
220240

241+
221242
/**
222243
* Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This
223244
* method returns a new instance of the Zarr array class and the old instance becomes invalid.

‎src/main/java/dev/zarr/zarrjava/v3/Array.java‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -201,26 +201,47 @@ private Array writeMetadata(ArrayMetadata newArrayMetadata) throws ZarrException
201201
}
202202

203203
/**
204-
* Sets a new shape for the Zarr array. It only changes the metadata, no array data is modified or
205-
* deleted. This method returns a new instance of the Zarr array class and the old instance
204+
* Sets a new shape for the Zarr array. Old array data outside the new shape will be deleted.
205+
* If data deletion is not desired, use {@link #resize(long[], boolean)} with
206+
* `resizeMetadataOnly` set to true.
207+
* This method returns a new instance of the Zarr array class and the old instance
206208
* becomes invalid.
207209
*
208210
* @param newShape the new shape of the Zarr array
209211
* @throws ZarrException if the new metadata is invalid
210212
* @throws IOException throws IOException if the new metadata cannot be serialized
211213
*/
212214
public Array resize(long[] newShape) throws ZarrException, IOException {
215+
return resize(newShape, false);
216+
}
217+
218+
/**
219+
* Sets a new shape for the Zarr array. This method returns a new instance of the Zarr array class
220+
* and the old instance becomes invalid.
221+
*
222+
* @param newShape the new shape of the Zarr array
223+
* @param resizeMetadataOnly if true, only the metadata is updated; if false, chunks outside the new
224+
* bounds are deleted and boundary chunks are trimmed
225+
* @throws ZarrException if the new metadata is invalid
226+
* @throws IOException throws IOException if the new metadata cannot be serialized
227+
*/
228+
public Array resize(long[] newShape, boolean resizeMetadataOnly) throws ZarrException, IOException {
213229
if (newShape.length != metadata.ndim()) {
214230
throw new IllegalArgumentException(
215231
"'newShape' needs to have rank '" + metadata.ndim() + "'.");
216232
}
217233

234+
if (!resizeMetadataOnly) {
235+
cleanupChunksForResize(newShape);
236+
}
237+
218238
ArrayMetadata newArrayMetadata = ArrayMetadataBuilder.fromArrayMetadata(metadata)
219239
.withShape(newShape)
220240
.build();
221241
return writeMetadata(newArrayMetadata);
222242
}
223243

244+
224245
/**
225246
* Sets the attributes of the Zarr array. It overwrites and removes any existing attributes. This
226247
* method returns a new instance of the Zarr array class and the old instance becomes invalid.

‎src/test/java/dev/zarr/zarrjava/ZarrV2Test.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,83 @@ public void testResizeArrayShrink() throws IOException, ZarrException {
415415
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
416416
}
417417

418+
@Test
419+
public void testResizeArrayShrinkWithChunkCleanup() throws IOException, ZarrException {
420+
int[] testData = new int[10 * 10];
421+
Arrays.setAll(testData, p -> p);
422+
423+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithChunkCleanupV2");
424+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
425+
.withShape(10, 10)
426+
.withDataType(DataType.UINT32)
427+
.withChunks(5, 5)
428+
.withFillValue(99)
429+
.build();
430+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
431+
Array array = Array.create(storeHandle, arrayMetadata);
432+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
433+
434+
// Verify all 4 chunks exist before resize
435+
Assertions.assertTrue(storeHandle.resolve("0.0").exists());
436+
Assertions.assertTrue(storeHandle.resolve("0.1").exists());
437+
Assertions.assertTrue(storeHandle.resolve("1.0").exists());
438+
Assertions.assertTrue(storeHandle.resolve("1.1").exists());
439+
440+
// Resize with chunk cleanup (resizeMetadataOnly=false)
441+
array = array.resize(new long[]{5, 5}, false);
442+
Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
443+
444+
// Verify only chunk (0,0) still exists
445+
Assertions.assertTrue(storeHandle.resolve("0.0").exists());
446+
Assertions.assertFalse(storeHandle.resolve("0.1").exists());
447+
Assertions.assertFalse(storeHandle.resolve("1.0").exists());
448+
Assertions.assertFalse(storeHandle.resolve("1.1").exists());
449+
450+
ucar.ma2.Array data = array.read();
451+
int[] expectedData = new int[5 * 5];
452+
for (int i = 0; i < 5; i++) {
453+
for (int j = 0; j < 5; j++) {
454+
expectedData[i * 5 + j] = testData[i * 10 + j];
455+
}
456+
}
457+
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
458+
}
459+
460+
@Test
461+
public void testResizeArrayShrinkWithBoundaryTrimming() throws IOException, ZarrException {
462+
int[] testData = new int[10 * 10];
463+
Arrays.setAll(testData, p -> p);
464+
465+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithBoundaryTrimmingV2");
466+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
467+
.withShape(10, 10)
468+
.withDataType(DataType.UINT32)
469+
.withChunks(5, 5)
470+
.withFillValue(99)
471+
.build();
472+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
473+
Array array = Array.create(storeHandle, arrayMetadata);
474+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
475+
476+
// Resize to 7x7 (crosses chunk boundary, should trim boundary chunks)
477+
array = array.resize(new long[]{7, 7}, false);
478+
Assertions.assertArrayEquals(new int[]{7, 7}, array.read().getShape());
479+
480+
// Verify chunks (0,0), (0,1), (1,0), (1,1) still exist (boundary trimmed, not deleted)
481+
Assertions.assertTrue(storeHandle.resolve("0.0").exists());
482+
Assertions.assertTrue(storeHandle.resolve("0.1").exists());
483+
Assertions.assertTrue(storeHandle.resolve("1.0").exists());
484+
Assertions.assertTrue(storeHandle.resolve("1.1").exists());
485+
486+
// Now resize to expand again and check that trimmed area has fill value
487+
array = array.resize(new long[]{10, 10}, true);
488+
ucar.ma2.Array data = array.read(new long[]{7, 0}, new int[]{3, 10});
489+
// All values in rows 7-9 should be fill value (99)
490+
int[] expectedFillData = new int[3 * 10];
491+
Arrays.fill(expectedFillData, 99);
492+
Assertions.assertArrayEquals(expectedFillData, (int[]) data.get1DJavaArray(ma2DataType));
493+
}
494+
418495
@Test
419496
public void testGroupAttributes() throws IOException, ZarrException {
420497
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV2");

‎src/test/java/dev/zarr/zarrjava/ZarrV3Test.java‎

Lines changed: 77 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -790,6 +790,83 @@ public void testResizeArrayShrink() throws IOException, ZarrException {
790790
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
791791
}
792792

793+
@Test
794+
public void testResizeArrayShrinkWithChunkCleanup() throws IOException, ZarrException {
795+
int[] testData = new int[10 * 10];
796+
Arrays.setAll(testData, p -> p);
797+
798+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithChunkCleanupV3");
799+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
800+
.withShape(10, 10)
801+
.withDataType(DataType.UINT32)
802+
.withChunkShape(5, 5)
803+
.withFillValue(99)
804+
.build();
805+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
806+
Array array = Array.create(storeHandle, arrayMetadata);
807+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
808+
809+
// Verify all 4 chunks exist before resize (v3 default encoding has "c" prefix)
810+
Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists());
811+
Assertions.assertTrue(storeHandle.resolve("c", "0", "1").exists());
812+
Assertions.assertTrue(storeHandle.resolve("c", "1", "0").exists());
813+
Assertions.assertTrue(storeHandle.resolve("c", "1", "1").exists());
814+
815+
// Resize with chunk cleanup (resizeMetadataOnly=false)
816+
array = array.resize(new long[]{5, 5}, false);
817+
Assertions.assertArrayEquals(new int[]{5, 5}, array.read().getShape());
818+
819+
// Verify only chunk (0,0) still exists
820+
Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists());
821+
Assertions.assertFalse(storeHandle.resolve("c", "0", "1").exists());
822+
Assertions.assertFalse(storeHandle.resolve("c", "1", "0").exists());
823+
Assertions.assertFalse(storeHandle.resolve("c", "1", "1").exists());
824+
825+
ucar.ma2.Array data = array.read();
826+
int[] expectedData = new int[5 * 5];
827+
for (int i = 0; i < 5; i++) {
828+
for (int j = 0; j < 5; j++) {
829+
expectedData[i * 5 + j] = testData[i * 10 + j];
830+
}
831+
}
832+
Assertions.assertArrayEquals(expectedData, (int[]) data.get1DJavaArray(ma2DataType));
833+
}
834+
835+
@Test
836+
public void testResizeArrayShrinkWithBoundaryTrimming() throws IOException, ZarrException {
837+
int[] testData = new int[10 * 10];
838+
Arrays.setAll(testData, p -> p);
839+
840+
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testResizeArrayShrinkWithBoundaryTrimmingV3");
841+
ArrayMetadata arrayMetadata = Array.metadataBuilder()
842+
.withShape(10, 10)
843+
.withDataType(DataType.UINT32)
844+
.withChunkShape(5, 5)
845+
.withFillValue(99)
846+
.build();
847+
ucar.ma2.DataType ma2DataType = arrayMetadata.dataType.getMA2DataType();
848+
Array array = Array.create(storeHandle, arrayMetadata);
849+
array.write(new long[]{0, 0}, ucar.ma2.Array.factory(ma2DataType, new int[]{10, 10}, testData));
850+
851+
// Resize to 7x7 (crosses chunk boundary, should trim boundary chunks)
852+
array = array.resize(new long[]{7, 7}, false);
853+
Assertions.assertArrayEquals(new int[]{7, 7}, array.read().getShape());
854+
855+
// Verify chunks (0,0), (0,1), (1,0), (1,1) still exist (boundary trimmed, not deleted)
856+
Assertions.assertTrue(storeHandle.resolve("c", "0", "0").exists());
857+
Assertions.assertTrue(storeHandle.resolve("c", "0", "1").exists());
858+
Assertions.assertTrue(storeHandle.resolve("c", "1", "0").exists());
859+
Assertions.assertTrue(storeHandle.resolve("c", "1", "1").exists());
860+
861+
// Now resize to expand again and check that trimmed area has fill value
862+
array = array.resize(new long[]{10, 10}, true);
863+
ucar.ma2.Array data = array.read(new long[]{7, 0}, new int[]{3, 10});
864+
// All values in rows 7-9 should be fill value (99)
865+
int[] expectedFillData = new int[3 * 10];
866+
Arrays.fill(expectedFillData, 99);
867+
Assertions.assertArrayEquals(expectedFillData, (int[]) data.get1DJavaArray(ma2DataType));
868+
}
869+
793870
@Test
794871
public void testGroupAttributes() throws IOException, ZarrException {
795872
StoreHandle storeHandle = new FilesystemStore(TESTOUTPUT).resolve("testGroupAttributesV3");

0 commit comments

Comments
 (0)