Skip to content

Commit fccf372

Browse files
maxrjonesdcherian
andauthored
fix: validate that dask-style chunks have regular shapes (#3779)
* fix: validate that dask-style chunks have regular shapes * Apply suggestion from @dcherian Co-authored-by: Deepak Cherian <dcherian@users.noreply.github.com> * Fix shortcircuit option --------- Co-authored-by: Deepak Cherian <dcherian@users.noreply.github.com>
1 parent 1bfa53f commit fccf372

File tree

2 files changed

+17
-5
lines changed

2 files changed

+17
-5
lines changed

src/zarr/core/chunk_grids.py

Lines changed: 8 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -126,11 +126,14 @@ def normalize_chunks(chunks: Any, shape: tuple[int, ...], typesize: int) -> tupl
126126
chunks = tuple(int(chunks) for _ in shape)
127127

128128
# handle dask-style chunks (iterable of iterables)
129-
if all(isinstance(c, (tuple | list)) for c in chunks):
130-
# take first chunk size for each dimension
131-
chunks = tuple(
132-
c[0] for c in chunks
133-
) # TODO: check/error/warn for irregular chunks (e.g. if c[0] != c[1:-1])
129+
if all(isinstance(c, (tuple, list)) for c in chunks):
130+
for i, c in enumerate(chunks):
131+
if any(x != y for x, y in itertools.pairwise(c[:-1])) or (len(c) > 1 and c[-1] > c[0]):
132+
raise ValueError(
133+
f"Irregular chunk sizes in dimension {i}: {tuple(c)}. "
134+
"Only uniform chunks (with an optional smaller final chunk) are supported."
135+
)
136+
chunks = tuple(c[0] for c in chunks)
134137

135138
# handle bad dimensionality
136139
if len(chunks) > len(shape):

tests/test_chunk_grids.py

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@ def test_guess_chunks(shape: tuple[int, ...], itemsize: int) -> None:
3535
((30, None, None), (100, 20, 10), 1, (30, 20, 10)),
3636
((30, 20, None), (100, 20, 10), 1, (30, 20, 10)),
3737
((30, 20, 10), (100, 20, 10), 1, (30, 20, 10)),
38+
# dask-style chunks (uniform with optional smaller final chunk)
39+
(((100, 100, 100), (50, 50)), (300, 100), 1, (100, 50)),
40+
(((100, 100, 50),), (250,), 1, (100,)),
41+
(((100,),), (100,), 1, (100,)),
3842
# auto chunking
3943
(None, (100,), 1, (100,)),
4044
(-1, (100,), 1, (100,)),
@@ -52,3 +56,8 @@ def test_normalize_chunks_errors() -> None:
5256
normalize_chunks("foo", (100,), 1)
5357
with pytest.raises(ValueError):
5458
normalize_chunks((100, 10), (100,), 1)
59+
# dask-style irregular chunks should raise
60+
with pytest.raises(ValueError, match="Irregular chunk sizes"):
61+
normalize_chunks(((10, 20, 30),), (60,), 1)
62+
with pytest.raises(ValueError, match="Irregular chunk sizes"):
63+
normalize_chunks(((100, 100), (10, 20)), (200, 30), 1)

0 commit comments

Comments
 (0)