Skip to content

Commit 1cd9d45

Browse files
d-v-bclaude
andcommitted
Store array_shape on ChunkGrid instead of passing it per-method
Move array_shape from a per-method parameter on ChunkGrid to a stored attribute, validated once at construction time. This simplifies call sites (especially in indexing.py) by eliminating redundant array_shape arguments throughout the codebase. Key changes: - Add abstract `array_shape` property to ChunkGrid ABC - Add `_array_shape` field and `array_shape` property to RegularChunkGrid - Remove `array_shape` parameter from all ChunkGrid abstract methods - Update `from_dict` to accept `array_shape` keyword argument - Update `update_shape` to return a new instance with updated array_shape - Refactor indexing.py to use chunk_grid methods directly instead of raw arithmetic on chunk_shape - Update all construction sites to pass array_shape Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
1 parent 7f758b3 commit 1cd9d45

12 files changed

Lines changed: 331 additions & 270 deletions

File tree

src/zarr/codecs/sharding.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -415,7 +415,7 @@ async def _decode_single(
415415
indexer = BasicIndexer(
416416
tuple(slice(0, s) for s in shard_shape),
417417
shape=shard_shape,
418-
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape),
418+
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape, array_shape=shard_shape),
419419
)
420420

421421
# setup output array
@@ -461,7 +461,7 @@ async def _decode_partial_single(
461461
indexer = get_indexer(
462462
selection,
463463
shape=shard_shape,
464-
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape),
464+
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape, array_shape=shard_shape),
465465
)
466466

467467
# setup output array
@@ -536,7 +536,7 @@ async def _encode_single(
536536
BasicIndexer(
537537
tuple(slice(0, s) for s in shard_shape),
538538
shape=shard_shape,
539-
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape),
539+
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape, array_shape=shard_shape),
540540
)
541541
)
542542

@@ -585,7 +585,9 @@ async def _encode_partial_single(
585585

586586
indexer = list(
587587
get_indexer(
588-
selection, shape=shard_shape, chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape)
588+
selection,
589+
shape=shard_shape,
590+
chunk_grid=RegularChunkGrid(chunk_shape=chunk_shape, array_shape=shard_shape),
589591
)
590592
)
591593

src/zarr/core/array.py

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -774,7 +774,7 @@ def _create_metadata_v3(
774774
else:
775775
fill_value_parsed = fill_value
776776

777-
chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape)
777+
chunk_grid_parsed = RegularChunkGrid(chunk_shape=chunk_shape, array_shape=shape)
778778
return ArrayV3Metadata(
779779
shape=shape,
780780
data_type=dtype,
@@ -4694,7 +4694,9 @@ async def init_array(
46944694
sharding_codec.validate(
46954695
shape=chunk_shape_parsed,
46964696
dtype=zdtype,
4697-
chunk_grid=RegularChunkGrid(chunk_shape=shard_shape_parsed),
4697+
chunk_grid=RegularChunkGrid(
4698+
chunk_shape=shard_shape_parsed, array_shape=chunk_shape_parsed
4699+
),
46984700
)
46994701
codecs_out = (sharding_codec,)
47004702
chunks_out = shard_shape_parsed
@@ -5995,8 +5997,8 @@ async def _resize(
59955997

59965998
if delete_outside_chunks and not only_growing:
59975999
# Remove all chunks outside of the new shape
5998-
old_chunk_coords = set(array.metadata.chunk_grid.all_chunk_coords(array.metadata.shape))
5999-
new_chunk_coords = set(array.metadata.chunk_grid.all_chunk_coords(new_shape))
6000+
old_chunk_coords = set(array.metadata.chunk_grid.all_chunk_coords())
6001+
new_chunk_coords = set(new_metadata.chunk_grid.all_chunk_coords())
60006002

60016003
async def _delete_key(key: str) -> None:
60026004
await (array.store_path / key).delete()

src/zarr/core/chunk_grids/__init__.py

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@
1818

1919
def parse_chunk_grid(
2020
data: dict[str, JSON] | ChunkGrid | NamedConfig[str, Any],
21+
*,
22+
array_shape: tuple[int, ...],
2123
) -> ChunkGrid:
2224
"""Parse a chunk grid from a dictionary, returning existing ChunkGrid instances as-is.
2325
@@ -28,6 +30,8 @@ def parse_chunk_grid(
2830
data : dict[str, JSON] | ChunkGrid | NamedConfig[str, Any]
2931
Either a ChunkGrid instance (returned as-is) or a dictionary with
3032
'name' and 'configuration' keys.
33+
array_shape : tuple[int, ...]
34+
The shape of the array this chunk grid is bound to.
3135
3236
Returns
3337
-------
@@ -46,7 +50,7 @@ def parse_chunk_grid(
4650
chunk_grid_cls = get_chunk_grid_class(name_parsed)
4751
except KeyError as e:
4852
raise ValueError(f"Unknown chunk grid. Got {name_parsed}.") from e
49-
return chunk_grid_cls.from_dict(data) # type: ignore[arg-type]
53+
return chunk_grid_cls.from_dict(data, array_shape=array_shape) # type: ignore[arg-type, call-arg]
5054

5155

5256
__all__ = [

src/zarr/core/chunk_grids/common.py

Lines changed: 31 additions & 113 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,8 @@
33
import math
44
import numbers
55
import warnings
6-
from abc import abstractmethod
7-
from dataclasses import dataclass
8-
from typing import TYPE_CHECKING, Any, Literal
6+
from abc import ABC, abstractmethod
7+
from typing import TYPE_CHECKING, Any, Literal, Self
98

109
import numpy as np
1110
import numpy.typing as npt
@@ -16,144 +15,63 @@
1615

1716
if TYPE_CHECKING:
1817
from collections.abc import Iterator
19-
from typing import Self
2018

2119
from zarr.core.array import ShardsLike
2220
from zarr.core.common import JSON
2321

2422

25-
@dataclass(frozen=True)
26-
class ChunkGrid(Metadata):
23+
class ChunkGrid(ABC, Metadata):
24+
@property
25+
@abstractmethod
26+
def array_shape(self) -> tuple[int, ...]:
27+
"""The shape of the array this chunk grid is bound to."""
28+
...
29+
2730
@abstractmethod
2831
def to_dict(self) -> dict[str, JSON]: ...
2932

3033
@abstractmethod
3134
def update_shape(self, new_shape: tuple[int, ...]) -> Self:
32-
pass
35+
"""Return a new ChunkGrid with the given array_shape."""
36+
...
3337

3438
@abstractmethod
35-
def all_chunk_coords(self, array_shape: tuple[int, ...]) -> Iterator[tuple[int, ...]]:
36-
pass
39+
def all_chunk_coords(self) -> Iterator[tuple[int, ...]]: ...
3740

3841
@abstractmethod
39-
def get_nchunks(self, array_shape: tuple[int, ...]) -> int:
40-
pass
42+
def get_nchunks(self) -> int: ...
4143

4244
@abstractmethod
43-
def get_chunk_shape(
44-
self, array_shape: tuple[int, ...], chunk_coord: tuple[int, ...]
45-
) -> tuple[int, ...]:
46-
"""
47-
Get the shape of a specific chunk.
48-
49-
Parameters
50-
----------
51-
array_shape : tuple[int, ...]
52-
Shape of the full array.
53-
chunk_coord : tuple[int, ...]
54-
Coordinates of the chunk in the chunk grid.
55-
56-
Returns
57-
-------
58-
tuple[int, ...]
59-
Shape of the chunk at the given coordinates.
60-
"""
45+
def get_chunk_shape(self, chunk_coord: tuple[int, ...]) -> tuple[int, ...]:
46+
"""Get the shape of a specific chunk."""
47+
...
6148

6249
@abstractmethod
63-
def get_chunk_start(
64-
self, array_shape: tuple[int, ...], chunk_coord: tuple[int, ...]
65-
) -> tuple[int, ...]:
66-
"""
67-
Get the starting position of a chunk in the array.
68-
69-
Parameters
70-
----------
71-
array_shape : tuple[int, ...]
72-
Shape of the full array.
73-
chunk_coord : tuple[int, ...]
74-
Coordinates of the chunk in the chunk grid.
75-
76-
Returns
77-
-------
78-
tuple[int, ...]
79-
Starting position (offset) of the chunk in the array.
80-
"""
50+
def get_chunk_start(self, chunk_coord: tuple[int, ...]) -> tuple[int, ...]:
51+
"""Get the starting position of a chunk in the array."""
52+
...
8153

8254
@abstractmethod
83-
def array_index_to_chunk_coord(
84-
self, array_shape: tuple[int, ...], array_index: tuple[int, ...]
85-
) -> tuple[int, ...]:
86-
"""
87-
Map an array index to the chunk coordinates that contain it.
88-
89-
Parameters
90-
----------
91-
array_shape : tuple[int, ...]
92-
Shape of the full array.
93-
array_index : tuple[int, ...]
94-
Index in the array.
95-
96-
Returns
97-
-------
98-
tuple[int, ...]
99-
Coordinates of the chunk containing the array index.
100-
"""
55+
def array_index_to_chunk_coord(self, array_index: tuple[int, ...]) -> tuple[int, ...]:
56+
"""Map an array index to the chunk coordinates that contain it."""
57+
...
10158

10259
@abstractmethod
10360
def array_indices_to_chunk_dim(
104-
self, array_shape: tuple[int, ...], dim: int, indices: npt.NDArray[np.intp]
61+
self, dim: int, indices: npt.NDArray[np.intp]
10562
) -> npt.NDArray[np.intp]:
106-
"""
107-
Map an array of indices along one dimension to chunk coordinates (vectorized).
108-
109-
Parameters
110-
----------
111-
array_shape : tuple[int, ...]
112-
Shape of the full array.
113-
dim : int
114-
Dimension index.
115-
indices : np.ndarray
116-
Array of indices along the given dimension.
117-
118-
Returns
119-
-------
120-
np.ndarray
121-
Array of chunk coordinates, same shape as indices.
122-
"""
63+
"""Map an array of indices along one dimension to chunk coordinates (vectorized)."""
64+
...
12365

12466
@abstractmethod
125-
def chunks_per_dim(self, array_shape: tuple[int, ...], dim: int) -> int:
126-
"""
127-
Get the number of chunks along a specific dimension.
128-
129-
Parameters
130-
----------
131-
array_shape : tuple[int, ...]
132-
Shape of the full array.
133-
dim : int
134-
Dimension index.
135-
136-
Returns
137-
-------
138-
int
139-
Number of chunks along the dimension.
140-
"""
67+
def chunks_per_dim(self, dim: int) -> int:
68+
"""Get the number of chunks along a specific dimension."""
69+
...
14170

14271
@abstractmethod
143-
def get_chunk_grid_shape(self, array_shape: tuple[int, ...]) -> tuple[int, ...]:
144-
"""
145-
Get the shape of the chunk grid (number of chunks along each dimension).
146-
147-
Parameters
148-
----------
149-
array_shape : tuple[int, ...]
150-
Shape of the full array.
151-
152-
Returns
153-
-------
154-
tuple[int, ...]
155-
Number of chunks along each dimension.
156-
"""
72+
def get_chunk_grid_shape(self) -> tuple[int, ...]:
73+
"""Get the shape of the chunk grid (number of chunks along each dimension)."""
74+
...
15775

15876

15977
def _guess_chunks(

0 commit comments

Comments
 (0)