Skip to content

Commit 913be10

Browse files
committed
feat(storage): add FsspecStore.get_ranges and coalesce_options kwarg
1 parent 3ab711d commit 913be10

2 files changed

Lines changed: 34 additions & 3 deletions

File tree

src/zarr/storage/_fsspec.py

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
import json
44
import warnings
55
from contextlib import suppress
6+
from functools import partial
67
from typing import TYPE_CHECKING, Any
78

89
from packaging.version import parse as parse_version
@@ -14,18 +15,24 @@
1415
Store,
1516
SuffixByteRequest,
1617
)
18+
from zarr.core._coalesce import (
19+
DEFAULT_COALESCE_OPTIONS,
20+
CoalesceOptions,
21+
coalesced_get,
22+
)
1723
from zarr.core.buffer import Buffer
1824
from zarr.errors import ZarrUserWarning
1925
from zarr.storage._utils import _join_paths, normalize_path
2026

2127
if TYPE_CHECKING:
22-
from collections.abc import AsyncIterator, Iterable
28+
from collections.abc import AsyncIterator, Iterable, Sequence
2329

2430
from fsspec import AbstractFileSystem
2531
from fsspec.asyn import AsyncFileSystem
2632
from fsspec.mapping import FSMap
2733

2834
from zarr.core.buffer import BufferPrototype
35+
from zarr.storage._protocols import SupportsGetRanges
2936

3037

3138
ALLOWED_EXCEPTIONS: tuple[type[Exception], ...] = (
@@ -124,11 +131,14 @@ def __init__(
124131
read_only: bool = False,
125132
path: str = "/",
126133
allowed_exceptions: tuple[type[Exception], ...] = ALLOWED_EXCEPTIONS,
134+
*,
135+
coalesce_options: CoalesceOptions = DEFAULT_COALESCE_OPTIONS,
127136
) -> None:
128137
super().__init__(read_only=read_only)
129138
self.fs = fs
130139
self.path = normalize_path(path)
131140
self.allowed_exceptions = allowed_exceptions
141+
self.coalesce_options = coalesce_options
132142

133143
if not self.fs.async_impl:
134144
raise TypeError("Filesystem needs to support async operations.")
@@ -315,6 +325,22 @@ async def get(
315325
else:
316326
return value
317327

328+
async def get_ranges(
329+
self,
330+
key: str,
331+
byte_ranges: Iterable[ByteRequest | None],
332+
*,
333+
prototype: BufferPrototype,
334+
) -> AsyncIterator[Sequence[tuple[int, Buffer | None]]]:
335+
"""Read many byte ranges from ``key``, coalescing nearby ranges and fetching concurrently.
336+
337+
See :class:`zarr.storage._protocols.SupportsGetRanges` for the contract and
338+
:func:`zarr.core._coalesce.coalesced_get` for the full semantics.
339+
"""
340+
fetch = partial(self.get, key, prototype)
341+
async for group in coalesced_get(fetch, byte_ranges, options=self.coalesce_options):
342+
yield group
343+
318344
async def set(
319345
self,
320346
key: str,
@@ -440,3 +466,8 @@ async def getsize(self, key: str) -> int:
440466
else:
441467
# fsspec doesn't have typing. We'll need to assume or verify this is true
442468
return int(size)
469+
470+
471+
# Module-level type assertion: FsspecStore structurally satisfies SupportsGetRanges.
472+
# This line is a no-op at runtime but causes mypy/pyright to complain if the shape drifts.
473+
_: type[SupportsGetRanges] = FsspecStore

tests/test_store/test_fsspec_get_ranges.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -98,13 +98,13 @@ async def test_coalesce_options_wired_through() -> None:
9898

9999
async def test_get_ranges_mixed_range_types(memory_store: FsspecStore) -> None:
100100
"""Covers RangeByteRequest, OffsetByteRequest, SuffixByteRequest, and None in one call."""
101-
from zarr.abc.store import OffsetByteRequest, SuffixByteRequest
101+
from zarr.abc.store import ByteRequest, OffsetByteRequest, SuffixByteRequest
102102

103103
blob = bytes(i % 256 for i in range(512))
104104
await _write(memory_store, "mixed", blob)
105105
proto = default_buffer_prototype()
106106

107-
ranges = [
107+
ranges: list[ByteRequest | None] = [
108108
RangeByteRequest(0, 10),
109109
OffsetByteRequest(500),
110110
SuffixByteRequest(12),

0 commit comments

Comments
 (0)