Merge branch 'main' into string-arguments-for-codecs

Author: brokkoli71

.pre-commit-config.yaml

@@ -38,6 +38,7 @@ repos:
           # Tests
           - pytest
           - hypothesis
+          - s3fs
   - repo: https://github.com/scientific-python/cookie
     rev: 2025.01.22
     hooks:

changes/2862.bugfix.rst

@@ -0,0 +1 @@
+Fix a bug that prevented the number of initialized chunks being counted properly.

docs/user-guide/groups.rst

@@ -140,7 +140,7 @@ property. E.g.::
    No. bytes          : 8000000 (7.6M)
    No. bytes stored   : 1614
    Storage ratio      : 4956.6
-   Chunks Initialized : 0
+   Chunks Initialized : 10
    >>> baz.info
    Type               : Array
    Zarr format        : 3

pyproject.toml

@@ -358,7 +358,11 @@ module = [
     "tests.package_with_entrypoint.*",
     "zarr.testing.stateful",
     "tests.test_codecs.test_transpose",
-    "tests.test_config"
+    "tests.test_config",
+    "tests.test_store.test_zip",
+    "tests.test_store.test_local",
+    "tests.test_store.test_fsspec",
+    "tests.test_store.test_memory",
 ]
 strict = false
 
@@ -368,7 +372,11 @@ strict = false
 module = [
     "tests.test_codecs.test_codecs",
     "tests.test_metadata.*",
-    "tests.test_store.*",
+    "tests.test_store.test_core",
+    "tests.test_store.test_logging",
+    "tests.test_store.test_object",
+    "tests.test_store.test_stateful",
+    "tests.test_store.test_wrapper",
     "tests.test_group",
     "tests.test_indexing",
     "tests.test_properties",

src/zarr/core/array.py

@@ -117,6 +117,7 @@
     get_pipeline_class,
 )
 from zarr.storage._common import StorePath, ensure_no_existing_node, make_store_path
+from zarr.storage._utils import _relativize_path
 
 if TYPE_CHECKING:
     from collections.abc import Iterator, Sequence
@@ -3737,7 +3738,12 @@ async def chunks_initialized(
     store_contents = [
         x async for x in array.store_path.store.list_prefix(prefix=array.store_path.path)
     ]
-    return tuple(chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents)
+    store_contents_relative = [
+        _relativize_path(path=key, prefix=array.store_path.path) for key in store_contents
+    ]
+    return tuple(
+        chunk_key for chunk_key in array._iter_chunk_keys() if chunk_key in store_contents_relative
+    )
 
 
 def _build_parents(

src/zarr/storage/_utils.py

@@ -74,11 +74,62 @@ def _join_paths(paths: Iterable[str]) -> str:
     """
     Filter out instances of '' and join the remaining strings with '/'.
 
-    Because the root node of a zarr hierarchy is represented by an empty string,
+    Parameters
+    ----------
+    paths : Iterable[str]
+
+    Returns
+    -------
+    str
+
+    Examples
+    --------
+    >>> _join_paths(["", "a", "b"])
+    'a/b'
+    >>> _join_paths(["a", "b", "c"])
+    'a/b/c'
     """
     return "/".join(filter(lambda v: v != "", paths))
 
 
+def _relativize_path(*, path: str, prefix: str) -> str:
+    """
+    Make a "/"-delimited path relative to some prefix. If the prefix is '', then the path is
+    returned as-is. Otherwise, the prefix is removed from the path as well as the separator
+    string "/".
+
+    If ``prefix`` is not the empty string and ``path`` does not start with ``prefix``
+    followed by a "/" character, then an error is raised.
+
+    This function assumes that the prefix does not end with "/".
+
+    Parameters
+    ----------
+    path : str
+        The path to make relative to the prefix.
+    prefix : str
+        The prefix to make the path relative to.
+
+    Returns
+    -------
+    str
+
+    Examples
+    --------
+    >>> _relativize_path(path="", prefix="a/b")
+    'a/b'
+    >>> _relativize_path(path="a/b", prefix="a/b/c")
+    'c'
+    """
+    if prefix == "":
+        return path
+    else:
+        _prefix = prefix + "/"
+        if not path.startswith(_prefix):
+            raise ValueError(f"The first component of {path} does not start with {prefix}.")
+        return path.removeprefix(f"{prefix}/")
+
+
 def _normalize_paths(paths: Iterable[str]) -> tuple[str, ...]:
     """
     Normalize the input paths according to the normalization scheme used for zarr node paths.

src/zarr/testing/store.py

@@ -58,7 +58,7 @@ async def get(self, store: S, key: str) -> Buffer:
 
     @abstractmethod
     @pytest.fixture
-    def store_kwargs(self) -> dict[str, Any]:
+    def store_kwargs(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
         """Kwargs for instantiating a store"""
         ...
 

src/zarr/testing/utils.py

@@ -1,7 +1,6 @@
 from __future__ import annotations
 
-from collections.abc import Callable, Coroutine
-from typing import TYPE_CHECKING, Any, TypeVar, cast
+from typing import TYPE_CHECKING, TypeVar, cast
 
 import pytest
 
@@ -38,13 +37,13 @@ def has_cupy() -> bool:
         return False
 
 
-T_Callable = TypeVar("T_Callable", bound=Callable[..., Coroutine[Any, Any, None] | None])
+T = TypeVar("T")
 
 
 # Decorator for GPU tests
-def gpu_test(func: T_Callable) -> T_Callable:
+def gpu_test(func: T) -> T:
     return cast(
-        "T_Callable",
+        "T",
         pytest.mark.gpu(
             pytest.mark.skipif(not has_cupy(), reason="CuPy not installed or no GPU available")(
                 func

tests/test_array.py

@@ -388,12 +388,13 @@ async def test_nchunks_initialized(test_cls: type[Array] | type[AsyncArray[Any]]
         assert observed == expected
 
 
-async def test_chunks_initialized() -> None:
+@pytest.mark.parametrize("path", ["", "foo"])
+async def test_chunks_initialized(path: str) -> None:
     """
     Test that chunks_initialized accurately returns the keys of stored chunks.
     """
     store = MemoryStore()
-    arr = zarr.create_array(store, shape=(100,), chunks=(10,), dtype="i4")
+    arr = zarr.create_array(store, name=path, shape=(100,), chunks=(10,), dtype="i4")
 
     chunks_accumulated = tuple(
         accumulate(tuple(tuple(v.split(" ")) for v in arr._iter_chunk_keys()))

tests/test_store/test_core.py

@@ -8,7 +8,13 @@
 from zarr.core.common import AccessModeLiteral, ZarrFormat
 from zarr.storage import FsspecStore, LocalStore, MemoryStore, StoreLike, StorePath
 from zarr.storage._common import contains_array, contains_group, make_store_path
-from zarr.storage._utils import _join_paths, _normalize_path_keys, _normalize_paths, normalize_path
+from zarr.storage._utils import (
+    _join_paths,
+    _normalize_path_keys,
+    _normalize_paths,
+    _relativize_path,
+    normalize_path,
+)
 
 
 @pytest.mark.parametrize("path", ["foo", "foo/bar"])
@@ -221,3 +227,27 @@ def test_normalize_path_keys():
     """
     data = {"a": 10, "//b": 10}
     assert _normalize_path_keys(data) == {normalize_path(k): v for k, v in data.items()}
+
+
+@pytest.mark.parametrize(
+    ("path", "prefix", "expected"),
+    [
+        ("a", "", "a"),
+        ("a/b/c", "a/b", "c"),
+        ("a/b/c", "a", "b/c"),
+    ],
+)
+def test_relativize_path_valid(path: str, prefix: str, expected: str) -> None:
+    """
+    Test the normal behavior of the _relativize_path function. Prefixes should be removed from the
+    path argument.
+    """
+    assert _relativize_path(path=path, prefix=prefix) == expected
+
+
+def test_relativize_path_invalid() -> None:
+    path = "a/b/c"
+    prefix = "b"
+    msg = f"The first component of {path} does not start with {prefix}."
+    with pytest.raises(ValueError, match=msg):
+        _relativize_path(path="a/b/c", prefix="b")