Merge branch 'main' of github.com:zarr-developers/zarr-python into feature/rectilinear-chunk-grid

jhamman · jhamman · commit 3187229f32ae · 2025-12-20T16:14:39.000-07:00
diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml
@@ -9,10 +9,10 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+      - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@85856786d1ce8acfbcc2f13a5f3fbd6b938f9f41 # v7.1.2
+        uses: astral-sh/setup-uv@1e862dfacbd1d6d858c55d9b792c756523627244 # v7.1.4
 
       - name: Check changelog entries
         run: uv run --no-sync python ci/check_changelog_entries.py
diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
@@ -29,7 +29,7 @@ jobs:
         dependency-set: ["minimal"]
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0  # grab all branches and tags
     # - name: cuda-toolkit
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -28,7 +28,7 @@ jobs:
         dependency-set: ["optional"]
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
     - name: Set HYPOTHESIS_PROFILE based on trigger
       run: |
         if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then
diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml
@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           submodules: true
           fetch-depth: 0
diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml
@@ -11,7 +11,7 @@ jobs:
       fail-fast: false
 
     steps:
-      - uses: actions/checkout@v5
+      - uses: actions/checkout@v6
         with:
           submodules: true
           fetch-depth: 0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -44,7 +44,7 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0  # grab all branches and tags
     - name: Set up Python
@@ -86,7 +86,7 @@ jobs:
           - python-version: "3.11"
             dependency-set: upstream
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0
     - name: Set up Python
@@ -115,7 +115,7 @@ jobs:
     name: doctests
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v5
+    - uses: actions/checkout@v6
       with:
         fetch-depth: 0 # required for hatch version discovery, which is needed for numcodecs.zarr3
     - name: Set up Python
diff --git a/changes/3603.bugfix.md b/changes/3603.bugfix.md
@@ -0,0 +1 @@
+Correct the target bytes number for auto-chunking when auto-sharding.
diff --git a/changes/3605.misc.md b/changes/3605.misc.md
@@ -0,0 +1 @@
+Fix a bug in the test suite that prevented stand-alone example scripts from being tested.
diff --git a/changes/3619.misc.md b/changes/3619.misc.md
@@ -0,0 +1 @@
+Remove upper bounds on `pytest` and `pytest-asyncio` test dependencies.
diff --git a/changes/3623.misc.md b/changes/3623.misc.md
@@ -0,0 +1,5 @@
+This PR contains minor, non-function-altering, changes to use `ZarrFormat` across the repo as opposed to duplicating is with `Literal[2,3]`.
+
+Additionally, it fixes broken linting by using a `Literal[True, False]` type hint for Numpy hypothesis testing, as opposed to `bool`.
+
+Basically improves the typehints and reduces fat-finger error surface area slightly.
diff --git a/docs/user-guide/performance.md b/docs/user-guide/performance.md
@@ -81,7 +81,7 @@ z6 = zarr.create_array(store={}, shape=(10000, 10000, 1000), shards=(1000, 1000,
 print(z6.info)
 ```
 
-`shards` can be `"auto"` as well, in which case the `array.target_shard_size_bytes` setting can be used to control the size of shards (i.e., the size of the shard will be as close to without being bigger than `target_shard_size_bytes`); otherwise, a default is used.
+`shards` can be `"auto"` as well, in which case the `array.target_shard_size_bytes` setting can be used to control the size of shards (i.e., the size of the chunks cumulatively and uncompressed within the shard will be as close to, without being bigger than, `array.target_shard_size_bytes`); otherwise, a default is used.
 
 ### Chunk memory layout
 
diff --git a/examples/custom_dtype/custom_dtype.py b/examples/custom_dtype/custom_dtype.py
@@ -217,7 +217,7 @@ def from_json_scalar(self, data: JSON, *, zarr_format: ZarrFormat) -> ml_dtypes.
 
 # this parametrized function will create arrays in zarr v2 and v3 using our new data type
 @pytest.mark.parametrize("zarr_format", [2, 3])
-def test_custom_dtype(tmp_path: Path, zarr_format: Literal[2, 3]) -> None:
+def test_custom_dtype(tmp_path: Path, zarr_format: ZarrFormat) -> None:
     # create array and write values
     z_w = zarr.create_array(
         store=tmp_path, shape=(4,), dtype="int2", zarr_format=zarr_format, compressors=None
diff --git a/pyproject.toml b/pyproject.toml
@@ -36,7 +36,7 @@ dependencies = [
     'numpy>=1.26',
     'numcodecs>=0.14',
     'google-crc32c>=1.5',
-    'typing_extensions>=4.9',
+    'typing_extensions>=4.12',
     'donfig>=0.8',
 ]
 
@@ -73,8 +73,7 @@ cli = ["typer"]
 # Development extras
 test = [
     "coverage>=7.10",
-    # Pin possibly due to https://github.com/pytest-dev/pytest-cov/issues/693
-    "pytest<8.4",
+    "pytest",
     "pytest-asyncio",
     "pytest-cov",
     "pytest-accept",
@@ -240,7 +239,7 @@ dependencies = [
     'fsspec==2023.10.0',
     's3fs==2023.10.0',
     'universal_pathlib==0.0.22',
-    'typing_extensions==4.9.*',
+    'typing_extensions==4.12.*',
     'donfig==0.8.*',
     'obstore==0.5.*',
     # test deps
diff --git a/src/zarr/_cli/cli.py b/src/zarr/_cli/cli.py
@@ -6,6 +6,7 @@
 
 import zarr
 import zarr.metadata.migrate_v3 as migrate_metadata
+from zarr.core.common import ZarrFormat
 from zarr.core.sync import sync
 from zarr.storage._common import make_store
 
@@ -23,12 +24,12 @@ def _set_logging_level(*, verbose: bool) -> None:
     zarr.set_format("%(message)s")
 
 
-class ZarrFormat(str, Enum):
+class CLIZarrFormat(str, Enum):
     v2 = "v2"
     v3 = "v3"
 
 
-class ZarrFormatV3(str, Enum):
+class CLIZarrFormatV3(str, Enum):
     """Limit CLI choice to only v3"""
 
     v3 = "v3"
@@ -37,7 +38,7 @@ class ZarrFormatV3(str, Enum):
 @app.command()  # type: ignore[misc]
 def migrate(
     zarr_format: Annotated[
-        ZarrFormatV3,
+        CLIZarrFormatV3,
         typer.Argument(
             help="Zarr format to migrate to. Currently only 'v3' is supported.",
         ),
@@ -122,7 +123,7 @@ def migrate(
 @app.command()  # type: ignore[misc]
 def remove_metadata(
     zarr_format: Annotated[
-        ZarrFormat,
+        CLIZarrFormat,
         typer.Argument(help="Which format's metadata to remove - v2 or v3."),
     ],
     store: Annotated[
@@ -160,7 +161,7 @@ def remove_metadata(
     sync(
         migrate_metadata.remove_metadata(
             store=input_zarr_store,
-            zarr_format=cast(Literal[2, 3], int(zarr_format[1:])),
+            zarr_format=cast(ZarrFormat, int(zarr_format[1:])),
             force=force,
             dry_run=dry_run,
         )
diff --git a/src/zarr/core/chunk_grids.py b/src/zarr/core/chunk_grids.py
@@ -236,6 +236,8 @@ def _guess_chunks(
     tuple[int, ...]
 
     """
+    if min_bytes >= max_bytes:
+        raise ValueError(f"Cannot have more min_bytes ({min_bytes}) than max_bytes ({max_bytes})")
     if isinstance(shape, int):
         shape = (shape,)
 
@@ -1177,7 +1179,7 @@ def _auto_partition(
     else:
         if chunk_shape == "auto":
             # aim for a 1MiB chunk
-            _chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1024)
+            _chunks_out = _guess_chunks(array_shape, item_size, max_bytes=1048576)
         else:
             _chunks_out = chunk_shape
 
diff --git a/src/zarr/metadata/migrate_v3.py b/src/zarr/metadata/migrate_v3.py
@@ -1,6 +1,6 @@
 import asyncio
 import logging
-from typing import Literal, cast
+from typing import cast
 
 import numcodecs.abc
 
@@ -140,7 +140,7 @@ async def remove_metadata(
             continue
 
         if force or await _metadata_exists(
-            cast(Literal[2, 3], alternative_metadata), store_path / parent_path
+            cast(ZarrFormat, alternative_metadata), store_path / parent_path
         ):
             _logger.info("Deleting metadata at %s", store_path / file_path)
             if not dry_run:
diff --git a/src/zarr/testing/strategies.py b/src/zarr/testing/strategies.py
@@ -30,6 +30,8 @@
 from zarr.storage._utils import normalize_path
 from zarr.types import AnyArray
 
+TrueOrFalse = Literal[True, False]
+
 # Copied from Xarray
 _attr_keys = st.text(st.characters(), min_size=1)
 _attr_values = st.recursive(
@@ -136,7 +138,7 @@ def array_metadata(
     draw: st.DrawFn,
     *,
     array_shapes: Callable[..., st.SearchStrategy[tuple[int, ...]]] = npst.array_shapes,
-    zarr_formats: st.SearchStrategy[Literal[2, 3]] = zarr_formats,
+    zarr_formats: st.SearchStrategy[ZarrFormat] = zarr_formats,
     attributes: SearchStrategy[Mapping[str, JSON] | None] = attrs,
 ) -> ArrayV2Metadata | ArrayV3Metadata:
     zarr_format = draw(zarr_formats)
@@ -472,8 +474,8 @@ def basic_indices(
     shape: tuple[int, ...],
     min_dims: int = 0,
     max_dims: int | None = None,
-    allow_newaxis: bool = False,
-    allow_ellipsis: bool = True,
+    allow_newaxis: TrueOrFalse = False,
+    allow_ellipsis: TrueOrFalse = True,
 ) -> Any:
     """Basic indices without unsupported negative slices."""
     strategy = npst.basic_indices(
@@ -486,7 +488,7 @@ def basic_indices(
         lambda idxr: (
             not (
                 is_negative_slice(idxr)
-                or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr))  # type: ignore[redundant-expr]
+                or (isinstance(idxr, tuple) and any(is_negative_slice(idx) for idx in idxr))
             )
         )
     )
diff --git a/tests/package_with_entrypoint/__init__.py b/tests/package_with_entrypoint/__init__.py
@@ -84,7 +84,7 @@ class TestDataType(Bool):
     _zarr_v3_name: ClassVar[Literal["test"]] = "test"  # type: ignore[assignment]
 
     @classmethod
-    def from_json(cls, data: DTypeJSON, *, zarr_format: Literal[2, 3]) -> Self:
+    def from_json(cls, data: DTypeJSON, *, zarr_format: ZarrFormat) -> Self:
         if zarr_format == 2 and data == {"name": cls._zarr_v3_name, "object_codec_id": None}:
             return cls()
         if zarr_format == 3 and data == cls._zarr_v3_name:
diff --git a/tests/test_array.py b/tests/test_array.py
@@ -1019,6 +1019,24 @@ def test_auto_partition_auto_shards(
     assert auto_shards == expected_shards
 
 
+def test_auto_partition_auto_shards_with_auto_chunks_should_be_close_to_1MiB() -> None:
+    """
+    Test that automatically picking a shard size and a chunk size gives roughly 1MiB chunks.
+    """
+    with pytest.warns(
+        ZarrUserWarning,
+        match="Automatic shard shape inference is experimental and may change without notice.",
+    ):
+        with zarr.config.set({"array.target_shard_size_bytes": 10_000_000}):
+            _, chunk_shape = _auto_partition(
+                array_shape=(10_000_000,),
+                chunk_shape="auto",
+                shard_shape="auto",
+                item_size=1,
+            )
+    assert chunk_shape == (625000,)
+
+
 def test_chunks_and_shards() -> None:
     store = StorePath(MemoryStore())
     shape = (100, 100)
diff --git a/tests/test_examples.py b/tests/test_examples.py
@@ -11,7 +11,7 @@
 from packaging.requirements import Requirement
 
 examples_dir = "examples"
-script_paths = Path(examples_dir).glob("*.py")
+script_paths = tuple(Path(examples_dir).rglob("*.py"))
 
 PEP_723_REGEX: Final = r"(?m)^# /// (?P<type>[a-zA-Z0-9-]+)$\s(?P<content>(^#(| .*)$\s)+)^# ///$"
 
@@ -62,6 +62,13 @@ def resave_script(source_path: Path, dest_path: Path) -> None:
     dest_path.write_text(dest_text)
 
 
+def test_script_paths() -> None:
+    """
+    Test that our test fixture is working properly and collecting script paths.
+    """
+    assert len(script_paths) > 0
+
+
 @pytest.mark.skipif(
     sys.platform in ("win32",), reason="This test fails due for unknown reasons on Windows in CI."
 )

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Correct the target bytes number for auto-chunking when auto-sharding.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Fix a bug in the test suite that prevented stand-alone example scripts from being tested.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Remove upper bounds on `pytest` and `pytest-asyncio` test dependencies.