Merge branch 'main' into ig/shard_order

ilan-gold · web-flow · commit 73352f81b9e1 · 2026-04-01T11:12:40.000+02:00
diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
@@ -72,4 +72,5 @@ jobs:
       uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3  # v5.3.1
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
+        flags: gpu
         verbose: true # optional (default = false)
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -81,6 +81,7 @@ jobs:
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
+        flags: tests
         verbose: true # optional (default = false)
 
     - name: Generate and publish the report
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -71,6 +71,7 @@ jobs:
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
+        flags: tests
         verbose: true # optional (default = false)
 
   test-upstream-and-min-deps:
@@ -110,6 +111,7 @@ jobs:
       uses: codecov/codecov-action@v5
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
+        flags: tests
         verbose: true # optional (default = false)
 
   doctests:
diff --git a/changes/2720.doc.md b/changes/2720.doc.md
@@ -0,0 +1 @@
+Document removal of `zarr.storage.init_group` in v3 migration guide, with replacement using `zarr.open_group`/`zarr.create_group`.
diff --git a/changes/3845.doc.md b/changes/3845.doc.md
@@ -0,0 +1 @@
+Remove result="ansi" from code blocks in the user guide that were causing empty output cells in the rendered documentation.
diff --git a/codecov.yml b/codecov.yml
@@ -8,9 +8,21 @@ coverage:
       default:
         target: auto
         threshold: 0.1
+        flags:
+          - tests
+flags:
+  tests:
+    paths:
+      - src/
+    carryforward: true
+  gpu:
+    paths:
+      - src/
+    carryforward: true
 codecov:
   notify:
-    after_n_builds: 10 # Wait for all 10 reports before updating the status
+    # 6 = test.yml: 3 (optional+ubuntu) + 2 (upstream + min_deps), hypothesis: 1
+    after_n_builds: 6
     wait_for_ci: yes
 comment:
   layout: "diff, files"
diff --git a/docs/user-guide/experimental.md b/docs/user-guide/experimental.md
@@ -55,7 +55,7 @@ such as a remote store for source data and a local store for persistent caching.
 
 The CacheStore provides significant performance improvements for repeated data access:
 
-```python exec="true" session="experimental" source="above" result="ansi"
+```python exec="true" session="experimental" source="above"
 import time
 
 # Benchmark reading with cache
@@ -121,7 +121,7 @@ cache = CacheStore(
 
 **cache_set_data**: Controls whether written data is cached
 
-```python exec="true" session="experimental" source="above" result="ansi"
+```python exec="true" session="experimental" source="above"
 # Cache data when writing (default)
 cache = CacheStore(
     store=source_store,
@@ -141,7 +141,7 @@ cache = CacheStore(
 
 The CacheStore provides statistics to monitor cache performance and state:
 
-```python exec="true" session="experimental" source="above" result="ansi"
+```python exec="true" session="experimental" source="above"
 # Access some data to generate cache activity
 data = zarr_array[0:50, 0:50]  # First access - cache miss
 data = zarr_array[0:50, 0:50]  # Second access - cache hit
@@ -232,7 +232,7 @@ of source and cache stores for your specific use case.
 
 Here's a complete example demonstrating cache effectiveness:
 
-```python exec="true" session="experimental-final" source="above" result="ansi"
+```python exec="true" session="experimental-final" source="above"
 import numpy as np
 import time
 from tempfile import mkdtemp
diff --git a/docs/user-guide/v3_migration.md b/docs/user-guide/v3_migration.md
@@ -114,6 +114,15 @@ The following sections provide details on breaking changes in Zarr-Python 3.
    - Use [`zarr.Group.require_array`][] in place of `zarr.Group.require_dataset`
 3. Disallow "." syntax for getting group members. To get a member of a group named `foo`,
    use `group["foo"]` in place of `group.foo`.
+4. The `zarr.storage.init_group` low-level helper function has been removed. Use
+   [`zarr.open_group`][] or [`zarr.create_group`][] instead:
+
+   ```diff
+   - from zarr.storage import init_group
+   - init_group(store, overwrite=True, path="my/path")
+   + import zarr
+   + zarr.open_group(store, mode="w", path="my/path")
+   ```
 
 ### The Store class
 
diff --git a/pyproject.toml b/pyproject.toml
@@ -155,10 +155,6 @@ hooks.vcs.version-file = "src/zarr/_version.py"
 dependency-groups = ["test"]
 
 [tool.hatch.envs.test.env-vars]
-# Required to test with a pytest plugin; see https://pytest-cov.readthedocs.io/en/latest/plugins.html
-COV_CORE_SOURCE = "src"
-COV_CORE_CONFIG = ".coveragerc"
-COV_CORE_DATAFILE = ".coverage.eager"
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.12", "3.13", "3.14"]
@@ -175,13 +171,23 @@ matrix.deps.dependency-groups = [
 ]
 
 [tool.hatch.envs.test.scripts]
-run-coverage = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy"
-run-coverage-html = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report html"
-run = "run-coverage --no-cov --ignore tests/benchmarks"
+run-coverage = [
+    "coverage run --source=src -m pytest --junitxml=junit.xml -o junit_family=legacy {args:}",
+    "coverage xml",
+]
+run-coverage-html = [
+    "coverage run --source=src -m pytest {args:}",
+    "coverage html",
+]
+run = "pytest --ignore tests/benchmarks"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
-run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
+run-hypothesis = [
+    "coverage run --source=src -m pytest -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful* {args:}",
+    "coverage xml",
+]
 run-benchmark = "pytest --benchmark-enable tests/benchmarks"
+serve-coverage-html = "python -m http.server -d htmlcov 8000"
 list-env = "pip list"
 
 [tool.hatch.envs.gputest]
@@ -195,8 +201,11 @@ features = ["gpu"]
 python = ["3.12", "3.13"]
 
 [tool.hatch.envs.gputest.scripts]
-run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=src --cov-report xml --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks"
-run = "run-coverage --no-cov"
+run-coverage = [
+    "coverage run --source=src -m pytest -m gpu --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks {args:}",
+    "coverage xml",
+]
+run = "pytest -m gpu --ignore tests/benchmarks"
 
 [tool.hatch.envs.upstream]
 template = 'test'
diff --git a/src/zarr/abc/codec.py b/src/zarr/abc/codec.py
@@ -67,20 +67,19 @@ def _check_codecjson_v2(data: object) -> TypeGuard[CodecJSON_V2[str]]:
 
 
 @runtime_checkable
-class SupportsSyncCodec(Protocol):
+class SupportsSyncCodec[CI: CodecInput, CO: CodecOutput](Protocol):
     """Protocol for codecs that support synchronous encode/decode.
 
-    Codecs implementing this protocol provide ``_decode_sync`` and ``_encode_sync``
+    Codecs implementing this protocol provide `_decode_sync` and `_encode_sync`
     methods that perform encoding/decoding without requiring an async event loop.
+
+    The type parameters mirror `BaseCodec`: `CI` is the decoded type and `CO` is
+    the encoded type.
     """
 
-    def _decode_sync(
-        self, chunk_data: NDBuffer | Buffer, chunk_spec: ArraySpec
-    ) -> NDBuffer | Buffer: ...
+    def _decode_sync(self, chunk_data: CO, chunk_spec: ArraySpec) -> CI: ...
 
-    def _encode_sync(
-        self, chunk_data: NDBuffer | Buffer, chunk_spec: ArraySpec
-    ) -> NDBuffer | Buffer | None: ...
+    def _encode_sync(self, chunk_data: CI, chunk_spec: ArraySpec) -> CO | None: ...
 
 
 class BaseCodec[CI: CodecInput, CO: CodecOutput](Metadata):
diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py
@@ -1,6 +1,6 @@
 from __future__ import annotations
 
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from itertools import islice, pairwise
 from typing import TYPE_CHECKING, Any
 from warnings import warn
@@ -14,6 +14,7 @@
     Codec,
     CodecPipeline,
     GetResult,
+    SupportsSyncCodec,
 )
 from zarr.core.common import concurrent_map
 from zarr.core.config import config
@@ -66,6 +67,111 @@ def fill_value_or_default(chunk_spec: ArraySpec) -> Any:
         return fill_value
 
 
+@dataclass(slots=True, kw_only=True)
+class ChunkTransform:
+    """A synchronous codec chain bound to an ArraySpec.
+
+    Provides `encode` and `decode` for pure-compute codec operations
+    (no IO, no threading, no batching).
+
+    All codecs must implement `SupportsSyncCodec`. Construction will
+    raise `TypeError` if any codec does not.
+    """
+
+    codecs: tuple[Codec, ...]
+    array_spec: ArraySpec
+
+    # (sync codec, input_spec) pairs in pipeline order.
+    _aa_codecs: tuple[tuple[SupportsSyncCodec[NDBuffer, NDBuffer], ArraySpec], ...] = field(
+        init=False, repr=False, compare=False
+    )
+    _ab_codec: SupportsSyncCodec[NDBuffer, Buffer] = field(init=False, repr=False, compare=False)
+    _ab_spec: ArraySpec = field(init=False, repr=False, compare=False)
+    _bb_codecs: tuple[SupportsSyncCodec[Buffer, Buffer], ...] = field(
+        init=False, repr=False, compare=False
+    )
+
+    def __post_init__(self) -> None:
+        non_sync = [c for c in self.codecs if not isinstance(c, SupportsSyncCodec)]
+        if non_sync:
+            names = ", ".join(type(c).__name__ for c in non_sync)
+            raise TypeError(
+                f"All codecs must implement SupportsSyncCodec. The following do not: {names}"
+            )
+
+        aa, ab, bb = codecs_from_list(list(self.codecs))
+
+        aa_codecs: list[tuple[SupportsSyncCodec[NDBuffer, NDBuffer], ArraySpec]] = []
+        spec = self.array_spec
+        for aa_codec in aa:
+            assert isinstance(aa_codec, SupportsSyncCodec)
+            aa_codecs.append((aa_codec, spec))
+            spec = aa_codec.resolve_metadata(spec)
+
+        self._aa_codecs = tuple(aa_codecs)
+        assert isinstance(ab, SupportsSyncCodec)
+        self._ab_codec = ab
+        self._ab_spec = spec
+        bb_sync: list[SupportsSyncCodec[Buffer, Buffer]] = []
+        for bb_codec in bb:
+            assert isinstance(bb_codec, SupportsSyncCodec)
+            bb_sync.append(bb_codec)
+        self._bb_codecs = tuple(bb_sync)
+
+    def decode(
+        self,
+        chunk_bytes: Buffer,
+    ) -> NDBuffer:
+        """Decode a single chunk through the full codec chain, synchronously.
+
+        Pure compute -- no IO.
+        """
+        data: Buffer = chunk_bytes
+        for bb_codec in reversed(self._bb_codecs):
+            data = bb_codec._decode_sync(data, self._ab_spec)
+
+        chunk_array: NDBuffer = self._ab_codec._decode_sync(data, self._ab_spec)
+
+        for aa_codec, spec in reversed(self._aa_codecs):
+            chunk_array = aa_codec._decode_sync(chunk_array, spec)
+
+        return chunk_array
+
+    def encode(
+        self,
+        chunk_array: NDBuffer,
+    ) -> Buffer | None:
+        """Encode a single chunk through the full codec chain, synchronously.
+
+        Pure compute -- no IO.
+        """
+        aa_data: NDBuffer = chunk_array
+        for aa_codec, spec in self._aa_codecs:
+            aa_result = aa_codec._encode_sync(aa_data, spec)
+            if aa_result is None:
+                return None
+            aa_data = aa_result
+
+        ab_result = self._ab_codec._encode_sync(aa_data, self._ab_spec)
+        if ab_result is None:
+            return None
+
+        bb_data: Buffer = ab_result
+        for bb_codec in self._bb_codecs:
+            bb_result = bb_codec._encode_sync(bb_data, self._ab_spec)
+            if bb_result is None:
+                return None
+            bb_data = bb_result
+
+        return bb_data
+
+    def compute_encoded_size(self, byte_length: int, array_spec: ArraySpec) -> int:
+        for codec in self.codecs:
+            byte_length = codec.compute_encoded_size(byte_length, array_spec)
+            array_spec = codec.resolve_metadata(array_spec)
+        return byte_length
+
+
 @dataclass(frozen=True)
 class BatchedCodecPipeline(CodecPipeline):
     """Default codec pipeline.
diff --git a/tests/test_sync_codec_pipeline.py b/tests/test_sync_codec_pipeline.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Document removal of `zarr.storage.init_group` in v3 migration guide, with replacement using `zarr.open_group`/`zarr.create_group`.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Remove result="ansi" from code blocks in the user guide that were causing empty output cells in the rendered documentation.`