Merge branch 'main' into chore/publish-metadata-package

d-v-b · web-flow · commit 713e5fa8e7a5 · 2026-05-06T11:53:43.000-04:00
diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml
@@ -17,12 +17,12 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
 
       - name: Install uv
-        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
+        uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
 
       - name: Check changelog entries
         run: uv run --no-sync python ci/check_changelog_entries.py
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -32,7 +32,7 @@ jobs:
       with:
         version: '1.16.5'
     - name: Run the benchmarks
-      uses: CodSpeedHQ/action@1c8ae4843586d3ba879736b7f6b7b0c990757fab # v4.12.1
+      uses: CodSpeedHQ/action@658a901452bb54c799643e060733b7afe9121b8d # v4.14.0
       with:
           mode: walltime
           run: hatch run test.py3.12-minimal:pytest tests/benchmarks --codspeed
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
@@ -22,7 +22,7 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
-      - uses: astral-sh/setup-uv@f0ec1fc3b38f5e7cd731bb6ce540c5af426746bb # v6.1.0
+      - uses: astral-sh/setup-uv@08807647e7069bb48b6ef5acd8ec9567f424441b # v8.1.0
       - run: uv sync --group docs
       - run: uv run mkdocs build
         env:
diff --git a/.github/workflows/downstream.yml b/.github/workflows/downstream.yml
@@ -3,7 +3,7 @@ name: Downstream
 on:
   workflow_dispatch:
   pull_request:
-    types: [labeled]
+    types: [labeled, synchronize, opened, reopened]
 
 permissions:
   contents: read
@@ -15,7 +15,9 @@ concurrency:
 jobs:
   xarray:
     name: Xarray zarr backend tests
-    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-downstream'
+    if: |
+      github.event_name == 'workflow_dispatch'
+      || contains(github.event.pull_request.labels.*.name, 'run-downstream')
     runs-on: ubuntu-latest
     steps:
       - name: Check out zarr-python
@@ -51,14 +53,16 @@ jobs:
       - name: Run xarray zarr backend tests
         working-directory: xarray
         run: |
-          pixi run -e test-py313 -- python -m pytest -x --no-header -q \
-            xarray/tests/test_backends.py -k zarr \
-            xarray/tests/test_backends_api.py -k zarr \
-            xarray/tests/test_backends_datatree.py -k zarr
+          pixi run -e test-py313 -- python -m pytest --no-header -q \
+            xarray/tests/test_backends.py \
+            xarray/tests/test_backends_api.py \
+            xarray/tests/test_backends_datatree.py
 
   numcodecs:
     name: numcodecs zarr3 codec tests
-    if: github.event_name == 'workflow_dispatch' || github.event.label.name == 'run-downstream'
+    if: |
+      github.event_name == 'workflow_dispatch'
+      || contains(github.event.pull_request.labels.*.name, 'run-downstream')
     runs-on: ubuntu-latest
     steps:
       - name: Check out zarr-python
diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
@@ -76,7 +76,7 @@ jobs:
         hatch env run --env "$HATCH_ENV" run-coverage
 
     - name: Upload coverage
-      uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3  # v5.3.1
+      uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2  # v6.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: gpu
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -67,7 +67,7 @@ jobs:
     # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
     - name: Restore cached hypothesis directory
       id: restore-hypothesis-cache
-      uses: actions/cache/restore@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+      uses: actions/cache/restore@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: .hypothesis/
         key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
@@ -87,13 +87,13 @@ jobs:
     - name: Save cached hypothesis directory
       id: save-hypothesis-cache
       if: always() && steps.status.outcome != 'skipped'
-      uses: actions/cache/save@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
+      uses: actions/cache/save@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5.0.5
       with:
         path: .hypothesis/
         key: cache-hypothesis-${{ runner.os }}-${{ github.run_id }}
 
     - name: Upload coverage
-      uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+      uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
diff --git a/.github/workflows/issue-metrics.yml b/.github/workflows/issue-metrics.yml
@@ -33,7 +33,7 @@ jobs:
         echo "last_month=$first_day..$last_day" >> "$GITHUB_ENV"
 
     - name: Run issue-metrics tool
-      uses: github/issue-metrics@67526e7bd8100b870f10b1c120780a8375777b43 # v3.25.5
+      uses: github/issue-metrics@c9e9838147fd355dace335ba787f01b6641a400a # v4.2.2
       env:
         GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         SEARCH_QUERY: 'repo:zarr-developers/zarr-python is:issue created:${{ env.last_month }} -reason:"not planned"'
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -22,4 +22,4 @@ jobs:
       - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
         with:
           persist-credentials: false
-      - uses: j178/prek-action@0bb87d7f00b0c99306c8bcb8b8beba1eb581c037 # v1.1.1
+      - uses: j178/prek-action@cbc2f23eb5539cf20d82d1aabd0d0ecbcc56f4e3 # v2.0.2
diff --git a/.github/workflows/nightly_wheels.yml b/.github/workflows/nightly_wheels.yml
@@ -42,7 +42,7 @@ jobs:
         run: hatch build
 
       - name: Upload nightly wheels
-        uses: scientific-python/upload-nightly-action@5748273c71e2d8d3a61f3a11a16421c8954f9ecf
+        uses: scientific-python/upload-nightly-action@e76cfec8a4611fd02808a801b0ff5a7d7c1b2d99
         with:
           artifacts_path: dist
           anaconda_nightly_upload_token: ${{ secrets.ANACONDA_ORG_UPLOAD_TOKEN }}
diff --git a/.github/workflows/releases.yml b/.github/workflows/releases.yml
@@ -43,7 +43,7 @@ jobs:
           version: '1.16.5'
       - name: Build wheel and sdist
         run: hatch build
-      - uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
+      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: releases
           path: dist
@@ -53,7 +53,7 @@ jobs:
     needs: [build_artifacts]
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: releases
           path: dist
@@ -76,7 +76,7 @@ jobs:
       attestations: write # Required for artifact attestation
       artifact-metadata: write # Required for artifact attestation metadata
     steps:
-      - uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0
+      - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
           name: releases
           path: dist
@@ -85,4 +85,4 @@ jobs:
         with:
           subject-path: dist/*
       - name: Publish package to PyPI
-        uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0
+        uses: pypa/gh-action-pypi-publish@cef221092ed1bacb1cc03d23a2d87d1d172e277b # v1.14.0
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -78,7 +78,7 @@ jobs:
         hatch env run --env "$HATCH_ENV" run-coverage
     - name: Upload coverage
       if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }}
-      uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+      uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
@@ -125,7 +125,7 @@ jobs:
       run: |
         hatch env run --env "$HATCH_ENV" run-coverage
     - name: Upload coverage
-      uses: codecov/codecov-action@1af58845a975a7985b0beb0cbe6fbbb71a41dbad # v5.5.3
+      uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
       with:
         token: ${{ secrets.CODECOV_TOKEN }}
         flags: tests
diff --git a/.github/workflows/zizmor.yml b/.github/workflows/zizmor.yml
@@ -32,4 +32,4 @@ jobs:
           persist-credentials: false
 
       - name: Run zizmor
-        uses: zizmorcore/zizmor-action@71321a20a9ded102f6e9ce5718a2fcec2c4f70d8 # v0.5.2
+        uses: zizmorcore/zizmor-action@b1d7e1fb5de872772f31590499237e7cce841e8e # v0.5.3
diff --git a/docs/release-notes.md b/docs/release-notes.md
@@ -2,6 +2,25 @@
 
 <!-- towncrier release notes start -->
 
+## 3.2.1 (2026-05-05)
+
+### Bugfixes
+
+- Fixed a `CastValue` validation bug where the "can we use an out-of-range mode" check
+  inspected the source dtype instead of the target dtype. This meant arrays with a
+  float source dtype and an integer target dtype incorrectly raised a `ValueError`
+  when configured with a `wrap` out-of-range mode. ([#3938](https://github.com/zarr-developers/zarr-python/issues/3938))
+- Fixed a bug where the codec pipeline evolved each codec against the original
+  array spec instead of the spec produced by upstream array-to-array codecs. This
+  caused failures whenever an upstream codec changed the dtype between codec
+  boundaries — e.g. arrays using `CastValue` to convert a single-byte source dtype
+  (`int8`) to a multi-byte target dtype (`int16`) raised a `ValueError` from
+  `BytesCodec` about a missing `endian` configuration. ([#3941](https://github.com/zarr-developers/zarr-python/issues/3941))
+- Fixed breakage in existing fsspec-dependent workflows caused by associating the "memory" URL scheme with
+instances of `ManagedMemoryStore` instead of fsspec's memory-backed store. After this change, store URLs with a "memory" scheme are handled differently when `fsspec` is installed:
+with `fsspec`, a `FsspecStore` backed by a `MemoryFileSystem` is used. Without `fsspec`,
+a `ManagedMemoryStore` is used. ([#3944](https://github.com/zarr-developers/zarr-python/issues/3944))
+
 ## 3.2.0 (2026-04-30)
 
 ### Features
diff --git a/pyproject.toml b/pyproject.toml
@@ -365,7 +365,9 @@ ignore = [
 python_version = "3.12"
 ignore_missing_imports = true
 namespace_packages = false
-
+pretty = true
+show_error_code_links = true
+show_error_context = true
 strict = true
 warn_unreachable = true
 enable_error_code = ["ignore-without-code", "redundant-expr", "truthy-bool"]
@@ -378,7 +380,6 @@ module = [
     "tests.test_config",
     "tests.test_store.test_zip",
     "tests.test_store.test_local",
-    "tests.test_store.test_fsspec",
     "tests.test_store.test_memory",
     "tests.test_codecs.test_codecs",
     "tests.test_metadata.*",
@@ -394,6 +395,7 @@ strict = false
 # and fix the errors
 [[tool.mypy.overrides]]
 module = [
+    "tests.test_store.test_fsspec",
     "tests.test_group",
     "tests.test_indexing",
     "tests.test_properties",
diff --git a/src/zarr/codecs/cast_value.py b/src/zarr/codecs/cast_value.py
@@ -54,20 +54,31 @@ class ScalarMap(TypedDict, total=False):
 
 
 # see https://github.com/zarr-developers/zarr-extensions/tree/main/codecs/cast_value
-PERMITTED_DATA_TYPE_NAMES: Final[set[str]] = {
+CAST_VALUE_INT_DTYPES: Final[set[str]] = {
+    # signed
     "int2",
     "int4",
     "int8",
     "int16",
     "int32",
     "int64",
-    "int64uint2",
+    # unsigned
+    "uint2",
     "uint4",
     "uint8",
     "uint16",
     "uint32",
     "uint64",
-    "uint64float4_e2m1fn",
+}
+"""Integer dtype identifiers permitted as the source or target of `cast_value`.
+
+Membership in this set drives the `out_of_range="wrap"` rule, which the
+spec restricts to integral targets that use two's-complement representation
+for modular arithmetic.
+"""
+
+CAST_VALUE_FLOAT_DTYPES: Final[set[str]] = {
+    "float4_e2m1fn",
     "float6_e2m3fn",
     "float6_e3m2fn",
     "float8_e3m4",
@@ -82,6 +93,10 @@ class ScalarMap(TypedDict, total=False):
     "float32",
     "float64",
 }
+"""Floating-point dtype identifiers permitted as the source or target of `cast_value`."""
+
+PERMITTED_DATA_TYPE_NAMES: Final[set[str]] = CAST_VALUE_INT_DTYPES | CAST_VALUE_FLOAT_DTYPES
+"""All dtype identifiers the `cast_value` codec is defined for."""
 
 
 def parse_scalar_map(obj: ScalarMapJSON | ScalarMap) -> ScalarMap:
@@ -240,15 +255,22 @@ def validate(
         dtype: ZDType[TBaseDType, TBaseScalar],
         chunk_grid: ChunkGridMetadata,
     ) -> None:
-        target_name = dtype.to_json(zarr_format=3)
-        if target_name not in PERMITTED_DATA_TYPE_NAMES:
+        # `dtype` is the source (the array's dtype); `self.dtype` is the
+        # cast target. The spec requires both to be permitted, and rules
+        # like `out_of_range="wrap"` apply to the target.
+        source_name = dtype.to_json(zarr_format=3)
+        target_name = self.dtype.to_json(zarr_format=3)
+        for role, name in (("source", source_name), ("target", target_name)):
+            if name not in PERMITTED_DATA_TYPE_NAMES:
+                raise ValueError(
+                    f"The cast_value codec only supports integer and floating-point data types. "
+                    f"Got {role} dtype {name}."
+                )
+        if self.out_of_range == "wrap" and target_name not in CAST_VALUE_INT_DTYPES:
             raise ValueError(
-                f"The cast_value codec only supports integer and floating-point data types. "
-                f"Got dtype {target_name}."
+                f"out_of_range='wrap' is only valid for integer target types. "
+                f"Got target dtype {target_name}."
             )
-        target_native = dtype.to_native_dtype()
-        if self.out_of_range == "wrap" and not np.issubdtype(target_native, np.integer):
-            raise ValueError("out_of_range='wrap' is only valid for integer target types.")
 
         if self.scalar_map is not None:
             self._validate_scalar_map(dtype, self.dtype)
diff --git a/src/zarr/core/codec_pipeline.py b/src/zarr/core/codec_pipeline.py
@@ -187,7 +187,18 @@ class BatchedCodecPipeline(CodecPipeline):
     batch_size: int
 
     def evolve_from_array_spec(self, array_spec: ArraySpec) -> Self:
-        return type(self).from_codecs(c.evolve_from_array_spec(array_spec=array_spec) for c in self)
+        # Each codec must be evolved against the spec it will actually see
+        # at run-time, not the original array spec. Earlier array->array
+        # codecs may transform the dtype (e.g. cast_value), so the spec
+        # threaded into later codecs (the array->bytes serializer and any
+        # bytes->bytes filters) must reflect those transformations.
+        evolved: list[Codec] = []
+        spec = array_spec
+        for codec in self:
+            evolved_codec = codec.evolve_from_array_spec(array_spec=spec)
+            evolved.append(evolved_codec)
+            spec = evolved_codec.resolve_metadata(spec)
+        return type(self).from_codecs(evolved)
 
     @classmethod
     def from_codecs(cls, codecs: Iterable[Codec], *, batch_size: int | None = None) -> Self:
diff --git a/src/zarr/core/metadata/v3.py b/src/zarr/core/metadata/v3.py
@@ -503,7 +503,21 @@ def __init__(
             config=ArrayConfig.from_dict({}),  # TODO: config is not needed here.
             prototype=default_buffer_prototype(),  # TODO: prototype is not needed here.
         )
-        codecs_parsed = tuple(c.evolve_from_array_spec(array_spec) for c in codecs_parsed_partial)
+        # Thread the spec through evolution: each codec must be evolved against
+        # the spec it will actually see at run-time, not the original array spec.
+        # Earlier array->array codecs may transform the dtype (e.g. cast_value),
+        # so the spec passed to later codecs must reflect those transformations.
+        # Per-codec validate() must run before resolve_metadata(), since the
+        # latter may rely on invariants the former checks (e.g. cast_value
+        # rejects complex source dtypes that would otherwise crash _do_cast).
+        evolved: list[Codec] = []
+        spec = array_spec
+        for c in codecs_parsed_partial:
+            evolved_codec = c.evolve_from_array_spec(spec)
+            evolved_codec.validate(shape=spec.shape, dtype=spec.dtype, chunk_grid=chunk_grid_parsed)
+            evolved.append(evolved_codec)
+            spec = evolved_codec.resolve_metadata(spec)
+        codecs_parsed = tuple(evolved)
         validate_codecs(codecs_parsed_partial, data_type)
 
         object.__setattr__(self, "shape", shape_parsed)
diff --git a/src/zarr/storage/_common.py b/src/zarr/storage/_common.py
@@ -369,7 +369,7 @@ async def make_store(
         return await LocalStore.open(root=store_like, mode=mode, read_only=_read_only)
 
     elif isinstance(store_like, str) and parsed is not None:
-        if parsed.scheme == "memory":
+        if parsed.scheme == "memory" and not _has_fsspec:
             # Create or get a ManagedMemoryStore
             return ManagedMemoryStore(name=parsed.name, path=parsed.path, read_only=_read_only)
         elif parsed.scheme == "file" or not parsed.scheme:
diff --git a/tests/test_codec_pipeline.py b/tests/test_codec_pipeline.py
diff --git a/tests/test_codecs/test_cast_value.py b/tests/test_codecs/test_cast_value.py
diff --git a/tests/test_store/test_fsspec.py b/tests/test_store/test_fsspec.py
