Merge branch 'main' into feat/get_json

d-v-b · web-flow · commit 6f71a6b802e5 · 2026-01-16T12:10:18.000+01:00
diff --git a/.github/workflows/check_changelogs.yml b/.github/workflows/check_changelogs.yml
@@ -12,7 +12,7 @@ jobs:
       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
 
       - name: Install uv
-        uses: astral-sh/setup-uv@681c641aba71e4a1c380be3ab5e12ad51f415867 # v7.1.6
+        uses: astral-sh/setup-uv@61cb8a9741eeb8a550a1b8544337180c0fc8476b # v7.2.0
 
       - name: Check changelog entries
         run: uv run --no-sync python ci/check_changelog_entries.py
diff --git a/.github/workflows/codspeed.yml b/.github/workflows/codspeed.yml
@@ -0,0 +1,35 @@
+name: CodSpeed Benchmarks
+
+on:
+  push:
+    branches:
+      - "main"
+  pull_request:
+  # `workflow_dispatch` allows CodSpeed to trigger backtest
+  # performance analysis in order to generate initial data.
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  benchmarks:
+    name: Run benchmarks
+    runs-on: codspeed-macro
+    steps:
+    - uses: actions/checkout@v5
+      with:
+        fetch-depth: 0  # grab all branches and tags
+    - name: Set up Python
+      uses: actions/setup-python@v6
+      with:
+        python-version: "3.11"
+    - name: Install Hatch
+      run: |
+        python -m pip install --upgrade pip
+        pip install hatch
+    - name: Run the benchmarks
+      uses: CodSpeedHQ/action@v4
+      with:
+          mode: walltime
+          run: hatch run test.py3.11-minimal:pytest tests/benchmarks --codspeed
diff --git a/.github/workflows/gpu_test.yml b/.github/workflows/gpu_test.yml
@@ -19,13 +19,12 @@ concurrency:
 
 jobs:
   test:
-    name: py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
+    name: py=${{ matrix.python-version }}, deps=${{ matrix.dependency-set }}
 
     runs-on: gpu-runner
     strategy:
       matrix:
         python-version: ['3.11']
-        numpy-version: ['2.2']
         dependency-set: ["minimal"]
 
     steps:
@@ -61,11 +60,11 @@ jobs:
         pip install hatch
     - name: Set Up Hatch Env
       run: |
-        hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
-        hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
+        hatch env create gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }}
+        hatch env run -e gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }} list-env
     - name: Run Tests
       run: |
-        hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage-gpu
+        hatch env run --env gputest.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run-coverage-gpu
 
     - name: Upload coverage
       uses: codecov/codecov-action@13ce06bfc6bbe3ecf90edbbf1bc32fe5978ca1d3  # v5.3.1
diff --git a/.github/workflows/hypothesis.yaml b/.github/workflows/hypothesis.yaml
@@ -24,7 +24,6 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.12']
-        numpy-version: ['2.2']
         dependency-set: ["optional"]
 
     steps:
@@ -47,8 +46,8 @@ jobs:
         pip install hatch
     - name: Set Up Hatch Env
       run: |
-        hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
-        hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
+        hatch env create test.py${{ matrix.python-version }}-${{ matrix.dependency-set }}
+        hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} list-env
     # https://github.com/actions/cache/blob/main/tips-and-workarounds.md#update-a-cache
     - name: Restore cached hypothesis directory
       id: restore-hypothesis-cache
@@ -64,7 +63,7 @@ jobs:
       id: status
       run: |
         echo "Using Hypothesis profile: $HYPOTHESIS_PROFILE"
-        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-hypothesis
+        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run-hypothesis
 
     # explicitly save the cache so it gets updated, also do this even if it fails.
     - name: Save cached hypothesis directory
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -16,29 +16,27 @@ concurrency:
 
 jobs:
   test:
-    name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, np=${{ matrix.numpy-version }}, deps=${{ matrix.dependency-set }}
+    name: os=${{ matrix.os }}, py=${{ matrix.python-version }}, deps=${{ matrix.dependency-set }}
 
     strategy:
       matrix:
         python-version: ['3.11', '3.12', '3.13']
-        numpy-version: ['1.26', '2.2']
         dependency-set: ["minimal", "optional"]
         os: ["ubuntu-latest"]
         include:
           - python-version: '3.11'
-            numpy-version: '1.26'
             dependency-set: 'optional'
             os: 'macos-latest'
+
           - python-version: '3.13'
-            numpy-version: '2.2'
             dependency-set: 'optional'
             os: 'macos-latest'
+
           - python-version: '3.11'
-            numpy-version: '1.26'
             dependency-set: 'optional'
             os: 'windows-latest'
+
           - python-version: '3.13'
-            numpy-version: '2.2'
             dependency-set: 'optional'
             os: 'windows-latest'
     runs-on: ${{ matrix.os }}
@@ -58,13 +56,13 @@ jobs:
         pip install hatch
     - name: Set Up Hatch Env
       run: |
-        hatch env create test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }}
-        hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} list-env
+        hatch env create test.py${{ matrix.python-version }}-${{ matrix.dependency-set }}
+        hatch env run -e test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} list-env
     - name: Run Tests
       env:
         HYPOTHESIS_PROFILE: ci
       run: |
-        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.numpy-version }}-${{ matrix.dependency-set }} run-coverage
+        hatch env run --env test.py${{ matrix.python-version }}-${{ matrix.dependency-set }} run-coverage
     - name: Upload coverage
       if: ${{ matrix.dependency-set == 'optional' && matrix.os == 'ubuntu-latest' }}
       uses: codecov/codecov-action@v5
@@ -152,4 +150,4 @@ jobs:
           contains(needs.*.result, 'cancelled')
         run: exit 1
       - name: Success
-        run: echo Success!
+        run: echo Success!
diff --git a/changes/3562.misc.md b/changes/3562.misc.md
@@ -0,0 +1 @@
+Add continuous performance benchmarking infrastructure.
diff --git a/changes/3636.misc.md b/changes/3636.misc.md
@@ -0,0 +1 @@
+The minimum required version of NumPy is now 2.0.
diff --git a/changes/3648.misc.md b/changes/3648.misc.md
@@ -0,0 +1 @@
+Fix deprecation of setting a shape on an array directly in ``numpy`` 2.5+.
diff --git a/changes/3656.misc.md b/changes/3656.misc.md
@@ -0,0 +1 @@
+Removed *rich* and *mypy* from the `[test]` dependencies, and added a new `[dev]` dependency group that can be used to install all the development dependencies.
diff --git a/docs/contributing.md b/docs/contributing.md
@@ -264,4 +264,13 @@ If an existing Zarr format version changes, or a new version of the Zarr format
 ## Release procedure
 
 Open an issue on GitHub announcing the release using the release checklist template:
-[https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md](https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md>). The release checklist includes all steps necessary for the release.
+[https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md](https://github.com/zarr-developers/zarr-python/issues/new?template=release-checklist.md>). The release checklist includes all steps necessary for the release.
+
+## Benchmarks
+
+Zarr uses [pytest-benchmark](https://pytest-benchmark.readthedocs.io/en/latest/) for running
+performance benchmarks as part of our test suite. The benchmarks can be are found in `tests/benchmarks`.
+By default pytest is configured to run these benchmarks as plain tests (i.e., no benchmarking). To run
+a benchmark with timing measurements, use the `--benchmark-enable` when invoking `pytest`.
+
+The benchmarks are run as part of the continuous integration suite through [codspeed](https://codspeed.io/zarr-developers/zarr-python).
diff --git a/docs/user-guide/installation.md b/docs/user-guide/installation.md
@@ -6,7 +6,7 @@ Required dependencies include:
 
 - [Python](https://docs.python.org/3/) (3.11 or later)
 - [packaging](https://packaging.pypa.io) (22.0 or later)
-- [numpy](https://numpy.org) (1.26 or later)
+- [numpy](https://numpy.org) (2.0 or later)
 - [numcodecs](https://numcodecs.readthedocs.io) (0.14 or later)
 - [google-crc32c](https://github.com/googleapis/python-crc32c) (1.5 or later)
 - [typing_extensions](https://typing-extensions.readthedocs.io) (4.9 or later)
diff --git a/pyproject.toml b/pyproject.toml
@@ -33,7 +33,7 @@ requires-python = ">=3.11"
 # If you add a new dependency here, please also add it to .pre-commit-config.yml
 dependencies = [
     'packaging>=22.0',
-    'numpy>=1.26',
+    'numpy>=2.0',
     'numcodecs>=0.14',
     'google-crc32c>=1.5',
     'typing_extensions>=4.12',
@@ -70,18 +70,18 @@ gpu = [
     "cupy-cuda12x",
 ]
 cli = ["typer"]
-# Development extras
+# Testing extras
 test = [
     "coverage>=7.10",
     "pytest",
     "pytest-asyncio",
     "pytest-cov",
     "pytest-accept",
-    "rich",
-    "mypy",
     'numpydoc',
     "hypothesis",
     "pytest-xdist",
+    "pytest-benchmark",
+    "pytest-codspeed",
     "packaging",
     "tomlkit",
     "uv",
@@ -114,6 +114,14 @@ docs = [
     'astroid<4',
     'pytest'
 ]
+# All development extras
+dev = [
+    "zarr[optional]",
+    "zarr[test]",
+    "zarr[remote_tests]",
+    "zarr[docs]",
+    "mypy",
+]
 
 [project.scripts]
 zarr = "zarr._cli.cli:app"
@@ -150,9 +158,6 @@ version.source = "vcs"
 hooks.vcs.version-file = "src/zarr/_version.py"
 
 [tool.hatch.envs.test]
-dependencies = [
-    "numpy~={matrix:numpy}",
-]
 features = ["test"]
 
 [tool.hatch.envs.test.env-vars]
@@ -163,7 +168,6 @@ COV_CORE_DATAFILE = ".coverage.eager"
 
 [[tool.hatch.envs.test.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.26", "2.2"]
 deps = ["minimal", "optional"]
 
 [tool.hatch.envs.test.overrides]
@@ -175,28 +179,27 @@ matrix.deps.dependencies = [
 run-coverage = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy"
 run-coverage-html = "pytest --cov-config=pyproject.toml --cov=src --cov-append --cov-report html"
 run-coverage-gpu = "pip install cupy-cuda12x && pytest -m gpu --cov-config=pyproject.toml --cov=src --cov-append --cov-report xml --junitxml=junit.xml -o junit_family=legacy"
-run = "run-coverage --no-cov"
+run = "run-coverage --no-cov --ignore tests/benchmarks"
 run-pytest = "run"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
 run-hypothesis = "run-coverage -nauto --run-slow-hypothesis tests/test_properties.py tests/test_store/test_stateful*"
+run-benchmark = "pytest --benchmark-enable tests/benchmarks"
 list-env = "pip list"
 
 [tool.hatch.envs.gputest]
 template = "test"
 dependencies = [
-    "numpy~={matrix:numpy}",
     "universal_pathlib",
 ]
 features = ["test", "gpu"]
 
 [[tool.hatch.envs.gputest.matrix]]
 python = ["3.11", "3.12", "3.13"]
-numpy = ["1.26", "2.2"]
 version = ["minimal"]
 
 [tool.hatch.envs.gputest.scripts]
-run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy"
+run-coverage = "pytest -m gpu --cov-config=pyproject.toml --cov=pkg --cov-report xml --cov=src --junitxml=junit.xml -o junit_family=legacy --ignore tests/benchmarks"
 run = "run-coverage --no-cov"
 run-verbose = "run-coverage --verbose"
 run-mypy = "mypy src"
@@ -234,7 +237,7 @@ python = "3.11"
 dependencies = [
     'zarr[remote]',
     'packaging==22.*',
-    'numpy==1.26.*',
+    'numpy==2.0.*',
     'numcodecs==0.14.*',  # 0.14 needed for zarr3 codecs
     'fsspec==2023.10.0',
     's3fs==2023.10.0',
@@ -405,7 +408,10 @@ doctest_optionflags = [
     "IGNORE_EXCEPTION_DETAIL",
 ]
 addopts = [
-    "--durations=10", "-ra", "--strict-config", "--strict-markers",
+    "--benchmark-columns", "min,mean,stddev,outliers,rounds,iterations",
+    "--benchmark-disable", # benchmark routines run as tests without benchmarking instrumentation
+    "--durations", "10",
+    "-ra", "--strict-config", "--strict-markers",
 ]
 filterwarnings = [
     "error",
diff --git a/src/zarr/_compat.py b/src/zarr/_compat.py
@@ -2,10 +2,16 @@
 from collections.abc import Callable
 from functools import wraps
 from inspect import Parameter, signature
-from typing import Any, TypeVar
+from typing import TYPE_CHECKING, Any, TypeVar
+
+import numpy as np
+from packaging.version import Version
 
 from zarr.errors import ZarrFutureWarning
 
+if TYPE_CHECKING:
+    from numpy.typing import NDArray
+
 T = TypeVar("T")
 
 # Based off https://github.com/scikit-learn/scikit-learn/blob/e87b32a81c70abed8f2e97483758eb64df8255e9/sklearn/utils/validation.py#L63
@@ -68,3 +74,37 @@ def inner_f(*args: Any, **kwargs: Any) -> T:
         return _inner_deprecate_positional_args(func)
 
     return _inner_deprecate_positional_args  # type: ignore[return-value]
+
+
+def _reshape_view(arr: "NDArray[Any]", shape: tuple[int, ...]) -> "NDArray[Any]":
+    """Reshape an array without copying data.
+
+    This function provides compatibility across NumPy versions for reshaping arrays
+    as views. On NumPy >= 2.1, it uses ``reshape(copy=False)`` which explicitly
+    fails if a view cannot be created. On older versions, it uses direct shape
+    assignment which has the same behavior but is deprecated in 2.5+.
+
+    Parameters
+    ----------
+    arr : NDArray
+        The array to reshape.
+    shape : tuple of int
+        The new shape.
+
+    Returns
+    -------
+    NDArray
+        A reshaped view of the array.
+
+    Raises
+    ------
+    AttributeError
+        If a view cannot be created (the array is not contiguous) on NumPy < 2.1.
+    ValueError
+        If a view cannot be created (the array is not contiguous) on NumPy >= 2.1.
+    """
+    if Version(np.__version__) >= Version("2.1"):
+        return arr.reshape(shape, copy=False)  # type: ignore[call-overload, no-any-return]
+    else:
+        arr.shape = shape
+        return arr
diff --git a/src/zarr/codecs/vlen_utf8.py b/src/zarr/codecs/vlen_utf8.py
@@ -6,6 +6,7 @@
 import numpy as np
 from numcodecs.vlen import VLenBytes, VLenUTF8
 
+from zarr._compat import _reshape_view
 from zarr.abc.codec import ArrayBytesCodec
 from zarr.core.buffer import Buffer, NDBuffer
 from zarr.core.common import JSON, parse_named_configuration
@@ -50,7 +51,7 @@ async def _decode_single(
         raw_bytes = chunk_bytes.as_array_like()
         decoded = _vlen_utf8_codec.decode(raw_bytes)
         assert decoded.dtype == np.object_
-        decoded.shape = chunk_spec.shape
+        decoded = _reshape_view(decoded, chunk_spec.shape)
         as_string_dtype = decoded.astype(chunk_spec.dtype.to_native_dtype(), copy=False)
         return chunk_spec.prototype.nd_buffer.from_numpy_array(as_string_dtype)
 
@@ -95,7 +96,7 @@ async def _decode_single(
         raw_bytes = chunk_bytes.as_array_like()
         decoded = _vlen_bytes_codec.decode(raw_bytes)
         assert decoded.dtype == np.object_
-        decoded.shape = chunk_spec.shape
+        decoded = _reshape_view(decoded, chunk_spec.shape)
         return chunk_spec.prototype.nd_buffer.from_numpy_array(decoded)
 
     async def _encode_single(
diff --git a/tests/benchmarks/__init__.py b/tests/benchmarks/__init__.py
diff --git a/tests/benchmarks/common.py b/tests/benchmarks/common.py
diff --git a/tests/benchmarks/conftest.py b/tests/benchmarks/conftest.py
diff --git a/tests/benchmarks/test_e2e.py b/tests/benchmarks/test_e2e.py
diff --git a/tests/benchmarks/test_indexing.py b/tests/benchmarks/test_indexing.py

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+Add continuous performance benchmarking infrastructure.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	`+The minimum required version of NumPy is now 2.0.`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Fix deprecation of setting a shape on an array directly in ``numpy`` 2.5+.
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1 @@`
	`1`	+Removed rich and mypy from the `[test]` dependencies, and added a new `[dev]` dependency group that can be used to install all the development dependencies.