Support zarr v3

.github/scripts/generate_zarr_v2_fixtures.py

@@ -0,0 +1,101 @@
+#!/usr/bin/env python3
+"""
+Generate zarr v2 fixtures for backward compatibility tests.
+
+Run this script with an old spikeinterface version and zarr<3, e.g.:
+    pip install "spikeinterface==0.104.0" "zarr<3"
+    python generate_zarr_v2_fixtures.py --output /tmp/zarr_v2_fixtures
+
+The script saves:
+  - recording.zarr  : a small ZarrRecordingExtractor
+  - sorting.zarr    : a small ZarrSortingExtractor
+  - expected_values.json : key values used to verify correct loading
+"""
+import argparse
+import shutil
+import json
+from pathlib import Path
+
+import numpy as np
+import zarr
+
+import spikeinterface as si
+
+
+def main(output_dir: Path) -> None:
+    print(f"spikeinterface version : {si.__version__}")
+    print(f"zarr version           : {zarr.__version__}")
+
+
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    recording, sorting = si.generate_ground_truth_recording(durations=[10, 5],num_channels=32, num_units=10, seed=0)
+    # save to binary to make them JSON serializable for later expected values extraction
+    recording = recording.save(folder=output_dir / "recording_binary", overwrite=True)
+    sorting = sorting.save(folder=output_dir / "sorting_binary", overwrite=True)
+    # --- save recording ---
+    recording_path = output_dir / "recording.zarr"
+    recording_zarr = recording.save(format="zarr", folder=recording_path, overwrite=True)
+    print(f"Saved recording  -> {recording_path}")
+
+    # --- save sorting ---
+    sorting_path = output_dir / "sorting.zarr"
+    sorting_zarr = sorting.save(format="zarr", folder=sorting_path, overwrite=True)
+    print(f"Saved sorting    -> {sorting_path}")
+
+    # --- save SortingAnalyzer ---
+    # Reload the recording from zarr so it is a serializable ZarrRecordingExtractor,
+    # which the analyzer can store as provenance.
+    analyzer_path = output_dir / "analyzer.zarr"
+    if analyzer_path.is_dir():
+        shutil.rmtree(analyzer_path)
+    analyzer = si.create_sorting_analyzer(
+        sorting_zarr, recording_zarr, format="zarr", folder=analyzer_path, overwrite=True
+    )
+    analyzer.compute(["random_spikes", "templates"])
+    print(f"Saved analyzer   -> {analyzer_path}")
+
+    # Reload to verify templates are accessible before writing expected values
+    templates_array = analyzer.get_extension("templates").get_data()
+
+    # --- capture expected values for later assertion ---
+    expected = {
+        "spikeinterface_version": si.__version__,
+        "zarr_version": zarr.__version__,
+        "recording": {
+            "num_channels": int(recording.get_num_channels()),
+            "num_segments": int(recording.get_num_segments()),
+            "sampling_frequency": float(recording.get_sampling_frequency()),
+            "num_samples_per_segment": [int(recording.get_num_samples(seg)) for seg in range(recording.get_num_segments())],
+            "channel_ids": recording.get_channel_ids().tolist(),
+            "dtype": str(recording.get_dtype()),
+            # first 10 frames of segment 0 for all channels
+            "traces_seg0_first10": recording.get_traces(start_frame=0, end_frame=10, segment_index=0).tolist(),
+        },
+        "sorting": {
+            "num_segments": int(sorting.get_num_segments()),
+            "sampling_frequency": float(sorting.get_sampling_frequency()),
+            "unit_ids": sorting.get_unit_ids().tolist(),
+            "spike_trains_seg0": {
+                str(uid): sorting.get_unit_spike_train(unit_id=uid, segment_index=0).tolist()
+                for uid in sorting.unit_ids
+            },
+        },
+        "analyzer": {
+            "num_units": int(analyzer.get_num_units()),
+            "num_channels": int(analyzer.get_num_channels()),
+            "templates_shape": list(templates_array.shape),
+        },
+    }
+
+    expected_path = output_dir / "expected_values.json"
+    with open(expected_path, "w") as f:
+        json.dump(expected, f, indent=2)
+    print(f"Saved expected   -> {expected_path}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Generate zarr v2 fixtures for backward compatibility tests")
+    parser.add_argument("--output", type=Path, required=True, help="Directory to write fixtures into")
+    args = parser.parse_args()
+    main(args.output)

.github/workflows/all-tests.yml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.10", "3.13"]  # Lower and higher versions we support
+        python-version: ["3.11", "3.13"]  # Lower and higher versions we support
         os: [macos-latest, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v6

.github/workflows/caches_cron_job.yml

@@ -19,10 +19,10 @@ jobs:
     steps:
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
           ignore-nothing-to-cache: true # some runs do not require building a cache (ie macOS) this says this is okay
       - name: Create the directory to store the data

.github/workflows/core-test.yml

@@ -22,10 +22,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
       - name: Install dependencies
         run: |

.github/workflows/cross_version_serialization.yml

@@ -43,7 +43,7 @@ jobs:
     steps:
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - id: set
         env:
           GITHUB_EVENT_NAME: ${{ github.event_name }}
@@ -93,17 +93,17 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Set up uv
         uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: false
 
       - name: Generate fixtures with spikeinterface ${{ matrix.si-version }}
         run: |
-          uv run --isolated --no-project --python 3.10 \
+          uv run --isolated --no-project --python 3.11 \
             --with "spikeinterface[core]==${{ matrix.si-version }}" \
             python .github/scripts/serialization/serialize_objects.py "$SI_SERIALIZATION_FIXTURES_DIR"
 

.github/workflows/deepinterpolation.yml

@@ -1,10 +1,8 @@
 name: Testing deepinterpolation
 
+# Manual only — deepinterpolation requires Python 3.11, incompatible with 3.11+ required by Zarr 3.0.0+
 on:
-  pull_request:
-    types: [synchronize, opened, reopened]
-    branches:
-      - main
+  workflow_dispatch:
 
 concurrency: # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -22,10 +20,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: false
       - name: Get changed files
         id: changed-files

.github/workflows/test_containers_docker.yml

@@ -14,10 +14,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: '3.11'
           enable-cache: true
       - name: Python version
         run: python --version

.github/workflows/test_containers_singularity.yml

@@ -15,10 +15,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: '3.11'
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: '3.11'
           enable-cache: true
       - uses: eWaterCycle/setup-singularity@v7
         with:

.github/workflows/test_imports.yml

@@ -22,10 +22,10 @@ jobs:
       - uses: actions/checkout@v6
       - uses: actions/setup-python@v6
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - uses: astral-sh/setup-uv@v7
         with:
-          python-version: "3.10"
+          python-version: "3.11"
           enable-cache: true
       - name: Install Spikeinterface with only core dependencies
         run: |

.github/workflows/test_zarr_compat.yml

@@ -0,0 +1,51 @@
+name: Test zarr backwards compatibility
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types: [synchronize, opened, reopened]
+    branches:
+      - main
+    paths:
+      - "src/spikeinterface/core/zarrextractors.py"
+      - "src/spikeinterface/core/zarrrecordingextractor.py"
+      - "src/spikeinterface/core/tests/test_zarr_backwards_compat.py"
+      - ".github/workflows/test_zarr_compat.yml"
+      - ".github/scripts/generate_zarr_v2_fixtures.py"
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  test-zarr-compat:
+    name: zarr v2 -> v3 backwards compatibility
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+
+      - name: Install SI 0.104.0 with zarr v2
+        run: uv pip install --system "spikeinterface==0.104.0" "zarr<3"
+
+      - name: Generate zarr v2 fixtures
+        run: python .github/scripts/generate_zarr_v2_fixtures.py --output /tmp/zarr_v2_fixtures
+
+      - name: Install current SI with zarr v3
+        run: uv pip install --system -e . --group test-core
+
+      - name: Check zarr version is v3
+        run: python -c "import zarr; v = zarr.__version__; print(f'zarr {v}'); assert int(v.split('.')[0]) >= 3"
+
+      - name: Run backward compatibility tests
+        env:
+          ZARR_V2_FIXTURES_PATH: /tmp/zarr_v2_fixtures
+        run: pytest src/spikeinterface/core/tests/test_zarr_backwards_compat.py -v

pyproject.toml

@@ -7,7 +7,7 @@ authors = [
 ]
 description = "Python toolkit for analysis, visualization, and comparison of spike sorting output"
 readme = "README.md"
-requires-python = ">=3.10"
+requires-python = ">=3.11"
 classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "License :: OSI Approved :: MIT License",
@@ -23,15 +23,16 @@ dependencies = [
     "numpy>=2.0.0",
     "threadpoolctl>=3.0.0",
     "tqdm",
-    "zarr>=2.18,<3",
+    "zarr>=3,<4",
     # for release we need pypi, so this needs to be commented
     "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    # for testing we need probeinterface Zarr v3 branch, so this needs to be commented
+    #"probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    "probeinterface @ git+https://github.com/alejoe91/probeinterface.git@zarrv3",
     # "neo>=0.14.4",
     # "probeinterface>=0.3.2",
     "packaging",
     "pydantic",
-    "numcodecs<0.16.0", # For supporting zarr < 3
 ]
 
 [build-system]
@@ -66,7 +67,9 @@ changelog = "https://spikeinterface.readthedocs.io/en/latest/whatisnew.html"
 extractors = [
     "MEArec>=1.8",
     "pynwb>=2.6.0",
-    "hdmf-zarr>=0.11.0",
+    # FOR TESTING
+    "hdmf-zarr @ git+https://github.com/hdmf-dev/hdmf-zarr.git@zarr-v3-migration",
+    # "hdmf-zarr>=0.11.0",
     "pyedflib>=0.1.30",
     "lxml", # lxml for neuroscope
     "scipy",
@@ -82,9 +85,11 @@ streaming_extractors = [
     "fsspec",
     "aiohttp",
     "requests",
-    "hdmf-zarr>=0.11.0",
+    # FOR TESTING
+    "hdmf-zarr @ git+https://github.com/hdmf-dev/hdmf-zarr.git@zarr-v3-migration",
+    # "hdmf-zarr>=0.11.0",
     "remfile",
-    "s3fs"
+    "s3fs>=2025.7.0"
 ]