Merge branch 'main' into ruff-test

Author: zm711

.github/scripts/check_kilosort4_releases.py

@@ -1,10 +1,9 @@
 import os
-import re
 from pathlib import Path
 import requests
 import json
 from packaging.version import parse
-import spikeinterface
+
 
 def get_pypi_versions(package_name):
     """
@@ -16,8 +15,10 @@ def get_pypi_versions(package_name):
     response.raise_for_status()
     data = response.json()
     versions = list(sorted(data["releases"].keys()))
-    # Filter out versions that are less than 4.0.16
-    versions = [ver for ver in versions if parse(ver) >= parse("4.0.16")]
+    # Filter out versions that are less than 4.0.16 and different from 4.0.26 and 4.0.27
+    # (buggy - https://github.com/MouseLand/Kilosort/releases/tag/v4.0.26)
+    versions = [ver for ver in versions if parse(ver) >= parse("4.0.16") and
+                parse(ver) not in [parse("4.0.26"), parse("4.0.27")]]
     return versions
 
 

.github/scripts/determine_testing_environment.py

@@ -48,7 +48,7 @@
     elif changed_file.name == "nwbextractors.py":
         extractors_changed = True  # There are NWB tests that are not streaming
         stream_extractors_changed = True
-    elif changed_file.name == "iblextractors.py":
+    elif changed_file.name == "iblextractors.py" or changed_file.name == "test_iblextractors.py":
         stream_extractors_changed = True
     elif "core" in changed_file.parts:
         core_changed = True

.github/scripts/test_kilosort4_ci.py

@@ -258,17 +258,22 @@ def test_compute_preprocessing_arguments(self):
         self._check_arguments(compute_preprocessing, ["ops", "device", "tic0", "file_object"])
 
     def test_compute_drift_location_arguments(self):
-        self._check_arguments(
-            compute_drift_correction, ["ops", "device", "tic0", "progress_bar", "file_object", "clear_cache"]
-        )
+        expected_arguments = ["ops", "device", "tic0", "progress_bar", "file_object", "clear_cache"]
+        if parse(kilosort.__version__) >= parse("4.0.28"):
+            expected_arguments += ["verbose"]
+        self._check_arguments(compute_drift_correction, expected_arguments)
 
     def test_detect_spikes_arguments(self):
-        self._check_arguments(detect_spikes, ["ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"])
+        expected_arguments = ["ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
+        if parse(kilosort.__version__) >= parse("4.0.28"):
+            expected_arguments += ["verbose"]
+        self._check_arguments(detect_spikes, expected_arguments)
 
     def test_cluster_spikes_arguments(self):
-        self._check_arguments(
-            cluster_spikes, ["st", "tF", "ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
-        )
+        expected_arguments = ["st", "tF", "ops", "device", "bfile", "tic0", "progress_bar", "clear_cache"]
+        if parse(kilosort.__version__) >= parse("4.0.28"):
+            expected_arguments += ["verbose"]
+        self._check_arguments(cluster_spikes, expected_arguments)
 
     def test_save_sorting_arguments(self):
         expected_arguments = ["ops", "results_dir", "st", "clu", "tF", "Wall", "imin", "tic0", "save_extra_vars"]

.github/workflows/all-tests.yml

@@ -10,9 +10,7 @@ on:
       - main
 
 env:
-  KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
-  KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
-  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
+  KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }}
 
 concurrency:  # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -36,7 +34,7 @@ jobs:
 
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v46.0.1
 
       - name: List all changed files
         shell: bash
@@ -76,6 +74,9 @@ jobs:
           pip install -e .[test_core]
         shell: bash
 
+      - name: Pip list
+        run: pip list
+
       - name: Test core
         run: pytest -m "core"
         shell: bash
@@ -136,93 +137,109 @@ jobs:
         if: env.RUN_EXTRACTORS_TESTS == 'true'
         run: |
           pip install -e .[extractors,streaming_extractors,test_extractors]
+          pip list
           ./.github/run_tests.sh "extractors and not streaming_extractors" --no-virtual-env
 
       - name: Test streaming extractors
         shell: bash
         if: env.RUN_STREAMING_EXTRACTORS_TESTS == 'true'
         run: |
           pip install -e .[streaming_extractors,test_extractors]
+          pip list
           ./.github/run_tests.sh "streaming_extractors" --no-virtual-env
 
       - name: Test preprocessing
         shell: bash
         if: env.RUN_PREPROCESSING_TESTS == 'true'
         run: |
           pip install -e .[preprocessing,test_preprocessing]
+          pip list
           ./.github/run_tests.sh "preprocessing and not deepinterpolation" --no-virtual-env
 
       - name: Install remaining testing dependencies # TODO: Remove this step once we have better modularization
         shell: bash
         run: |
           pip install -e .[test]
+          pip list
 
       - name: Test postprocessing
         shell: bash
         if: env.RUN_POSTPROCESSING_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh postprocessing --no-virtual-env
 
       - name: Test quality metrics
         shell: bash
         if: env.RUN_QUALITYMETRICS_TESTS == 'true'
         run: |
           pip install -e .[qualitymetrics]
+          pip list
           ./.github/run_tests.sh qualitymetrics --no-virtual-env
 
       - name: Test comparison
         shell: bash
         if: env.RUN_COMPARISON_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh comparison --no-virtual-env
 
       - name: Test core sorters
         shell: bash
         if: env.RUN_SORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh sorters --no-virtual-env
 
       - name: Test internal sorters
         shell: bash
         if: env.RUN_INTERNAL_SORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh sorters_internal --no-virtual-env
 
       - name: Test curation
         shell: bash
         if: env.RUN_CURATION_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh curation --no-virtual-env
 
       - name: Test widgets
         shell: bash
         if: env.RUN_WIDGETS_TESTS == 'true'
+        env:
+          KACHERY_ZONE: "scratch"
         run: |
           pip install -e .[full,widgets]
+          pip list
           ./.github/run_tests.sh widgets --no-virtual-env
 
       - name: Test exporters
         shell: bash
         if: env.RUN_EXPORTERS_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh exporters --no-virtual-env
 
       - name: Test sortingcomponents
         shell: bash
         if: env.RUN_SORTINGCOMPONENTS_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh sortingcomponents --no-virtual-env
 
       - name: Test generation
         shell: bash
         if: env.RUN_GENERATION_TESTS == 'true'
         run: |
           pip install -e .[full]
+          pip list
           ./.github/run_tests.sh generation --no-virtual-env

.github/workflows/deepinterpolation.yml

@@ -25,7 +25,7 @@ jobs:
           python-version: '3.9'
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v46.0.1
       - name: Deepinteprolation changes
         id: modules-changed
         run: |
@@ -43,7 +43,12 @@ jobs:
           pip install tensorflow==2.7.0
           pip install deepinterpolation@git+https://github.com/AllenInstitute/deepinterpolation.git
           pip install protobuf==3.20.*
+          pip install numpy==1.26.4
           pip install -e .[full,test_core]
+      - name: Pip list
+        if: ${{ steps.modules-changed.outputs.DEEPINTERPOLATION_CHANGED == 'true' }}
+        run: |
+          pip list
       - name: Test DeepInterpolation with pytest
         if: ${{ steps.modules-changed.outputs.DEEPINTERPOLATION_CHANGED == 'true' }}
         run: |

.github/workflows/full-test-with-codecov.yml

@@ -6,9 +6,7 @@ on:
     - cron: "0 12 * * *"  # Daily at noon UTC
 
 env:
-  KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
-  KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
-  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
+  KACHERY_API_KEY: ${{ secrets.KACHERY_API_KEY }}
 
 jobs:
   full-tests-with-codecov:
@@ -41,6 +39,8 @@ jobs:
           restore-keys: ${{ runner.os }}-datasets
       - name: Install packages
         uses: ./.github/actions/build-test-environment
+      - name: Pip list
+        run: pip list
       - name: run tests
         env:
           HDF5_PLUGIN_PATH: ${{ github.workspace }}/hdf5_plugin_path_maxwell

.github/workflows/issue-on-change-matlab.yml

@@ -18,7 +18,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Get changed files
         id: changed-files
-        uses: tj-actions/changed-files@v41
+        uses: tj-actions/changed-files@v46.0.1
         with:
           files: |
             **/*.m

doc/api.rst

@@ -143,7 +143,9 @@ Non-NEO-based
     .. autofunction:: read_mcsh5
     .. autofunction:: read_mda_recording
     .. autofunction:: read_mda_sorting
-    .. autofunction:: read_nwb
+    .. autofunction:: read_nwb_sorting
+    .. autofunction:: read_nwb_recording
+    .. autofunction:: read_nwb_timeseries
     .. autofunction:: read_phy
     .. autofunction:: read_shybrid_recording
     .. autofunction:: read_shybrid_sorting
@@ -168,7 +170,7 @@ spikeinterface.preprocessing
     .. autofunction:: astype
     .. autofunction:: average_across_direction
     .. autofunction:: bandpass_filter
-    .. autofunction:: blank_staturation
+    .. autofunction:: blank_saturation
     .. autofunction:: center
     .. autofunction:: clip
     .. autofunction:: common_reference

doc/how_to/process_by_channel_group.rst

@@ -87,40 +87,28 @@ Splitting a recording by channel group returns a dictionary containing separate
 Preprocessing a Recording by Channel Group
 ------------------------------------------
 
-The essence of preprocessing by channel group is to first split the recording
-into separate recordings, perform the preprocessing steps, then aggregate
-the channels back together.
-
-In the below example, we loop over the split recordings, preprocessing each channel group
-individually. At the end, we use the :py:func:`~aggregate_channels` function
-to combine the separate channel group recordings back together.
+If a preprocessing function is given a dictionary of recordings, it will apply the preprocessing
+seperately to each recording in the dict, and return a dictionary of preprocessed recordings.
+Hence we can pass the ``split_recording_dict`` in the same way as we would pass a single recording
+to any preprocessing function.
 
 .. code-block:: python
 
-    preprocessed_recordings = []
-
-   # loop over the recordings contained in the dictionary
-    for chan_group_rec in split_recordings_dict.values():
-
-        # Apply the preprocessing steps to the channel group in isolation
-        shifted_recording = spre.phase_shift(chan_group_rec)
+    shifted_recordings = spre.phase_shift(split_recording_dict)
+    filtered_recording = spre.bandpass_filter(shifted_recording)
+    referenced_recording = spre.common_reference(filtered_recording)
 
-        filtered_recording = spre.bandpass_filter(shifted_recording)
+We can then aggregate the recordings back together using the ``aggregate_channels`` function
 
-        referenced_recording = spre.common_reference(filtered_recording)
-
-        preprocessed_recordings.append(referenced_recording)
+.. code-block:: python
 
-    # Combine our preprocessed channel groups back together
-    combined_preprocessed_recording = aggregate_channels(preprocessed_recordings)
+    combined_preprocessed_recording = aggregate_channels(referenced_recording)
 
-Now, when this recording is used in sorting, plotting, or whenever
+Now, when ``combined_preprocessed_recording`` is used in sorting, plotting, or whenever
 calling its :py:func:`~get_traces` method, the data will have been
 preprocessed separately per-channel group (then concatenated
 back together under the hood).
 
-It is strongly recommended to use the above structure to preprocess by channel group.
-
 .. note::
 
     The splitting and aggregation of channels for preprocessing is flexible.

doc/modules/preprocessing.rst

@@ -19,7 +19,7 @@ In this code example, we build a preprocessing chain with two steps:
 
 .. code-block:: python
 
-    import spikeinterface.preprocessing import bandpass_filter, common_reference
+    from spikeinterface.preprocessing import bandpass_filter, common_reference
 
     # recording is a RecordingExtractor object
     recording_f = bandpass_filter(recording=recording, freq_min=300, freq_max=6000)
@@ -196,18 +196,18 @@ The whitened traces are then the dot product between the traces and the :code:`W
 
 * :py:func:`~spikeinterface.preprocessing.whiten()`
 
-clip() / blank_staturation()
-^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+clip() / blank_saturation()
+^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
 We can limit traces between a user-defined minimum and maximum using :code:`clip()` function.
-The :code:`blank_staturation()` function is similar, but it automatically estimates the limits by using quantiles.
+The :code:`blank_saturation()` function is similar, but it automatically estimates the limits by using quantiles.
 
 .. code-block:: python
 
     rec_w = clip(recording=rec, a_min=-250., a_max=260)
 
 * :py:func:`~spikeinterface.preprocessing.clip()`
-* :py:func:`~spikeinterface.preprocessing.blank_staturation()`
+* :py:func:`~spikeinterface.preprocessing.blank_saturation()`
 
 
 highpass_spatial_filter()