Merge branch 'main' into fast-correlogram-merge

Author: chrishalcrow

.github/workflows/full-test-with-codecov.yml

@@ -23,7 +23,7 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
       - name: Get ephy_testing_data current head hash
         # the key depends on the last comit repo https://gin.g-node.org/NeuralEnsemble/ephy_testing_data.git
         id: vars

doc/conf.py

@@ -119,7 +119,9 @@
 
 # for sphinx gallery plugin
 sphinx_gallery_conf = {
-    'only_warn_on_example_error': True,
+    # This is the default but including here explicitly. Should build all docs and fail on gallery failures only.
+    # other option would be abort_on_example_error, but this fails on first failure. So we decided against this.
+    'only_warn_on_example_error': False,
     'examples_dirs': ['../examples/tutorials'],
     'gallery_dirs': ['tutorials' ],  # path where to save gallery generated examples
     'subsection_order': ExplicitOrder([

doc/development/development.rst

@@ -61,7 +61,7 @@ for the :code:`spikeinterface.extractors` module, you can use the following comm
 
 The markers are located in the :code:`pyproject.toml` file in the root of the repository.
 
-Note that you should install the package before running the tests. You can do this by running the following command:
+Note that you should install spikeinterface before running the tests. You can do this by running the following command:
 
 .. code-block:: bash
 
@@ -72,7 +72,7 @@ You can change the :code:`[test,extractors,full]` to install only the dependenci
 The specific environment for the CI is specified in the :code:`.github/actions/build-test-environment/action.yml` and you can
 find the full tests in the :code:`.github/workflows/full_test.yml` file.
 
-The extractor tests require datalad for some of the tests. Here are instructions for installing datalad:
+Some of the extractor tests require datalad. Here are instructions for installing datalad:
 
 Installing Datalad
 ------------------
@@ -87,13 +87,13 @@ Stylistic conventions
 
 SpikeInterface maintains a consistent coding style across the project. This helps to ensure readability and
 maintainability of the code, making it easier for contributors to collaborate. To facilitate code style
-for the developer we use the follwing tools and conventions:
+for the developer we use the following tools and conventions:
 
 
 Install Black and pre-commit
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-We use the python formatter Black, with defaults set in the :code:`pyproject.toml`. This allows for
+We use the Python formatter Black, with defaults set in the :code:`pyproject.toml`. This allows for
 easy local formatting of code.
 
 To install Black, you can use pip, the Python package installer. Run the following command in your terminal:
@@ -218,7 +218,7 @@ Adding new documentation
 
 Documentation can be added as a
 `sphinx-gallery <https://sphinx-gallery.github.io/stable/index.html>`_
-python file ('tutorials')
+Python file ('tutorials')
 or a
 `sphinx rst <https://sphinx-tutorial.readthedocs.io/step-1/>`_
 file (all other sections).
@@ -242,7 +242,7 @@ To run code coverage locally, you can use the following command:
 
 This will run the tests and generate a report in the :code:`htmlcov` folder. You can open the :code:`index.html` file in your browser to see the report.
 
-Note, however, that the running time of the command above will be slow. If you want to run the tests for a specific module, you can use the following command:
+Note, however, that the running time of the command above will be quite long. If you want to run the tests for a specific module, you can use the following command:
 
 .. code-block:: bash
 
@@ -252,21 +252,21 @@ Implement a new extractor
 -------------------------
 
 SpikeInterface already supports over 30 file formats, but the acquisition system you use might not be among the
-supported formats list (***ref***). Most of the extractors rely on the `NEO <https://github.com/NeuralEnsemble/python-neo>`_
+supported formats list (****ref****). Most of the extractors rely on the `NEO <https://github.com/NeuralEnsemble/python-neo>`_
 package to read information from files.
 Therefore, to implement a new extractor to handle the unsupported format, we recommend making a new :code:`neo.rawio.BaseRawIO` class (see `example <https://github.com/NeuralEnsemble/python-neo/blob/master/neo/rawio/examplerawio.py#L44>`_).
 Once that is done, the new class can be easily wrapped into SpikeInterface as an extension of the
 :py:class:`~spikeinterface.extractors.neoextractors.neobaseextractors.NeoBaseRecordingExtractor`
 (for :py:class:`~spikeinterface.core.BaseRecording` objects) or
 :py:class:`~spikeinterface.extractors.neoextractors.neobaseextractors.NeoBaseRecordingExtractor`
 (for :py:class:`~spikeinterface.core.BaseSorting` objects) or with a few lines of
-code (e.g., see reader for `SpikeGLX <https://github.com/SpikeInterface/spikeinterface/blob/0.96.1/spikeinterface/extractors/neoextractors/spikeglx.py>`_
-or `Neuralynx <https://github.com/SpikeInterface/spikeinterface/blob/0.96.1/spikeinterface/extractors/neoextractors/neuralynx.py>`_).
+code (e.g., see reader for `SpikeGLX <https://github.com/SpikeInterface/spikeinterface/blob/main/src/spikeinterface/extractors/neoextractors/spikeglx.py>`_
+or `Neuralynx <https://github.com/SpikeInterface/spikeinterface/blob/main/src/spikeinterface/extractors/neoextractors/neuralynx.py>`_).
 
-**NOTE:** implementing a `neo.rawio` Class is not required, but recommended. Several extractors (especially) for :code:`Sorting`
+**NOTE:** implementing a `neo.rawio` class is not required, but recommended. Several extractors (especially) for :code:`Sorting`
 objects are implemented directly in SpikeInterface and inherit from the base classes.
-As examples, see the `CompressedBinaryIblExtractor <https://github.com/SpikeInterface/spikeinterface/blob/0.96.1/spikeinterface/extractors/cbin_ibl.py>`_
-for a :py:class:`~spikeinterface.core.BaseRecording` object, or the `SpykingCircusSortingExtractor <https://github.com/SpikeInterface/spikeinterface/blob/0.96.1/spikeinterface/extractors/spykingcircusextractors.py>`_
+As examples, see the `IblRecordingExtractor <https://github.com/SpikeInterface/spikeinterface/blob/main/src/spikeinterface/extractors/iblextractors.py>`_
+for a :py:class:`~spikeinterface.core.BaseRecording` object, or the `SpykingCircusSortingExtractor <https://github.com/SpikeInterface/spikeinterface/blob/main/src/spikeinterface/extractors/spykingcircusextractors.py>`_
 for a a :py:class:`~spikeinterface.core.BaseSorting` object.
 
 
@@ -309,7 +309,7 @@ Then, you can start creating a new class:
 
     class MySpikeSorter(BaseSorter):
     """
-    Brief description (optional)
+    Brief description
     """
 
     sorter_name = 'myspikesorter'
@@ -367,7 +367,7 @@ Now you can start filling out the required methods:
     def _check_apply_filter_in_params(cls, params):
         return False
 
-        #  optional
+        # optional
         # can be implemented in subclass to check if the filter will be applied
 
 
@@ -388,10 +388,12 @@ Now you can start filling out the required methods:
         return sorting
 
 When your spike sorter class is implemented, you have to add it to the list of available spike sorters in the
-`sorterlist.py`. Then you need to write a test in **tests/test_myspikesorter.py**. In order to be tested, you can
-install the required packages by changing the **pyproject.toml**. Note that MATLAB based tests cannot be run at the moment,
-but we recommend testing the implementation locally.
+`sorterlist.py <https://github.com/SpikeInterface/spikeinterface/blob/main/src/spikeinterface/sorters/sorterlist.py>`_ .
+Then you need to write a test in **tests/test_myspikesorter.py**. In order to be tested, you can
+install the required packages by changing the `pyproject.toml <https://github.com/SpikeInterface/spikeinterface/blob/main/pyproject.toml>`_.
+Note that MATLAB based tests cannot be run at the moment,but we recommend testing the implementation locally.
 
-After this you need to add a block in **doc/sorters_info.rst** to describe your sorter.
+After this you need to add a block in `Install Sorters <https://github.com/SpikeInterface/spikeinterface/blob/main/doc/get_started/install_sorters.rst>`_
+to describe your sorter.
 
 Finally, make a pull request so we can review the code and incorporate into the sorters module of SpikeInterface!

doc/get_started/quickstart.rst

@@ -287,7 +287,7 @@ available parameters are dictionaries and can be accessed with:
      'detect_threshold': 5,
      'freq_max': 5000.0,
      'freq_min': 400.0,
-     'max_threads_per_process': 1,
+     'max_threads_per_worker': 1,
      'mp_context': None,
      'n_jobs': 20,
      'nested_params': None,

doc/how_to/benchmark_with_hybrid_recordings.rst

@@ -2531,9 +2531,8 @@ Although non of the sorters find all units perfectly, ``Kilosort2.5``,
 ``Kilosort4``, and ``SpyKING CIRCUS 2`` all find around 10-12 hybrid
 units with accuracy greater than 80%. ``Kilosort4`` has a better overall
 curve, being able to find almost all units with an accuracy above 50%.
-``Kilosort2.5`` performs well when looking at precision (finding all
-spikes in a hybrid unit), at the cost of lower recall (finding spikes
-when it shouldn’t).
+``Kilosort2.5`` performs well when looking at precision (not finding spikes
+when it shouldn’t), but it has a lower recall (finding all spikes in the ground truth).
 
 In this example, we showed how to:
 

examples/how_to/benchmark_with_hybrid_recordings.py

@@ -276,7 +276,8 @@
 # From the performance plots, we can see that there is no clear "winner", but `Kilosort3` definitely performs worse than the other options.
 #
 # Although non of the sorters find all units perfectly, `Kilosort2.5`, `Kilosort4`, and `SpyKING CIRCUS 2` all find around 10-12 hybrid units with accuracy greater than 80%.
-# `Kilosort4` has a better overall curve, being able to find almost all units with an accuracy above 50%. `Kilosort2.5` performs well when looking at precision (finding all spikes in a hybrid unit), at the cost of lower recall (finding spikes when it shouldn't).
+# `Kilosort4` has a better overall curve, being able to find almost all units with an accuracy above 50%. `Kilosort2.5` performs well when looking at precision (not finding spikes
+# when it shouldn’t), but it has a lower recall (finding all spikes in the ground truth).
 #
 #
 # In this example, we showed how to:

examples/tutorials/curation/plot_1_automated_curation.py

@@ -209,8 +209,8 @@ def calculate_moving_avg(label_df, confidence_label, window_size):
 # trained on real data.
 #
 # For example, the following classifiers are trained on Neuropixels data from 11 mice recorded in
-# V1,SC and ALM: https://huggingface.co/AnoushkaJain3/noise_neural_classifier/ and
-# https://huggingface.co/AnoushkaJain3/sua_mua_classifier/ . One will classify units into
+# V1,SC and ALM: https://huggingface.co/SpikeInterface/UnitRefine_noise_neural_classifier/ and
+# https://huggingface.co/SpikeInterface/UnitRefine_sua_mua_classifier/. One will classify units into
 # `noise` or `not-noise` and the other will classify the `not-noise` units into single
 # unit activity (sua) units and multi-unit activity (mua) units.
 #
@@ -221,8 +221,8 @@ def calculate_moving_avg(label_df, confidence_label, window_size):
 
 # Apply the noise/not-noise model
 noise_neuron_labels = sc.auto_label_units(
-    sorting_analyzer = sorting_analyzer,
-    repo_id = "AnoushkaJain3/noise_neural_classifier",
+    sorting_analyzer=sorting_analyzer,
+    repo_id="SpikeInterface/UnitRefine_noise_neural_classifier",
     trust_model=True,
 )
 
@@ -231,8 +231,8 @@ def calculate_moving_avg(label_df, confidence_label, window_size):
 
 # Apply the sua/mua model
 sua_mua_labels = sc.auto_label_units(
-    sorting_analyzer = analyzer_neural,
-    repo_id = "AnoushkaJain3/sua_mua_classifier",
+    sorting_analyzer=analyzer_neural,
+    repo_id="SpikeInterface/UnitRefine_sua_mua_classifier",
     trust_model=True,
 )
 

pyproject.toml

@@ -7,7 +7,7 @@ authors = [
 ]
 description = "Python toolkit for analysis, visualization, and comparison of spike sorting output"
 readme = "README.md"
-requires-python = ">=3.9,<3.13"  # Only numpy 2.1 supported on python 3.13 for windows. We need to wait for fix on neo
+requires-python = ">=3.9,<3.13"
 classifiers = [
     "Programming Language :: Python :: 3 :: Only",
     "License :: OSI Approved :: MIT License",
@@ -20,11 +20,11 @@ classifiers = [
 
 
 dependencies = [
-    "numpy>=1.20, <2.0",  # 1.20 np.ptp, 1.26 might be necessary for avoiding pickling errors when numpy >2.0
+    "numpy>=1.20",
     "threadpoolctl>=3.0.0",
     "tqdm",
-    "zarr>=2.16,<2.18",
-    "neo>=0.13.0",
+    "zarr>=2.18,<3",
+    "neo>=0.14.0",
     "probeinterface>=0.2.23",
     "packaging",
 ]
@@ -61,8 +61,8 @@ changelog = "https://spikeinterface.readthedocs.io/en/latest/whatisnew.html"
 extractors = [
     "MEArec>=1.8",
     "pynwb>=2.6.0",
-    "hdmf-zarr>=0.5.0",
-    "pyedflib>=0.1.30",
+    "hdmf-zarr>=0.11.0",
+    "pyedflib>=0.1.30,<0.1.39",
     "sonpy;python_version<'3.10'",
     "lxml", # lxml for neuroscope
     "scipy",
@@ -80,7 +80,7 @@ streaming_extractors = [
     "fsspec",
     "aiohttp",
     "requests",
-    "hdmf-zarr>=0.5.0",
+    "hdmf-zarr>=0.11.0",
     "remfile",
     "s3fs"
 ]

readthedocs.yml

@@ -1,5 +1,9 @@
 version: 2
 
+sphinx:
+  # Path to your Sphinx configuration file.
+  configuration: doc/conf.py
+
 build:
   os: ubuntu-22.04
   tools:

src/spikeinterface/benchmark/benchmark_base.py

@@ -11,7 +11,7 @@
 
 from spikeinterface.core import SortingAnalyzer
 
-from spikeinterface import load_extractor, create_sorting_analyzer, load_sorting_analyzer
+from spikeinterface import load, create_sorting_analyzer, load_sorting_analyzer
 from spikeinterface.widgets import get_some_colors
 
 
@@ -150,13 +150,13 @@ def scan_folder(self):
             analyzer = load_sorting_analyzer(folder)
             self.analyzers[key] = analyzer
             # the sorting is in memory here we take the saved one because comparisons need to pickle it later
-            sorting = load_extractor(analyzer.folder / "sorting")
+            sorting = load(analyzer.folder / "sorting")
             self.datasets[key] = analyzer.recording, sorting
 
         # for rec_file in (self.folder / "datasets" / "recordings").glob("*.pickle"):
         #     key = rec_file.stem
-        #     rec = load_extractor(rec_file)
-        #     gt_sorting = load_extractor(self.folder / f"datasets" / "gt_sortings" / key)
+        #     rec = load(rec_file)
+        #     gt_sorting = load(self.folder / f"datasets" / "gt_sortings" / key)
         #     self.datasets[key] = (rec, gt_sorting)
 
         with open(self.folder / "cases.pickle", "rb") as f:
@@ -428,7 +428,7 @@ def load_folder(cls, folder):
             elif format == "sorting":
                 from spikeinterface.core import load_extractor
 
-                result[k] = load_extractor(folder / k)
+                result[k] = load(folder / k)
             elif format == "Motion":
                 from spikeinterface.sortingcomponents.motion import Motion