Merge branch 'main' into rt-sort

Author: zm711

.github/scripts/test_kilosort4_ci.py

@@ -20,7 +20,7 @@
 
 import pytest
 import copy
-from typing import Any
+from packaging.version import parse
 from inspect import signature
 
 import numpy as np
@@ -84,8 +84,6 @@
     "duplicate_spike_ms": 0.3,
 }
 
-PARAMS_TO_TEST = list(PARAMS_TO_TEST_DICT.keys())
-
 PARAMETERS_NOT_AFFECTING_RESULTS = [
     "artifact_threshold",
     "ccg_threshold",
@@ -95,13 +93,24 @@
     "duplicate_spike_ms",  # this is because ground-truth spikes don't have violations
 ]
 
-# THIS IS A PLACEHOLDER FOR FUTURE PARAMS TO TEST
-# if parse(version("kilosort")) >= parse("4.0.X"):
-#     PARAMS_TO_TEST_DICT.update(
-#         [
-#             {"new_param": new_value},
-#         ]
-#     )
+
+# Add/Remove version specific parameters
+if parse(kilosort.__version__) >= parse("4.0.22"):
+    PARAMS_TO_TEST_DICT.update(
+        {"position_limit": 50}
+    )
+    # Position limit only affects computing spike locations after sorting
+    PARAMETERS_NOT_AFFECTING_RESULTS.append("position_limit")
+
+if parse(kilosort.__version__) >= parse("4.0.24"):
+    PARAMS_TO_TEST_DICT.update(
+        {"max_peels": 200},
+    )
+    # max_peels is not affecting the results in this short dataset
+    PARAMETERS_NOT_AFFECTING_RESULTS.append("max_peels")
+
+
+PARAMS_TO_TEST = list(PARAMS_TO_TEST_DICT.keys())
 
 
 class TestKilosort4Long:

doc/releases/0.102.1.rst

@@ -0,0 +1,33 @@
+.. _release0.102.1:
+
+SpikeInterface 0.102.1 release notes
+------------------------------------
+
+19th February 2025
+
+Minor release with bug fixes
+
+core:
+
+* Add parents in HTML representation and always print class name (#3700)
+* Add `SortingAnalyzer.set_sorting_property()/get_sorting_property()` functions (#3694)
+* Add `_parent` to `select_segment` classes (#3692)
+* Fix `chunk_size_limit` in `get_random_recording_slices` (#3691)
+* Fix bug in `super_zarr_open` (#3686)
+* Fix `si.load` for `WaveformExtarctor` (#3680)
+
+sorters:
+
+* Add `BaseSorter._dynamic_params()` to be to retrieve parameters dynamically (#3697)
+
+continuous integration:
+
+* Add `torch` to test installation (#3706)
+
+testing:
+
+* Test Python 3.13 (#3683)
+
+Contributors:
+
+* @alejoe91

doc/whatisnew.rst

@@ -8,6 +8,7 @@ Release notes
 .. toctree::
   :maxdepth: 1
 
+  releases/0.102.1.rst
   releases/0.102.0.rst
   releases/0.101.2.rst
   releases/0.101.1.rst
@@ -46,6 +47,21 @@ Release notes
   releases/0.9.1.rst
 
 
+Version 0.102.1
+===============
+
+* Minor release with bug fixes
+
+Version 0.102.0
+===============
+
+* Added auto-label functions in curation module (#2918)
+* Refactored and improved auto-merge functions in curation module (#3435, #3601)
+* Added `spikeinterface.load()` function to load any SpikeInterface object (#3613, #3651)
+* Improved handling of time in base recording (#3509, #3623)
+* Multi-segment handling of motion interpolation (#3659)
+* Support for Numpy 2.0 and Zarr<3.0 (#3481,#3598)
+
 Version 0.101.2
 ===============
 

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spikeinterface"
-version = "0.102.1"
+version = "0.102.2"
 authors = [
   { name="Alessio Buccino", email="alessiop.buccino@gmail.com" },
   { name="Samuel Garcia", email="sam.garcia.die@gmail.com" },
@@ -159,12 +159,16 @@ test = [
     "s3fs",
 
     # tridesclous
-    "numba",
+    "numba<0.61.0;python_version<'3.13'",
+    "numba>=0.61.0;python_version>='3.13'",
     "hdbscan>=0.8.33",  # Previous version had a broken wheel
 
     # for sortingview backend
     "sortingview",
 
+    # for motion and sortingcomponents
+    "torch",
+
     # curation
     "skops",
     "huggingface_hub",

src/spikeinterface/benchmark/benchmark_base.py

@@ -228,11 +228,11 @@ def run(self, case_keys=None, keep=True, verbose=False, **job_kwargs):
             benchmark.result["run_time"] = float(t1 - t0)
             benchmark.save_main(bench_folder)
 
-    def set_colors(self, colors=None, map_name="tab20"):
+    def set_colors(self, colors=None, map_name="tab10"):
         if colors is None:
             case_keys = list(self.cases.keys())
             self.colors = get_some_colors(
-                case_keys, map_name=map_name, color_engine="matplotlib", shuffle=False, margin=0
+                case_keys, map_name=map_name, color_engine="matplotlib", shuffle=False, margin=0, resample=False
             )
         else:
             self.colors = colors

src/spikeinterface/benchmark/benchmark_matching.py

@@ -138,7 +138,7 @@ def plot_unit_counts(self, case_keys=None, figsize=None):
 
         plot_study_unit_counts(self, case_keys, figsize=figsize)
 
-    def plot_unit_losses(self, before, after, metric=["precision"], figsize=None):
+    def plot_unit_losses(self, before, after, metric=["accuracy"], figsize=None):
         import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=1, nrows=len(metric), figsize=figsize, squeeze=False)

src/spikeinterface/benchmark/benchmark_peak_detection.py

@@ -15,6 +15,7 @@
 from .benchmark_base import Benchmark, BenchmarkStudy
 from spikeinterface.core.basesorting import minimum_spike_dtype
 from spikeinterface.core.sortinganalyzer import create_sorting_analyzer
+from .benchmark_plot_tools import fit_sigmoid, sigmoid
 
 
 class PeakDetectionBenchmark(Benchmark):
@@ -126,7 +127,7 @@ def create_benchmark(self, key):
     def plot_agreements_by_channels(self, case_keys=None, figsize=(15, 15)):
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -138,7 +139,7 @@ def plot_agreements_by_channels(self, case_keys=None, figsize=(15, 15)):
     def plot_agreements_by_units(self, case_keys=None, figsize=(15, 15)):
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -151,32 +152,39 @@ def plot_performances_vs_snr(self, case_keys=None, figsize=(15, 15), detect_thre
         if case_keys is None:
             case_keys = list(self.cases.keys())
 
+        import matplotlib.pyplot as plt
+
         fig, axs = plt.subplots(ncols=1, nrows=3, figsize=figsize)
 
         for count, k in enumerate(("accuracy", "recall", "precision")):
 
             ax = axs[count]
             for key in case_keys:
+                color = self.get_colors()[key]
                 label = self.cases[key]["label"]
 
                 analyzer = self.get_sorting_analyzer(key)
                 metrics = analyzer.get_extension("quality_metrics").get_data()
                 x = metrics["snr"].values
                 y = self.get_result(key)["sliced_gt_comparison"].get_performance()[k].values
-                ax.scatter(x, y, marker=".", label=label)
+                ax.scatter(x, y, marker=".", label=label, color=color)
                 ax.set_title(k)
                 if detect_threshold is not None:
                     ymin, ymax = ax.get_ylim()
                     ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
 
+                popt = fit_sigmoid(x, y, p0=None)
+                xfit = np.linspace(0, max(metrics["snr"].values), 100)
+                ax.plot(xfit, sigmoid(xfit, *popt), color=color)
+
             if count == 2:
                 ax.legend()
 
     def plot_detected_amplitudes(self, case_keys=None, figsize=(15, 5), detect_threshold=None):
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -201,7 +209,7 @@ def plot_deltas_per_cells(self, case_keys=None, figsize=(15, 5)):
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
         for count, key in enumerate(case_keys):
@@ -222,12 +230,12 @@ def plot_template_similarities(self, case_keys=None, metric="l2", figsize=(15, 5
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, ax = plt.subplots(ncols=1, nrows=1, figsize=figsize, squeeze=True)
         for key in case_keys:
 
-            import sklearn
+            import sklearn.metrics
 
             gt_templates = self.get_result(key)["gt_templates"]
             found_templates = self.get_result(key)["templates"]
@@ -244,14 +252,22 @@ def plot_template_similarities(self, case_keys=None, metric="l2", figsize=(15, 5
                 else:
                     distances[i] = sklearn.metrics.pairwise_distances(a[None, :], b[None, :], metric)[0, 0]
 
+            color = self.get_colors()[key]
+
             label = self.cases[key]["label"]
             analyzer = self.get_sorting_analyzer(key)
             metrics = analyzer.get_extension("quality_metrics").get_data()
             x = metrics["snr"].values
-            ax.scatter(x, distances, marker=".", label=label)
-            if detect_threshold is not None:
-                ymin, ymax = ax.get_ylim()
-                ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
+            y = distances
+            ax.scatter(x, y, marker=".", label=label, color=color)
+
+            popt = fit_sigmoid(x, y, p0=None)
+            xfit = np.linspace(0, max(metrics["snr"].values), 100)
+            ax.plot(xfit, sigmoid(xfit, *popt), color=color)
+
+        if detect_threshold is not None:
+            ymin, ymax = ax.get_ylim()
+            ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
 
         ax.legend()
         ax.set_xlabel("snr")

src/spikeinterface/benchmark/benchmark_plot_tools.py

@@ -1,4 +1,5 @@
 import numpy as np
+import warnings
 
 
 def _simpleaxis(ax):
@@ -193,7 +194,7 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None):
         case_keys = list(study.cases.keys())
 
     num_axes = len(case_keys)
-    fig, axs = plt.subplots(ncols=num_axes)
+    fig, axs = plt.subplots(ncols=num_axes, squeeze=False)
 
     for count, key in enumerate(case_keys):
         ax = axs.flatten()[count]
@@ -214,7 +215,9 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None):
         ax.set_xticks([])
 
 
-def plot_performances_vs_snr(study, case_keys=None, figsize=None, metrics=["accuracy", "recall", "precision"]):
+def plot_performances_vs_snr(
+    study, case_keys=None, figsize=None, metrics=["accuracy", "recall", "precision"], snr_dataset_reference=None
+):
     import matplotlib.pyplot as plt
 
     if case_keys is None:
@@ -228,7 +231,13 @@ def plot_performances_vs_snr(study, case_keys=None, figsize=None, metrics=["accu
         for key in case_keys:
             label = study.cases[key]["label"]
 
-            analyzer = study.get_sorting_analyzer(key)
+            if snr_dataset_reference is None:
+                # use the SNR of each dataset
+                analyzer = study.get_sorting_analyzer(key)
+            else:
+                # use the same SNR from a reference dataset
+                analyzer = study.get_sorting_analyzer(dataset_key=snr_dataset_reference)
+
             metrics = analyzer.get_extension("quality_metrics").get_data()
             x = metrics["snr"].values
             y = study.get_result(key)["gt_comparison"].get_performance()[k].values
@@ -303,3 +312,16 @@ def plot_performances_comparison(
     ax.legend(handles=patches)
     fig.tight_layout()
     return fig
+
+
+def sigmoid(x, x0, k, b):
+    with warnings.catch_warnings(action="ignore"):
+        out = (1 / (1 + np.exp(-k * (x - x0)))) + b
+    return out
+
+
+def fit_sigmoid(xdata, ydata, p0=None):
+    from scipy.optimize import curve_fit
+
+    popt, pcov = curve_fit(sigmoid, xdata, ydata, p0)
+    return popt