Merge branch 'main' into add-BaseRasterWidget

Author: chrishalcrow

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "spikeinterface"
-version = "0.102.1"
+version = "0.102.2"
 authors = [
   { name="Alessio Buccino", email="alessiop.buccino@gmail.com" },
   { name="Samuel Garcia", email="sam.garcia.die@gmail.com" },
@@ -127,16 +127,16 @@ test_core = [
 
     # for github test : probeinterface and neo from master
     # for release we need pypi, so this need to be commented
-    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 test_extractors = [
     # Functions to download data in neo test suite
     "pooch>=1.8.2",
     "datalad>=1.0.2",
-    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 test_preprocessing = [
@@ -175,8 +175,8 @@ test = [
 
     # for github test : probeinterface and neo from master
     # for release we need pypi, so this need to be commented
-    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 docs = [
@@ -201,8 +201,8 @@ docs = [
     "datalad>=1.0.2",
 
     # for release we need pypi, so this needs to be commented
-    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",  # We always build from the latest version
-    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",  # We always build from the latest version
+    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",  # We always build from the latest version
+    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",  # We always build from the latest version
 
 ]
 

src/spikeinterface/__init__.py

@@ -27,5 +27,5 @@
 # This flag must be set to False for release
 # This avoids using versioning that contains ".dev0" (and this is a better choice)
 # This is mainly useful when using run_sorter in a container and spikeinterface install
-# DEV_MODE = True
-DEV_MODE = False
+DEV_MODE = True
+# DEV_MODE = False

src/spikeinterface/benchmark/benchmark_base.py

@@ -228,11 +228,11 @@ def run(self, case_keys=None, keep=True, verbose=False, **job_kwargs):
             benchmark.result["run_time"] = float(t1 - t0)
             benchmark.save_main(bench_folder)
 
-    def set_colors(self, colors=None, map_name="tab20"):
+    def set_colors(self, colors=None, map_name="tab10"):
         if colors is None:
             case_keys = list(self.cases.keys())
             self.colors = get_some_colors(
-                case_keys, map_name=map_name, color_engine="matplotlib", shuffle=False, margin=0
+                case_keys, map_name=map_name, color_engine="matplotlib", shuffle=False, margin=0, resample=False
             )
         else:
             self.colors = colors

src/spikeinterface/benchmark/benchmark_matching.py

@@ -138,7 +138,7 @@ def plot_unit_counts(self, case_keys=None, figsize=None):
 
         plot_study_unit_counts(self, case_keys, figsize=figsize)
 
-    def plot_unit_losses(self, before, after, metric=["precision"], figsize=None):
+    def plot_unit_losses(self, before, after, metric=["accuracy"], figsize=None):
         import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=1, nrows=len(metric), figsize=figsize, squeeze=False)

src/spikeinterface/benchmark/benchmark_peak_detection.py

@@ -15,6 +15,7 @@
 from .benchmark_base import Benchmark, BenchmarkStudy
 from spikeinterface.core.basesorting import minimum_spike_dtype
 from spikeinterface.core.sortinganalyzer import create_sorting_analyzer
+from .benchmark_plot_tools import fit_sigmoid, sigmoid
 
 
 class PeakDetectionBenchmark(Benchmark):
@@ -126,7 +127,7 @@ def create_benchmark(self, key):
     def plot_agreements_by_channels(self, case_keys=None, figsize=(15, 15)):
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -138,7 +139,7 @@ def plot_agreements_by_channels(self, case_keys=None, figsize=(15, 15)):
     def plot_agreements_by_units(self, case_keys=None, figsize=(15, 15)):
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -151,32 +152,39 @@ def plot_performances_vs_snr(self, case_keys=None, figsize=(15, 15), detect_thre
         if case_keys is None:
             case_keys = list(self.cases.keys())
 
+        import matplotlib.pyplot as plt
+
         fig, axs = plt.subplots(ncols=1, nrows=3, figsize=figsize)
 
         for count, k in enumerate(("accuracy", "recall", "precision")):
 
             ax = axs[count]
             for key in case_keys:
+                color = self.get_colors()[key]
                 label = self.cases[key]["label"]
 
                 analyzer = self.get_sorting_analyzer(key)
                 metrics = analyzer.get_extension("quality_metrics").get_data()
                 x = metrics["snr"].values
                 y = self.get_result(key)["sliced_gt_comparison"].get_performance()[k].values
-                ax.scatter(x, y, marker=".", label=label)
+                ax.scatter(x, y, marker=".", label=label, color=color)
                 ax.set_title(k)
                 if detect_threshold is not None:
                     ymin, ymax = ax.get_ylim()
                     ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
 
+                popt = fit_sigmoid(x, y, p0=None)
+                xfit = np.linspace(0, max(metrics["snr"].values), 100)
+                ax.plot(xfit, sigmoid(xfit, *popt), color=color)
+
             if count == 2:
                 ax.legend()
 
     def plot_detected_amplitudes(self, case_keys=None, figsize=(15, 5), detect_threshold=None):
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
 
@@ -201,7 +209,7 @@ def plot_deltas_per_cells(self, case_keys=None, figsize=(15, 5)):
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, axs = plt.subplots(ncols=len(case_keys), nrows=1, figsize=figsize, squeeze=False)
         for count, key in enumerate(case_keys):
@@ -222,12 +230,12 @@ def plot_template_similarities(self, case_keys=None, metric="l2", figsize=(15, 5
 
         if case_keys is None:
             case_keys = list(self.cases.keys())
-        import pylab as plt
+        import matplotlib.pyplot as plt
 
         fig, ax = plt.subplots(ncols=1, nrows=1, figsize=figsize, squeeze=True)
         for key in case_keys:
 
-            import sklearn
+            import sklearn.metrics
 
             gt_templates = self.get_result(key)["gt_templates"]
             found_templates = self.get_result(key)["templates"]
@@ -244,14 +252,22 @@ def plot_template_similarities(self, case_keys=None, metric="l2", figsize=(15, 5
                 else:
                     distances[i] = sklearn.metrics.pairwise_distances(a[None, :], b[None, :], metric)[0, 0]
 
+            color = self.get_colors()[key]
+
             label = self.cases[key]["label"]
             analyzer = self.get_sorting_analyzer(key)
             metrics = analyzer.get_extension("quality_metrics").get_data()
             x = metrics["snr"].values
-            ax.scatter(x, distances, marker=".", label=label)
-            if detect_threshold is not None:
-                ymin, ymax = ax.get_ylim()
-                ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
+            y = distances
+            ax.scatter(x, y, marker=".", label=label, color=color)
+
+            popt = fit_sigmoid(x, y, p0=None)
+            xfit = np.linspace(0, max(metrics["snr"].values), 100)
+            ax.plot(xfit, sigmoid(xfit, *popt), color=color)
+
+        if detect_threshold is not None:
+            ymin, ymax = ax.get_ylim()
+            ax.plot([detect_threshold, detect_threshold], [ymin, ymax], "k--")
 
         ax.legend()
         ax.set_xlabel("snr")

src/spikeinterface/benchmark/benchmark_plot_tools.py

@@ -1,4 +1,5 @@
 import numpy as np
+import warnings
 
 
 def _simpleaxis(ax):
@@ -193,7 +194,7 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None):
         case_keys = list(study.cases.keys())
 
     num_axes = len(case_keys)
-    fig, axs = plt.subplots(ncols=num_axes)
+    fig, axs = plt.subplots(ncols=num_axes, squeeze=False)
 
     for count, key in enumerate(case_keys):
         ax = axs.flatten()[count]
@@ -214,7 +215,9 @@ def plot_agreement_matrix(study, ordered=True, case_keys=None):
         ax.set_xticks([])
 
 
-def plot_performances_vs_snr(study, case_keys=None, figsize=None, metrics=["accuracy", "recall", "precision"]):
+def plot_performances_vs_snr(
+    study, case_keys=None, figsize=None, metrics=["accuracy", "recall", "precision"], snr_dataset_reference=None
+):
     import matplotlib.pyplot as plt
 
     if case_keys is None:
@@ -228,7 +231,13 @@ def plot_performances_vs_snr(study, case_keys=None, figsize=None, metrics=["accu
         for key in case_keys:
             label = study.cases[key]["label"]
 
-            analyzer = study.get_sorting_analyzer(key)
+            if snr_dataset_reference is None:
+                # use the SNR of each dataset
+                analyzer = study.get_sorting_analyzer(key)
+            else:
+                # use the same SNR from a reference dataset
+                analyzer = study.get_sorting_analyzer(dataset_key=snr_dataset_reference)
+
             metrics = analyzer.get_extension("quality_metrics").get_data()
             x = metrics["snr"].values
             y = study.get_result(key)["gt_comparison"].get_performance()[k].values
@@ -303,3 +312,16 @@ def plot_performances_comparison(
     ax.legend(handles=patches)
     fig.tight_layout()
     return fig
+
+
+def sigmoid(x, x0, k, b):
+    with warnings.catch_warnings(action="ignore"):
+        out = (1 / (1 + np.exp(-k * (x - x0)))) + b
+    return out
+
+
+def fit_sigmoid(xdata, ydata, p0=None):
+    from scipy.optimize import curve_fit
+
+    popt, pcov = curve_fit(sigmoid, xdata, ydata, p0)
+    return popt

src/spikeinterface/core/binaryrecordingextractor.py

@@ -23,8 +23,6 @@ class BinaryRecordingExtractor(BaseRecording):
         The sampling frequency
     num_channels : int
         Number of channels
-    num_chan : int [deprecated, use num_channels instead, will be removed as early as v0.100.0]
-        Number of channels
     dtype : str or dtype
         The dtype of the binary file
     time_axis : int, default: 0
@@ -57,23 +55,15 @@ def __init__(
         file_paths,
         sampling_frequency,
         dtype,
-        num_channels=None,
+        num_channels: int,
         t_starts=None,
         channel_ids=None,
         time_axis=0,
         file_offset=0,
         gain_to_uV=None,
         offset_to_uV=None,
         is_filtered=None,
-        num_chan=None,
     ):
-        # This assigns num_channels if num_channels is not None, otherwise num_chan is assigned
-        # num_chan needs to be be kept for backward compatibility but should not be used by the
-        # end user
-        num_channels = num_channels or num_chan
-        assert num_channels is not None, "You must provide num_channels or num_chan"
-        if num_chan is not None:
-            warnings.warn("`num_chan` is to be deprecated as of version 0.100, please use `num_channels` instead")
 
         if channel_ids is None:
             channel_ids = list(range(num_channels))

src/spikeinterface/core/motion.py

@@ -245,6 +245,14 @@ def copy(self):
             interpolation_method=self.interpolation_method,
         )
 
+    def get_boundaries(self):
+        max_ = -np.inf
+        min_ = np.inf
+        for segment_index, displacement_array in enumerate(self.displacement):
+            min_ = min(min_, np.min(displacement_array))
+            max_ = max(max_, np.max(displacement_array))
+        return min_, max_
+
 
 def ensure_time_bins(time_bin_centers_s=None, time_bin_edges_s=None):
     """Ensure that both bin edges and bin centers are present

src/spikeinterface/core/node_pipeline.py

@@ -29,6 +29,11 @@
 
 
 class PipelineNode:
+
+    # If False (general case) then compute(traces_chunk, *node_input_args)
+    # If True then compute(traces_chunk, start_frame, end_frame, segment_index, max_margin, *node_input_args)
+    _compute_has_extended_signature = False
+
     def __init__(
         self,
         recording: BaseRecording,
@@ -684,7 +689,13 @@ def _compute_peak_pipeline_chunk(segment_index, start_frame, end_frame, worker_c
                 node_output = node.compute(traces_chunk, start_frame, end_frame, segment_index, max_margin, peak_slice)
             else:
                 # TODO later when in master: change the signature of all nodes (or maybe not!)
-                node_output = node.compute(traces_chunk, *node_input_args)
+                if not node._compute_has_extended_signature:
+                    node_output = node.compute(traces_chunk, *node_input_args)
+                else:
+                    node_output = node.compute(
+                        traces_chunk, start_frame, end_frame, segment_index, max_margin, *node_input_args
+                    )
+
             pipeline_outputs[node] = node_output
 
             if skip_after_n_peaks_per_worker is not None and isinstance(node, PeakSource):

src/spikeinterface/core/tests/test_binaryrecordingextractor.py

@@ -93,10 +93,8 @@ def test_sequential_reading_of_small_traces(folder_with_binary_files):
     dtype = "float32"
 
     file_paths = [folder / "traces_cached_seg0.raw"]
-    # `num_chan` is kept for backward compatibility so including it at least one test
-    # run is good to ensure that it is appropriately accepted as an argument
     recording = BinaryRecordingExtractor(
-        num_chan=num_channels,
+        num_channels=num_channels,
         file_paths=file_paths,
         sampling_frequency=sampling_frequency,
         dtype=dtype,