Merge branch 'main' into node_pipeline_skip_no_peaks

Author: samuelgarcia

.github/scripts/determine_testing_environment.py

@@ -31,6 +31,7 @@
 sortingcomponents_changed = False
 generation_changed = False
 stream_extractors_changed = False
+github_actions_changed = False
 
 
 for changed_file in changed_files_in_the_pull_request_paths:
@@ -78,9 +79,12 @@
             sorters_internal_changed = True
         else:
             sorters_changed = True
+    elif ".github" in changed_file.parts:
+        if "workflows" in changed_file.parts:
+            github_actions_changed = True
 
 
-run_everything = core_changed or pyproject_toml_changed or neobaseextractor_changed
+run_everything = core_changed or pyproject_toml_changed or neobaseextractor_changed or github_actions_changed
 run_generation_tests = run_everything or generation_changed
 run_extractor_tests = run_everything or extractors_changed or plexon2_changed
 run_preprocessing_tests = run_everything or preprocessing_changed
@@ -96,7 +100,7 @@
 run_sorters_test = run_everything or sorters_changed
 run_internal_sorters_test = run_everything or run_sortingcomponents_tests or sorters_internal_changed
 
-run_streaming_extractors_test = stream_extractors_changed
+run_streaming_extractors_test = stream_extractors_changed or github_actions_changed
 
 install_plexon_dependencies = plexon2_changed
 

.github/workflows/all-tests.yml

@@ -12,6 +12,7 @@ on:
 env:
   KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
   KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
+  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
 
 concurrency:  # Cancel previous workflows on the same pull request
   group: ${{ github.workflow }}-${{ github.ref }}
@@ -25,7 +26,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.9", "3.12"]  # Lower and higher versions we support
-        os: [macos-13, windows-latest, ubuntu-latest]
+        os: [macos-latest, windows-latest, ubuntu-latest]
     steps:
       - uses: actions/checkout@v4
       - name: Setup Python ${{ matrix.python-version }}

.github/workflows/full-test-with-codecov.yml

@@ -8,6 +8,7 @@ on:
 env:
   KACHERY_CLOUD_CLIENT_ID: ${{ secrets.KACHERY_CLOUD_CLIENT_ID }}
   KACHERY_CLOUD_PRIVATE_KEY: ${{ secrets.KACHERY_CLOUD_PRIVATE_KEY }}
+  KACHERY_ZONE: ${{ secrets.KACHERY_ZONE }}
 
 jobs:
   full-tests-with-codecov:

doc/development/development.rst

@@ -189,9 +189,17 @@ so that the user knows what the options are.
 Miscelleaneous Stylistic Conventions
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-#. Avoid using abreviations in variable names (e.g., use :code:`recording` instead of :code:`rec`). It is especially important to avoid single letter variables.
+#. Avoid using abbreviations in variable names (e.g. use :code:`recording` instead of :code:`rec`). It is especially important to avoid single letter variables.
 #. Use index as singular and indices for plural following the NumPy convention. Avoid idx or indexes. Plus, id and ids are reserved for identifiers (i.e. channel_ids)
 #. We use file_path and folder_path (instead of file_name and folder_name) for clarity.
+#. For creating headers to divide sections of code we use the following convention (see issue `#3019 <https://github.com/SpikeInterface/spikeinterface/issues/3019>`_):
+
+
+.. code:: python
+
+    #########################################
+    # A header
+    #########################################
 
 
 How to build the documentation

doc/releases/0.101.0.rst

@@ -3,7 +3,7 @@
 SpikeInterface 0.101.0 release notes
 ------------------------------------
 
-Estimated: 19th July 2024
+19th July 2024
 
 Main changes:
 

doc/releases/0.101.1.rst

@@ -0,0 +1,144 @@
+.. _release0.101.1:
+
+SpikeInterface 0.101.1 release notes
+------------------------------------
+
+16th September 2024
+
+Main changes:
+
+* Enabled support for consolidated Zarr-backend for `SortingAnalyzer`, including cloud support (#3314, #3318, #3349, #3351)
+* Improved support for Kilosort4 **ONLY VERSIONS >= 4.0.16** (#3339, #3276)
+* Skip recomputation of quality and template metrics if already computed (#3292)
+* Improved `estimate_sparsity` with new `amplitude` method and deprecated `from_ptp` option (#3369)
+* Dropped support for Python<3.9 (#3267)
+
+core:
+
+* Update the method of creating an empty file with right size when saving binary files (#3408)
+* Refactor pandas save load and convert dtypes (#3412)
+* Check run info completed only if it exists (back-compatibility) (#3407)
+* Fix argument spelling in check for binary compatibility (#3409)
+* Fix proposal for channel location when probegroup (#3392)
+* Fix time handling test memory (#3379)
+* Add `BaseRecording.reset_times()` function (#3363, #3380, #3391)
+* Extend `estimate_sparsity` methods and update `from_ptp`` (#3369)
+* Add `load_sorting_analyzer_or_waveforms` function (#3352)
+* Fix zarr folder suffix handling (#3349)
+* Analyzer extension exit status (#3347)
+* Lazy loading of zarr timestamps (#3318)
+* Enable cloud-loading for analyzer Zarr (#3314, #3351, #3378)
+* Refactor `set_property` in base (#3287)
+* Job kwargs fix (#3259)
+* Add `is_filtered` to annotations in `binary.json` (#3245)
+* Add check for None in `NoiseGeneratorRecordingSegment`` get_traces() (#3230)
+
+extractors:
+
+* Load phy channel_group as group (#3368)
+* "quality" property to be read as string instead of object in `BasePhyKilosortSortingExtractor` (#3365)
+
+preprocessing:
+
+* Auto-cast recording to float prior to interpolation (#3415)
+* Update doc handle drift  + better preset (#3232)
+* Add causal filtering to filter.py (#3172)
+
+sorters:
+
+* Updates to kilosort 4: version >= 4.0.16, `bad_channels`, `clear_cache`, `use_binary_file` (#3339)
+* Download apptainer images without docker client (#3335)
+* Expose save preprocessing in ks4 (#3276)
+* Fix KS2/2.5/3 skip_kilosort_preprocessing (#3265)
+* HS: Added lowpass parameter, fixed verbose option (#3262)
+* Now exclusive support for HS v0.4 (Lightning) (#3210)
+
+postprocessing:
+
+* Add extra protection for template metrics (#3364)
+* Protect median against nans in get_prototype_spike (#3270)
+* Fix docstring and error for spike_amplitudes (#3269)
+
+qualitymetrics:
+
+* Do not delete quality and template metrics on recompute (#3292)
+* Refactor quality metrics tests to use fixture (#3249)
+
+
+curation:
+
+* Clean-up identity merges in `get_potential_auto_merges` (#3346)
+* Fix sortingview curation no merge case  (#3309)
+* Start apply_curation() (#3208)
+
+widgets:
+
+* Fix metrics widgets for convert_dtypes (#3417)
+* Fix plot motion for multi-segment (#3414)
+* Sortingview: only round float properties (#3406)
+* Fix widgets tests and add test on unit_table_properties (#3354)
+* Allow quality and template metrics in sortingview's unit table (#3299)
+* Add subwidget parameters for UnitSummaryWidget (#3242)
+* Fix `ipympl`/`widget` backend check (#3238)
+
+generators:
+
+* Handle case where channel count changes from probeA to probeB (#3237)
+
+sortingcomponents:
+
+* Update doc handle drift  + better preset (#3232)
+* Make `InterpolateMotionRecording`` not JSON-serializable (#3341)
+
+documentation:
+
+* Clarify meaning of `delta_time` in  `compare_sorter_to_ground_truth` (#3360)
+* Added sphinxcontrib-jquery (#3307)
+* Adding return type annotations (#3304)
+* More docstring updates for multiple modules (#3298)
+* Fix sampling frequency repr (#3294)
+* Proposal for adding Examples to docstrings (#3279)
+* More numpydoc fixes (#3275)
+* Fix docstring and error for spike_amplitudes (#3269)
+* Fix postprocessing docs (#3268)
+* Fix name of principal_components ext in qm docs (take 2) (#3261)
+* Update doc handle drift  + better preset (#3232)
+* Add `int` type to `num_samples` on `InjectTemplatesRecording`. (#3229)
+
+continuous integration:
+
+* Fix streaming extractor condition in the CI (#3362)
+
+packaging:
+
+* Minor typing fixes (#3374)
+* Drop python 3.8 in pyproject.toml (#3267)
+
+testing:
+
+* Fix time handling test memory (#3379)
+* Fix streaming extractor condition in the CI (#3362)
+* Test IBL skip when the setting up the one client fails (#3289)
+* Refactor `set_property` in base (#3287)
+* Refactor quality metrics tests to use fixture (#3249)
+* Add kilosort4 wrapper tests (#3085)
+* Test IBL skip when the setting up the one client fails (#3289)
+* Add kilosort4 wrapper tests (#3085)
+
+Contributors:
+
+* @Djoels
+* @JoeZiminski
+* @JuanPimientoCaicedo
+* @alejoe91
+* @chrishalcrow
+* @cwindolf
+* @florian6973
+* @h-mayorquin
+* @jiumao2
+* @jonahpearl
+* @mhhennig
+* @rkim48
+* @samuelgarcia
+* @tabedzki
+* @zm711

doc/scripts/auto-release-notes.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 if [ $# -eq 0 ]; then
-    echo "Usage: $0 START_DATE END_DATE [LABEL] [BRANCH1,BRANCH2] [LIMIT]"
+    echo "Usage: $0 START_DATE (format: YEAR-MM-DD) END_DATE [LABEL] [BRANCH1,BRANCH2] [LIMIT]"
     exit 1
 fi
 

doc/whatisnew.rst

@@ -8,6 +8,7 @@ Release notes
 .. toctree::
   :maxdepth: 1
 
+  releases/0.101.1.rst
   releases/0.101.0.rst
   releases/0.100.8.rst
   releases/0.100.7.rst
@@ -43,6 +44,16 @@ Release notes
   releases/0.9.1.rst
 
 
+Version 0.101.1
+===============
+
+* Enabled support for consolidated Zarr-backend for `SortingAnalyzer`, including cloud support (#3314, #3318, #3349, #3351)
+* Improved support for Kilosort4 **ONLY VERSIONS >= 4.0.16** (#3339, #3276)
+* Skip recomputation of quality and template metrics if already computed (#3292)
+* Improved `estimate_sparsity` with new `amplitude` method and deprecated `from_ptp` option (#3369)
+* Dropped support for Python<3.9 (#3267)
+
+
 Version 0.101.0
 ===============
 

pyproject.toml

@@ -94,7 +94,6 @@ preprocessing = [
 full = [
     "h5py",
     "pandas",
-    "xarray",
     "scipy",
     "scikit-learn",
     "networkx",
@@ -125,16 +124,16 @@ test_core = [
 
     # for github test : probeinterface and neo from master
     # for release we need pypi, so this need to be commented
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 test_extractors = [
     # Functions to download data in neo test suite
     "pooch>=1.8.2",
     "datalad>=1.0.2",
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 test_preprocessing = [
@@ -148,7 +147,6 @@ test = [
     "pytest-dependency",
     "pytest-cov",
 
-    "xarray",
     "huggingface_hub",
 
     # preprocessing
@@ -175,8 +173,8 @@ test = [
 
     # for github test : probeinterface and neo from master
     # for release we need pypi, so this need to be commented
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
-    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
+    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",
+    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",
 ]
 
 docs = [
@@ -193,15 +191,14 @@ docs = [
     "pandas", # in the modules gallery comparison tutorial
     "hdbscan>=0.8.33",   # For sorters spykingcircus2 + tridesclous
     "numba", # For many postprocessing functions
-    "xarray", # For use of SortingAnalyzer zarr format
     "networkx",
     # Download data
     "pooch>=1.8.2",
     "datalad>=1.0.2",
 
     # for release we need pypi, so this needs to be commented
-    "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",  # We always build from the latest version
-    "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",  # We always build from the latest version
+    # "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git",  # We always build from the latest version
+    # "neo @ git+https://github.com/NeuralEnsemble/python-neo.git",  # We always build from the latest version
 
 ]
 

src/spikeinterface/__init__.py

@@ -30,5 +30,5 @@
 # This flag must be set to False for release
 # This avoids using versioning that contains ".dev0" (and this is a better choice)
 # This is mainly useful when using run_sorter in a container and spikeinterface install
-DEV_MODE = True
-# DEV_MODE = False
+# DEV_MODE = True
+DEV_MODE = False