Support NeuroConv 0.9.1

Kyle · Kyle · commit b070be25760b · 2026-02-13T00:21:00.000-05:00
Key changes:
- Bump neuroconv from 0.6.1 to 0.9.1 across all environments
- Sync ElectrodeColumns/UnitColumns metadata with all schema properties (incl. data_type inference)
- Fix Table.js: value == '' was true for 0 (JS coercion bug)
- Add metadata/conversion_options kwargs to temporally_align_data_interfaces (0.7.0 API change)
- Add _ensure_dandi_staging_alias() for dandi &gt;= 0.74.0 compat
- Update automatic_dandi_upload param: staging → sandbox (0.9.0 API change)
- Pin h5py &lt; 3.13 on macOS arm64 (HDF5 symbol mismatch with pytables)
- Remove deprecated macos-13 CI runners from all workflows
- Update E2E tests: SpikeGLX now uses folder_path + stream_id (0.9.0 API change)
- Loosen ndx-pose (&gt;= 0.1.1) and nwbinspector (&gt;= 0.6.2) pins
- Remove neo and pandas pins (no longer needed with neuroconv &gt;= 0.8.0)
diff --git a/.github/workflows/build_and_deploy_mac.yml b/.github/workflows/build_and_deploy_mac.yml
@@ -7,9 +7,9 @@ on:
 
 jobs:
   deploy-on-mac:
-    runs-on: macos-13
+    runs-on: macos-latest
     # NOTE: macos-latest is an arm64 mac, and the dependency sonpy (Spike2RecordingInterface) has a .so file that
-    # works only on mac x64. This causes issues building and deploying on mac arm64. So we use macos-13 (x64)
+    # Mac runner
     # to build and deploy both the x64 and arm64 versions of the app.
     # NOTE: if changing this to macos-latest, make sure to use the apple-silicon conda environment.
 
diff --git a/.github/workflows/example_data_cache.yml b/.github/workflows/example_data_cache.yml
@@ -21,7 +21,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.12"]
-        os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
+        os: [ubuntu-latest, windows-latest, macos-latest]
 
     steps:
 
diff --git a/.github/workflows/testing_dev.yml b/.github/workflows/testing_dev.yml
@@ -26,8 +26,6 @@ jobs:
           - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
 
-          - os: macos-13  # Mac x64 runner
-            label: environments/environment-MAC-intel.yml
 
           - os: windows-latest
             label: environments/environment-Windows.yml
diff --git a/.github/workflows/testing_dev_e2e_with_live_services.yml b/.github/workflows/testing_dev_e2e_with_live_services.yml
@@ -27,10 +27,6 @@ jobs:
 
           - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
-
-          - os: macos-13  # Mac x64 runner
-            label: environments/environment-MAC-intel.yml
-
           - os: windows-latest
             label: environments/environment-Windows.yml
 
diff --git a/.github/workflows/testing_dev_with_live_services.yml b/.github/workflows/testing_dev_with_live_services.yml
@@ -26,8 +26,6 @@ jobs:
           - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
 
-          - os: macos-13  # Mac x64 runner
-            label: environments/environment-MAC-intel.yml
 
           - os: windows-latest
             label: environments/environment-Windows.yml
diff --git a/.github/workflows/testing_flask_build_and_dist.yml b/.github/workflows/testing_flask_build_and_dist.yml
@@ -22,8 +22,6 @@ jobs:
           - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
 
-          - os: macos-13  # Mac x64 runner
-            label: environments/environment-MAC-intel.yml
 
           - os: windows-latest
             label: environments/environment-Windows.yml
@@ -76,7 +74,7 @@ jobs:
         run: pip uninstall matplotlib --yes
 
       # Fix for macos build - remove bad sonpy file
-      - if: matrix.os == 'macos-latest' || matrix.os == 'macos-13'
+      - if: matrix.os == 'macos-latest'
         run: rm -f "$CONDA_PREFIX/lib/python3.9/site-packages/sonpy/linux/sonpy.so"
 
       - name: Build PyFlask distribution
diff --git a/.github/workflows/testing_pipelines.yml b/.github/workflows/testing_pipelines.yml
@@ -24,8 +24,6 @@ jobs:
           - os: macos-latest  # Mac arm64 runner
             label: environments/environment-MAC-apple-silicon.yml
 
-          - os: macos-13  # Mac x64 runner
-            label: environments/environment-MAC-intel.yml
 
           - os: windows-latest
             label: environments/environment-Windows.yml
diff --git a/environments/environment-Linux.yml b/environments/environment-Linux.yml
@@ -15,13 +15,11 @@ dependencies:
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
       - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.1
-      - pandas < 3.0  # pandas 3.0 returns read-only arrays, breaking spikeinterface Phy extractor
-      - neo == 0.14.1 # 0.14.2 is not compatible with neuroconv < 0.7.5
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.9.1
       - scikit-learn == 1.4.0 # Tutorial data generation
       - tqdm_publisher >= 0.0.1 # Progress bars
       - tzlocal >= 5.2 # Frontend timezone handling
-      - ndx-pose == 0.1.1
-      - nwbinspector == 0.6.2
+      - ndx-pose >= 0.1.1
+      - nwbinspector >= 0.6.2
       - tables
       - numcodecs < 0.16.0  # numcodecs 0.16.0 is not compatible with zarr 2.18.5
diff --git a/environments/environment-MAC-apple-silicon.yml b/environments/environment-MAC-apple-silicon.yml
@@ -9,7 +9,7 @@ dependencies:
   - lxml = 4.9.3 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
   - pyedflib = 0.1.38 # PyPI build fails due to x64/arm64 mismatch so install from conda-forge
   - numpy # May have x64/arm64 mismatch issues so install from conda-forge
-  - pytables = 3.9.1 # PyPI build fails on arm64; must be <3.9.2 per neuroconv 0.6.1 constraint
+  - pytables = 3.10.2 # PyPI build fails on arm64 so install from conda-forge (used by neuroconv deps)
   - jsonschema = 4.18.0 # Also installs jsonschema-specifications
   - pip
   - pip:
@@ -23,12 +23,11 @@ dependencies:
       - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
       # NOTE: the NeuroConv wheel on PyPI includes sonpy which is not compatible with arm64, so build and install
       # NeuroConv from GitHub, which will remove the sonpy dependency when building from Mac arm64
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.1
-      - pandas < 3.0  # pandas 3.0 returns read-only arrays, breaking spikeinterface Phy extractor
-      - neo == 0.14.1 # 0.14.2 is not compatible with neuroconv < 0.7.5
+      - h5py < 3.13 # 3.13+ uses HDF5 1.14.4 features not in pytables 3.10.2's bundled HDF5 1.14.2
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.9.1
       - scikit-learn == 1.4.0 # Tutorial data generation
       - tqdm_publisher >= 0.0.1 # Progress bars
       - tzlocal >= 5.2 # Frontend timezone handling
-      - ndx-pose == 0.1.1
-      - nwbinspector == 0.6.2
+      - ndx-pose >= 0.1.1
+      - nwbinspector >= 0.6.2
       - numcodecs < 0.16.0  # numcodecs 0.16.0 is not compatible with zarr 2.18.5
diff --git a/environments/environment-MAC-intel.yml b/environments/environment-MAC-intel.yml
@@ -18,14 +18,12 @@ dependencies:
       - flask-cors == 4.0.0
       - flask_restx == 1.1.0
       - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.1
-      - pandas < 3.0  # pandas 3.0 returns read-only arrays, breaking spikeinterface Phy extractor
-      - neo == 0.14.1 # 0.14.2 is not compatible with neuroconv < 0.7.5
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.9.1
       - scikit-learn == 1.4.0 # Tutorial data generation
       - tqdm_publisher >= 0.0.1 # Progress bars
       - tzlocal >= 5.2 # Frontend timezone handling
-      - ndx-pose == 0.1.1
-      - nwbinspector == 0.6.2
+      - ndx-pose >= 0.1.1
+      - nwbinspector >= 0.6.2
       - numcodecs < 0.16.0  # numcodecs 0.16.0 is not compatible with zarr 2.18.5
       - h5py == 3.12.1 # 3.13.0 uses features in hdf5 1.14.4 that is not available in earlier hdf5 libs packaged
                        # with tables==3.9.1 (latest that can be used by neuroconv 0.6.0).
diff --git a/environments/environment-Windows.yml b/environments/environment-Windows.yml
@@ -18,13 +18,11 @@ dependencies:
       - flask-cors === 3.0.10
       - flask_restx == 1.1.0
       - werkzeug < 3.0 # werkzeug 3.0 deprecates features used by flask 2.3.2. Remove this when updating flask.
-      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.6.1
-      - pandas < 3.0  # pandas 3.0 returns read-only arrays, breaking spikeinterface Phy extractor
-      - neo == 0.14.1 # 0.14.2 is not compatible with neuroconv < 0.7.5
+      - neuroconv[dandi,compressors,ecephys,ophys,behavior,text] == 0.9.1
       - scikit-learn == 1.4.0 # Tutorial data generation
       - tqdm_publisher >= 0.0.1 # Progress bars
       - tzlocal >= 5.2 # Frontend timezone handling
-      - ndx-pose == 0.1.1
-      - nwbinspector == 0.6.2
+      - ndx-pose >= 0.1.1
+      - nwbinspector >= 0.6.2
       - tables
       - numcodecs < 0.16.0  # numcodecs 0.16.0 is not compatible with zarr 2.18.5
diff --git a/src/electron/frontend/core/components/Table.js b/src/electron/frontend/core/components/Table.js
@@ -377,7 +377,7 @@ export class Table extends LitElement {
                     return;
                 }
 
-                const isUndefined = value == "";
+                const isUndefined = value === "" || value == null;
 
                 if (isUndefined && required) {
                     instanceThis.#handleValidationResult(
diff --git a/src/electron/frontend/core/components/pages/guided-mode/data/GuidedSourceData.js b/src/electron/frontend/core/components/pages/guided-mode/data/GuidedSourceData.js
@@ -28,7 +28,7 @@ const propsToIgnore = {
         es_key: true,
         exclude_shanks: true,
         load_sync_channel: true,
-        stream_id: true, // NOTE: May be desired for other interfaces
+        // stream_id: true, // NOTE: Unhidden — required by SpikeGLX in neuroconv >= 0.9.0
         nsx_override: true,
         combined: true,
         plane_no: true,
diff --git a/src/pyflask/manageNeuroconv/manage_neuroconv.py b/src/pyflask/manageNeuroconv/manage_neuroconv.py
@@ -363,7 +363,7 @@ class CustomNWBConverter(NWBConverter):
 
         # Handle temporal alignment inside the converter
         # TODO: this currently works off of cross-scoping injection of `alignment_info` - refactor to be more explicit
-        def temporally_align_data_interfaces(self):
+        def temporally_align_data_interfaces(self, metadata=None, conversion_options=None):
             set_interface_alignment(self, alignment_info=alignment_info)
 
         # From previous issue regarding SpikeGLX not generating previews of correct size
@@ -596,7 +596,6 @@ def on_recording_interface(name, recording_interface):
 
         # Configure electrode columns
         defs["ElectrodeColumn"] = electrode_def
-        defs["ElectrodeColumn"]["required"] = list(electrode_def["properties"].keys())
 
         new_electrodes_properties = {
             properties["name"]: {key: value for key, value in properties.items() if key != "name"}
@@ -610,6 +609,24 @@ def on_recording_interface(name, recording_interface):
             "additionalProperties": True,  # Allow for new columns
         }
 
+        if has_electrodes:
+            # Ensure ElectrodeColumns includes entries for all Electrode schema properties
+            # (needed for frontend linked-table validation in neuroconv >= 0.6.2)
+            existing_electrode_columns = ecephys_metadata.get("ElectrodeColumns", [])
+            existing_col_names = {col["name"] for col in existing_electrode_columns}
+            for prop_name, prop_info in new_electrodes_properties.items():
+                if prop_name not in existing_col_names:
+                    # Infer data_type from schema type (required by update_recording_properties_from_table_as_json)
+                    schema_type = prop_info.get("type", "str")
+                    data_type = {"number": "float64", "integer": "int64", "boolean": "bool", "array": "object"}.get(schema_type, "str")
+                    existing_electrode_columns.append(
+                        {
+                            "name": prop_name,
+                            "description": prop_info.get("description", "No description."),
+                            "data_type": data_type,
+                        }
+                    )
+
         if has_units:
 
             unitprops_def = defs["UnitProperties"]
@@ -619,7 +636,6 @@ def on_recording_interface(name, recording_interface):
 
             # Configure electrode columns
             defs["UnitColumn"] = unitprops_def
-            defs["UnitColumn"]["required"] = list(unitprops_def["properties"].keys())
 
             new_units_properties = {
                 properties["name"]: {key: value for key, value in properties.items() if key != "name"}
@@ -633,6 +649,29 @@ def on_recording_interface(name, recording_interface):
                 "additionalProperties": True,  # Allow for new columns
             }
 
+            # Ensure UnitColumns includes entries for all Unit schema properties
+            # (needed for frontend linked-table validation in neuroconv >= 0.6.2)
+            existing_unit_columns = metadata["Ecephys"].get("UnitColumns", [])
+            existing_col_names = {col["name"] for col in existing_unit_columns}
+            for prop_name, prop_info in new_units_properties.items():
+                if prop_name not in existing_col_names:
+                    schema_type = prop_info.get("type", "str")
+                    data_type = {"number": "float64", "integer": "int64", "boolean": "bool", "array": "object"}.get(schema_type, "str")
+                    existing_unit_columns.append(
+                        {
+                            "name": prop_name,
+                            "description": prop_info.get("description", "No description."),
+                            "data_type": data_type,
+                        }
+                    )
+
+    # Allow additional properties on Device definitions (e.g. manufacturer from neuroconv)
+    for modality_key in ("Ecephys", "Ophys"):
+        modality_schema = schema.get("properties", {}).get(modality_key, {})
+        device_def = modality_schema.get("definitions", {}).get("Device", {})
+        if device_def:
+            device_def["additionalProperties"] = True
+
     # TODO: generalize logging stuff
     log_base = GUIDE_ROOT_FOLDER / "logs"
     log_base.mkdir(exist_ok=True)
@@ -1342,6 +1381,34 @@ def upload_multiple_filesystem_objects_to_dandi(**kwargs) -> list[Path]:
     return results
 
 
+def _ensure_dandi_staging_alias():
+    """Ensure 'dandi-staging' works as an alias for 'dandi-sandbox'.
+
+    dandi >= 0.74.0 renamed 'dandi-staging' to 'dandi-sandbox' and added an
+    explicit ValueError check rejecting the old name. neuroconv < 0.9.0 uses
+    'dandi-staging' internally. We patch both the known_instances dict and
+    the get_instance function to bypass the rejection.
+    """
+    from dandi.consts import known_instances
+    if "dandi-staging" not in known_instances and "dandi-sandbox" in known_instances:
+        known_instances["dandi-staging"] = known_instances["dandi-sandbox"]
+
+    # Patch get_instance to not raise ValueError for dandi-staging
+    try:
+        import dandi.utils as _dandi_utils
+        _original_get_instance = getattr(_dandi_utils, '_original_get_instance_fn', None)
+        if _original_get_instance is None and hasattr(_dandi_utils, 'get_instance'):
+            _original = _dandi_utils.get_instance
+            def _patched_get_instance(dandi_instance_id, *args, **kwargs):
+                if dandi_instance_id == "dandi-staging":
+                    dandi_instance_id = "dandi-sandbox"
+                return _original(dandi_instance_id, *args, **kwargs)
+            _dandi_utils._original_get_instance_fn = _original
+            _dandi_utils.get_instance = _patched_get_instance
+    except (ImportError, AttributeError):
+        pass
+
+
 def upload_folder_to_dandi(
     dandiset_id: str,
     api_key: str,
@@ -1354,6 +1421,8 @@ def upload_folder_to_dandi(
 ) -> list[Path]:
     from neuroconv.tools.data_transfers import automatic_dandi_upload
 
+    _ensure_dandi_staging_alias()
+
     # Set API key env var for both old (< 0.73.2) and new dandi versions
     os.environ["DANDI_API_KEY"] = api_key
     if sandbox:
@@ -1367,7 +1436,7 @@ def upload_folder_to_dandi(
     return automatic_dandi_upload(
         dandiset_id=dandiset_id,
         nwb_folder_path=Path(nwb_folder_path),
-        staging=sandbox,  # Map sandbox parameter to staging for external API
+        sandbox=sandbox,
         cleanup=cleanup,
         number_of_jobs=number_of_jobs or 1,
         number_of_threads=number_of_threads or 1,
@@ -1386,6 +1455,8 @@ def upload_project_to_dandi(
 ) -> list[Path]:
     from neuroconv.tools.data_transfers import automatic_dandi_upload
 
+    _ensure_dandi_staging_alias()
+
     # CONVERSION_SAVE_FOLDER_PATH.mkdir(exist_ok=True, parents=True)  # Ensure base directory exists
     # Set API key env var for both old (< 0.73.2) and new dandi versions
     os.environ["DANDI_API_KEY"] = api_key
@@ -1400,7 +1471,7 @@ def upload_project_to_dandi(
     return automatic_dandi_upload(
         dandiset_id=dandiset_id,
         nwb_folder_path=CONVERSION_SAVE_FOLDER_PATH / project,  # Scope valid DANDI upload paths to GUIDE projects
-        staging=sandbox,  # Map sandbox parameter to staging for external API
+        sandbox=sandbox,
         cleanup=cleanup,
         number_of_jobs=number_of_jobs,
         number_of_threads=number_of_threads,
diff --git a/tests/e2e/config.ts b/tests/e2e/config.ts
@@ -53,7 +53,7 @@ export const testInterfaceInfo = {
   },
   multi: {
     SpikeGLXRecordingInterface: {
-      format: '{subject_id}/{subject_id}_{session_id}/{subject_id}_{session_id}_g0/{subject_id}_{session_id}_g0_imec0/{subject_id}_{session_id}_g0_t0.imec0.ap.bin',
+      format: '{subject_id}/{subject_id}_{session_id}/{subject_id}_{session_id}_g0',
       autocomplete: {}
     },
     PhySortingInterface: {
@@ -63,7 +63,8 @@ export const testInterfaceInfo = {
   },
   single: {
     SpikeGLXRecordingInterface: {
-      file_path: join(testDataPath, 'spikeglx', 'Session1_g0', 'Session1_g0_imec0', 'Session1_g0_t0.imec0.ap.bin')
+      folder_path: join(testDataPath, 'spikeglx', 'Session1_g0'),
+      stream_id: 'imec0.ap'
     },
     PhySortingInterface: {
       folder_path: join(testDataPath, 'phy')
diff --git a/tests/e2e/workflow.ts b/tests/e2e/workflow.ts
@@ -322,6 +322,21 @@ export default async function runWorkflow(name, workflow, identifier) {
 
     test('Review source data information', async () => {
 
+      // Set stream_id for SpikeGLX interfaces (required in neuroconv >= 0.9.0)
+      await evaluate(async () => {
+        const dashboard = document.querySelector('nwb-dashboard')
+        const page = dashboard.page
+        for (const { form } of Object.values(page.forms)) {
+          const accordions = form.accordions
+          for (const [name, accordion] of Object.entries(accordions)) {
+            if (name.includes('SpikeGLX') && name.includes('Recording')) {
+              const streamInput = form.forms[name]?.getFormElement(['stream_id'])
+              if (streamInput) streamInput.updateData('imec0.ap')
+            }
+          }
+        }
+      })
+
       await takeScreenshot(join(identifier, 'sourcedata-page'), 100)
       await toNextPage('metadata')
 
