HERD Changes internal file (#2111)

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
Co-authored-by: Ben Dichter <ben.dichter@gmail.com>
Co-authored-by: Ryan Ly <310197+rly@users.noreply.github.com>

Author: 4 people

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Fixed broken and redirecting links in documentation. @bendichter [#2165](https://github.com/NeurodataWithoutBorders/pynwb/pull/2165)
 
 ### Added
+- Added support for HERD (HDMF External Resources Data Structure) as the `external_resources` field on `NWBFile`, enabling users to associate external resource annotations (e.g., ontology term mappings) with their NWB files. `link_resources` and `get_external_resources` are inherited from `HERDManager` in hdmf. @mavaylon1, @rly [#2111](https://github.com/NeurodataWithoutBorders/pynwb/pull/2111)
 - Added `get_starting_time()` and `get_duration()` methods to `TimeSeries` to get the starting time and duration of the time series. @h-mayorquin [#2146](https://github.com/NeurodataWithoutBorders/pynwb/pull/2146)
 - Added `get_starting_time()` and `get_duration()` methods to `TimeIntervals` to get the earliest start time and total duration (span from earliest start to latest stop) of all intervals. @h-mayorquin [#2146](https://github.com/NeurodataWithoutBorders/pynwb/pull/2146)
 - Added `get_starting_time()` and `get_duration()` methods to `Units` to get the earliest spike time and total duration (span from earliest to latest spike) across all units. @h-mayorquin [#2164](https://github.com/NeurodataWithoutBorders/pynwb/pull/2164)

environment-ros3.yml

@@ -6,9 +6,10 @@ channels:
 dependencies:
   - python==3.14
   - h5py==3.15.1
+  - hdmf==5.1.0
   - matplotlib==3.10.8
   - numpy==2.4.2
-  - pandas==3.0.1
+  - pandas==2.3.3
   - python-dateutil==2.9.0
   - setuptools
   - pytest==9.0.2
@@ -17,6 +18,5 @@ dependencies:
   - aiohttp==3.13.3
   - pip
   - pip:
-    - hdmf==5.0.0  # not yet available on conda-forge
     - remfile==0.1.13
-    - dandi==0.62.1  # NOTE: dandi is not available on conda for osx-arm64
+    - dandi==0.74.3  # NOTE: dandi is not available on conda for osx-arm64

pyproject.toml

@@ -34,9 +34,9 @@ classifiers = [
 ]
 dependencies = [
     "h5py>=3.6.0",
-    "hdmf>=5.0.0,<6",
+    "hdmf>=5.1.0,<6",
     "numpy>=1.24.0",
-    "pandas>=1.4.0",
+    "pandas>=1.4.0,<3",
     "python-dateutil>=2.8.2",
     "platformdirs>=4.2.2"
 ]

requirements-min.txt

@@ -1,6 +1,6 @@
 # minimum versions of package dependencies for installing PyNWB
 h5py==3.6.0
-hdmf==5.0.0
+hdmf==5.1.0
 numpy==1.24.0
 pandas==1.4.0
 python-dateutil==2.8.2

requirements.txt

@@ -1,6 +1,6 @@
 # pinned dependencies to reproduce an entire development environment to use PyNWB
 h5py==3.16.0
-hdmf==5.0.0
+hdmf==5.1.0
 numpy==2.4.3
 pandas==2.3.3
 python-dateutil==2.9.0.post0

src/pynwb/file.py

@@ -7,7 +7,7 @@
 import numpy as np
 import pandas as pd
 
-from hdmf.common import DynamicTableRegion, DynamicTable
+from hdmf.common import DynamicTableRegion, DynamicTable, HERD
 from hdmf.container import HERDManager
 from hdmf.utils import docval, getargs, get_docval, popargs, popargs_to_dict, AllowPositional
 
@@ -287,6 +287,7 @@ class NWBFile(MultiContainerInterface, HERDManager):
                      {'name': 'trials', 'child': True, 'required_name': 'trials'},
                      {'name': 'units', 'child': True, 'required_name': 'units'},
                      {'name': 'subject', 'child': True, 'required_name': 'subject'},
+                     {'name': 'external_resources', 'child': True, 'required_name': 'external_resources'},
                      {'name': 'sweep_table', 'child': True, 'required_name': 'sweep_table'},
                      {'name': 'invalid_times', 'child': True, 'required_name': 'invalid_times'},
                      # icephys_filtering is temporary. /intracellular_ephys/filtering dataset will be deprecated
@@ -339,6 +340,8 @@ class NWBFile(MultiContainerInterface, HERDManager):
             {'name': 'keywords', 'type': 'array_data', 'doc': 'Terms to search over', 'default': None},
             {'name': 'notes', 'type': str,
              'doc': 'Notes about the experiment.', 'default': None},
+            {'name': 'external_resources', 'type': HERD,
+             'doc': 'the HERD external resources object for this NWBFile', 'default': None},
             {'name': 'pharmacology', 'type': str,
              'doc': 'Description of drugs used, including how and when they were administered. '
                     'Anesthesia(s), painkiller(s), etc., plus dosage, concentration, etc.', 'default': None},
@@ -480,7 +483,8 @@ def __init__(self, **kwargs):
             'icephys_simultaneous_recordings',
             'icephys_sequential_recordings',
             'icephys_repetitions',
-            'icephys_experimental_conditions'
+            'icephys_experimental_conditions',
+            'external_resources'
         ]
         args_to_set = popargs_to_dict(keys_to_set, kwargs)
         kwargs['name'] = 'root'
@@ -1152,4 +1156,4 @@ def ElectrodeTable(name='electrodes',
                    description='metadata about extracellular electrodes'):
     warn("The ElectrodeTable convenience function is deprecated. Please create a new instance of "
          "the ElectrodesTable class instead.", DeprecationWarning)
-    return ElectrodesTable()
+    return ElectrodesTable()

src/pynwb/io/file.py

@@ -112,6 +112,8 @@ def __init__(self, spec):
 
         self.map_spec('subject', general_spec.get_group('subject'))
 
+        self.map_spec('external_resources', general_spec.get_group('external_resources'))
+
         device_spec = general_spec.get_group('devices')
         self.unmap(device_spec)
         self.map_spec('devices', device_spec.get_neurodata_type('Device'))

src/pynwb/nwb-schema

@@ -1,10 +1,53 @@
+import os
+import tempfile
 import warnings
+from datetime import datetime
+from uuid import uuid4
 
+import numpy as np
+from dateutil import tz
+
+from pynwb import NWBHDF5IO, NWBFile
+from pynwb.file import Subject
 from pynwb.resources import HERD
 from pynwb.testing import TestCase
 
 
 class TestNWBContainer(TestCase):
+    def setUp(self):
+        self.tmpdir = tempfile.TemporaryDirectory()
+        self.path = os.path.join(self.tmpdir.name, "resources_file.nwb")
+        self.export_path = os.path.join(self.tmpdir.name, "export_file.nwb")
+
+    def tearDown(self):
+        self.tmpdir.cleanup()
+
+    def _create_nwbfile_with_herd(self):
+        """Create an NWBFile with a Subject and HERD containing a species annotation."""
+        session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific"))
+        nwbfile = NWBFile(
+            session_description="ECoG recording during audio speech perception task",
+            identifier=str(uuid4()),
+            session_start_time=session_start_time,
+        )
+        subject = Subject(
+            subject_id="001",
+            age="P26Y",
+            description="human subject",
+            species="Homo sapiens",
+            sex="M",
+        )
+        nwbfile.subject = subject
+        herd = HERD()
+        nwbfile.external_resources = herd
+        nwbfile.external_resources.add_ref(
+            container=nwbfile.subject,
+            key=nwbfile.subject.species,
+            entity_id="NCBI_TAXON:9606",
+            entity_uri="https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606",
+        )
+        return nwbfile, subject
+
     def test_constructor(self):
         """
         Test constructor
@@ -17,3 +60,134 @@ def test_constructor(self):
             )
             er = HERD()
             self.assertIsInstance(er, HERD)
+
+    def test_nwbfile_init_herd(self):
+        session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific"))
+        herd = HERD()
+        nwbfile = NWBFile(
+            session_description="ECoG recording during audio speech perception task",
+            identifier=str(uuid4()),
+            session_start_time=session_start_time,
+            external_resources=herd,
+        )
+        self.assertIsInstance(nwbfile.external_resources, HERD)
+
+    def test_nwbfile_set_herd(self):
+        session_start_time = datetime(2018, 4, 25, 2, 30, 3, tzinfo=tz.gettz("US/Pacific"))
+        herd = HERD()
+        nwbfile = NWBFile(
+            session_description="ECoG recording during audio speech perception task",
+            identifier=str(uuid4()),
+            session_start_time=session_start_time,
+        )
+        nwbfile.external_resources = herd
+        self.assertIsInstance(nwbfile.external_resources, HERD)
+        self.assertEqual(nwbfile.external_resources.parent, nwbfile)
+
+    def test_resources_roundtrip(self):
+        nwbfile, subject = self._create_nwbfile_with_herd()
+
+        with NWBHDF5IO(self.path, "w") as io:
+            io.write(nwbfile)
+
+        with NWBHDF5IO(self.path, "r") as io:
+            read_nwbfile = io.read()
+            self.assertEqual(
+                read_nwbfile.external_resources.keys[:],
+                np.array(
+                    [("Homo sapiens",)],
+                    dtype=[("key", "O")],
+                ),
+            )
+            self.assertEqual(
+                read_nwbfile.external_resources.entities[:],
+                np.array(
+                    [
+                        (
+                            "NCBI_TAXON:9606",
+                            "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606",
+                        )
+                    ],
+                    dtype=[("entity_id", "O"), ("entity_uri", "O")],
+                ),
+            )
+            self.assertEqual(
+                read_nwbfile.external_resources.objects[:],
+                np.array(
+                    [(0, subject.object_id, "Subject", "", "")],
+                    dtype=[
+                        ("files_idx", "<u4"),
+                        ("object_id", "O"),
+                        ("object_type", "O"),
+                        ("relative_path", "O"),
+                        ("field", "O"),
+                    ],
+                ),
+            )
+
+    def test_get_external_resources(self):
+        """Test get_external_resources returns the correct HERD based on the linked parameter."""
+        nwbfile, subject = self._create_nwbfile_with_herd()
+        original_herd = nwbfile.external_resources
+
+        linked_herd = HERD()
+        nwbfile.link_resources(linked_herd)
+
+        self.assertIs(nwbfile.get_external_resources(linked=False), original_herd)
+        self.assertIs(nwbfile.get_external_resources(linked=True), linked_herd)
+        # attribute returns the original, not the linked one
+        self.assertIs(nwbfile.external_resources, original_herd)
+
+    def test_link_resources(self):
+        """Make sure that the original HERD is not overwritten on export."""
+        nwbfile, subject = self._create_nwbfile_with_herd()
+
+        with NWBHDF5IO(self.path, "w") as io:
+            io.write(nwbfile)
+
+        with NWBHDF5IO(self.path, mode="r") as read_io:
+            read_nwbfile = read_io.read()
+            read_nwbfile.link_resources(HERD())
+
+            linked = read_nwbfile.get_external_resources(linked=True)
+            self.assertEqual(linked.keys.data, [])
+            self.assertEqual(linked.entities.data, [])
+            self.assertEqual(linked.objects.data, [])
+
+            with NWBHDF5IO(self.export_path, mode="w") as export_io:
+                export_io.export(src_io=read_io, nwbfile=read_nwbfile)
+
+            with NWBHDF5IO(self.export_path, mode="r") as read_export_io:
+                read_export_nwbfile = read_export_io.read()
+                self.assertEqual(
+                    read_export_nwbfile.external_resources.keys[:],
+                    np.array(
+                        [("Homo sapiens",)],
+                        dtype=[("key", "O")],
+                    ),
+                )
+                self.assertEqual(
+                    read_export_nwbfile.external_resources.entities[:],
+                    np.array(
+                        [
+                            (
+                                "NCBI_TAXON:9606",
+                                "https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606",
+                            )
+                        ],
+                        dtype=[("entity_id", "O"), ("entity_uri", "O")],
+                    ),
+                )
+                self.assertEqual(
+                    read_export_nwbfile.external_resources.objects[:],
+                    np.array(
+                        [(0, subject.object_id, "Subject", "", "")],
+                        dtype=[
+                            ("files_idx", "<u4"),
+                            ("object_id", "O"),
+                            ("object_type", "O"),
+                            ("relative_path", "O"),
+                            ("field", "O"),
+                        ],
+                    ),
+                )