Implement NeurodataWithoutBorders/nwb-schema#690: add source_description to EventsTable; add merge/query helpers to NWBFile

CHANGELOG.md

@@ -10,6 +10,7 @@
 - Added example of setting `Units.resolution` in the ecephys tutorial. @h-mayorquin [#2174](https://github.com/NeurodataWithoutBorders/pynwb/pull/2174)
 
 ### Added
+- Added optional `source_description` attribute to `EventsTable` for a short free-text label of where events originated (e.g., `"Acquisition system"`, `"Manual video review"`). Added `NWBFile.merge_events_tables()` to merge a list of `EventsTable` objects into a single DataFrame sorted by timestamp with a `source_events_table` column. Added `NWBFile.get_all_events()` to merge all tables in `NWBFile.events`. @rly [#2192](https://github.com/NeurodataWithoutBorders/pynwb/pull/2192)
 - Added support for NWB Schema 2.10.0 ([NWBEP001](https://nwb-schema.readthedocs.io/)), which introduces the `EventsTable`, `TimestampVectorData`, and `DurationVectorData` neurodata types and a new `events` group on `NWBFile` for storing `EventsTable` instances. Use `NWBFile.add_events_table()` to add an `EventsTable` and `NWBFile.get_events_table()` to retrieve one. NWB Schema 2.10.0 also incorporates hdmf-common-schema 1.9.0, which adds the `MeaningsTable` neurodata type (re-exported via `hdmf.common`) and support for attaching one or more `MeaningsTable` instances to a `DynamicTable` to document the meanings of values in a column. See the [NWB Schema release notes](https://nwb-schema.readthedocs.io/en/latest/format_release_notes.html) and the [hdmf-common-schema release notes](https://hdmf-common-schema.readthedocs.io/en/latest/format_release_notes.html) for the full list of changes. @rly [#2156](https://github.com/NeurodataWithoutBorders/pynwb/pull/2156)
 - Added support for HERD (HDMF External Resources Data Structure) as the `external_resources` field on `NWBFile`, enabling users to associate external resource annotations (e.g., ontology term mappings) with their NWB files. `link_resources` and `get_external_resources` are inherited from `HERDManager` in hdmf. @mavaylon1, @rly [#2111](https://github.com/NeurodataWithoutBorders/pynwb/pull/2111)
 - Added `get_starting_time()` and `get_duration()` methods to `TimeSeries` to get the starting time and duration of the time series. @h-mayorquin [#2146](https://github.com/NeurodataWithoutBorders/pynwb/pull/2146)

src/pynwb/behavior.py

@@ -97,10 +97,14 @@ class BehavioralEpochs(MultiContainerInterface):
 
 @register_class('BehavioralEvents', CORE_NAMESPACE)
 class BehavioralEvents(MultiContainerInterface):
-    """
-    DEPRECATED. TimeSeries for storing behavioral events. See description of BehavioralEpochs for more details.
-
-    BehavioralEvents is deprecated. Use an EventsTable in NWBFile.events instead for event data.
+    """DEPRECATED. Use an :py:class:`~pynwb.event.EventsTable` instead, placed in the top-level ``/events``
+    group of the NWBFile. Each TimeSeries formerly stored under BehavioralEvents becomes one EventsTable.
+    The ``timestamps`` field maps to the ``timestamp`` column, and the ``data`` field maps to an additional
+    column named after the event marker (e.g., ``reward_magnitude``, ``port_number``); for multi-dimensional
+    ``data``, use one column per field. Any other per-event metadata becomes additional columns. Use the
+    ``source_description`` attribute on the EventsTable to record where the events came from (e.g.,
+    "Acquisition system", "Thresholding of analog signal ANALOG1 at 3 V", "Manual video review"). Original
+    definition: TimeSeries for storing behavioral events. See description of BehavioralEpochs for more details.
     """
 
     __clsconf__ = {
@@ -119,7 +123,8 @@ def __init__(self, **kwargs):
         super().__init__(**kwargs)
         self.add_timeseries(time_series)
         self._warn_on_new_pass_on_construct(
-            "BehavioralEvents is deprecated. Use an EventsTable in NWBFile.events instead for event data. "
+            "BehavioralEvents is deprecated. Use an EventsTable instead, added to the NWBFile via "
+            "nwbfile.add_events_table() or nwbfile.create_events_table(). "
             "Creating a new BehavioralEvents will not be allowed in a future version of PyNWB."
         )
 

src/pynwb/event.py

@@ -68,10 +68,14 @@ def unit(self):
 
 @register_class('EventsTable', CORE_NAMESPACE)
 class EventsTable(DynamicTable):
-    """
-    A column-based table to store information about events (event instances), one
-    event per row. Additional columns may be added to store metadata about each event,
-    such as the duration of the event.
+    """A column-based table to store information about events, one event per row.
+
+    Use EventsTable when each row is anchored at a single timestamp and duration is absent, optional, or
+    mixed across rows. Additional columns may be added to store metadata about each event, such as the
+    duration of the event. Examples include TTL pulses, licks, rewards, stimulus onsets, and detected
+    ripples. Each EventsTable should hold events of a single type, so that all rows share the same set
+    of per-event metadata columns. Events of different types (e.g., licks and stimulus presentations)
+    should be stored in separate EventsTable instances.
     """
 
     __columns__ = (
@@ -82,19 +86,24 @@ class EventsTable(DynamicTable):
         {'name': 'annotation', 'description': 'User annotations about events.'},
     )
 
-    # The override exists to give EventsTable an explicit, narrowed docval signature
-    # (required name/description, AllowPositional.ERROR, and a curated subset of
-    # DynamicTable kwargs) rather than to add init-time logic.
     @docval(
         {'name': 'name', 'type': str, 'doc': 'Name of this EventsTable'},
         {'name': 'description', 'type': str,
          'doc': ('A description of the events stored in the table, including information about '
                  'how the event times were computed.')},
+        {'name': 'source_description', 'type': str,
+         'doc': ('Optional short text description of where the events came from, applying to every row '
+                 'in the table. For example, "Acquisition system" for events emitted directly by the '
+                 'acquisition system, "Thresholding of analog signal ANALOG1 at 3 V" for events produced '
+                 'by a detection algorithm, or "Manual video review" for events added by a human annotator.'),
+         'default': None},
         *get_docval(DynamicTable.__init__, 'id', 'columns', 'colnames', 'target_tables', 'meanings_tables'),
         allow_positional=AllowPositional.ERROR,
     )
     def __init__(self, **kwargs):
+        source_description = kwargs.pop('source_description')
         super().__init__(**kwargs)
+        self.source_description = source_description
 
     @docval(
         {'name': 'timestamp', 'type': float,

src/pynwb/file.py

@@ -254,6 +254,7 @@ class NWBFile(MultiContainerInterface, HERDManager):
         {
             'attr': 'events',
             'add': 'add_events_table',
+            'create': 'create_events_table',
             'type': EventsTable,
             'get': 'get_events_table'
         },
@@ -1087,6 +1088,29 @@ def add_scratch(self, **kwargs):
                      'DynamicTable to scratch.')
         return self._add_scratch(data)
 
+    def merge_events_tables(self, tables: list[EventsTable]) -> pd.DataFrame:
+        """Merge a list of EventsTable objects into a single DataFrame indexed by timestamp.
+
+        Each table is converted to a DataFrame with the timestamp column as the index. A
+        ``source_events_table`` column is added to identify which table each row came from.
+        Columns present in only some tables are filled with NaN. Rows are sorted by timestamp.
+        """
+        frames = []
+        for table in tables:
+            df = table.to_dataframe().set_index("timestamp")
+            df.insert(0, "source_events_table", table.name)
+            frames.append(df)
+        return pd.concat(frames, sort=True).sort_index()
+
+    def get_all_events(self) -> pd.DataFrame:
+        """Merge all EventsTable objects in ``NWBFile.events`` into a single DataFrame indexed by timestamp.
+
+        Returns an empty DataFrame if no events tables exist.
+        """
+        if not self.events:
+            return pd.DataFrame()
+        return self.merge_events_tables(list(self.events.values()))
+
     @docval({'name': 'name', 'type': str, 'doc': 'the name of the object to get'},
             {'name': 'convert', 'type': bool, 'doc': 'return the original data, not the NWB object', 'default': True})
     def get_scratch(self, **kwargs):

src/pynwb/misc.py

@@ -21,13 +21,14 @@
 
 @register_class('AnnotationSeries', CORE_NAMESPACE)
 class AnnotationSeries(TimeSeries):
-    """DEPRECATED. Stores text-based records about the experiment.
-
-    AnnotationSeries is deprecated. Use an EventsTable with an 'annotation' column instead.
-    See :py:class:`~pynwb.event.EventsTable`.
-
-    To use the AnnotationSeries, add records individually through add_annotation(). Alternatively, if all annotations
-    are already stored in a list or numpy array, set the data and timestamps in the constructor.
+    """DEPRECATED. Use an :py:class:`~pynwb.event.EventsTable` instead, placed in the top-level ``/events``
+    group of the NWBFile. The ``timestamps`` field maps to the ``timestamp`` column, and the ``data`` field
+    (annotation strings) maps to the ``annotation`` column on EventsTable. Use the ``source_description``
+    attribute on the EventsTable to record where the events came from (e.g., "Acquisition system",
+    "Thresholding of analog signal ANALOG1 at 3 V", "Manual video review"). Original definition: Stores
+    user annotations made during an experiment. The data[] field stores a text array, and timestamps are
+    stored for each annotation (i.e., interval=1). This is largely an alias to a standard TimeSeries
+    storing a text array but that is identifiable as storing annotations in a machine-readable way.
     """
 
     __nwbfields__ = ()
@@ -42,7 +43,8 @@ def __init__(self, **kwargs):
         name, data, timestamps = popargs('name', 'data', 'timestamps', kwargs)
         super().__init__(name=name, data=data, unit='n/a', resolution=-1.0, timestamps=timestamps, **kwargs)
         self._warn_on_new_pass_on_construct(
-            "AnnotationSeries is deprecated. Use an EventsTable with an 'annotation' column instead. "
+            "AnnotationSeries is deprecated. Use an EventsTable with an 'annotation' column instead, "
+            "added to the NWBFile via nwbfile.add_events_table() or nwbfile.create_events_table(). "
             "Creating a new AnnotationSeries will not be allowed in a future version of PyNWB."
         )
 

tests/integration/hdf5/test_event.py

@@ -159,6 +159,25 @@ def test_roundtrip_with_extra_columns(self):
             self.assertEqual(read_table['event_type'].data[0], 'stimulus')
             self.assertEqual(read_table['confidence'].data[0], 0.95)
 
+    def test_roundtrip_source_description(self):
+        """Test that source_description is preserved after roundtrip"""
+        table = EventsTable(
+            name='test_events',
+            description='Test events table',
+            source_description='Acquisition system',
+        )
+        table.add_event(timestamp=1.0)
+
+        nwbfile = self._create_nwbfile()
+        nwbfile.add_events_table(table)
+        with NWBHDF5IO(self.path, 'w') as io:
+            io.write(nwbfile)
+
+        with NWBHDF5IO(self.path, 'r') as io:
+            read_nwbfile = io.read()
+            read_table = read_nwbfile.events['test_events']
+            self.assertEqual(read_table.source_description, 'Acquisition system')
+
     def test_roundtrip_timestamp_resolution(self):
         """Test that timestamp resolution is preserved after roundtrip"""
         # Create a table with a pre-created timestamp column with resolution

tests/unit/test_behavior.py

@@ -64,7 +64,8 @@ class BehavioralEventsConstructor(TestCase):
     def test_init_deprecated(self):
         """Test that creating a BehavioralEvents warns about deprecation."""
         msg = (
-            "BehavioralEvents is deprecated. Use an EventsTable in NWBFile.events instead for event data. "
+            "BehavioralEvents is deprecated. Use an EventsTable instead, added to the NWBFile via "
+            "nwbfile.add_events_table() or nwbfile.create_events_table(). "
             "Creating a new BehavioralEvents will not be allowed in a future version of PyNWB."
         )
         ts = TimeSeries(name='test_ts', data=np.ones((3, 2)), unit='unit', timestamps=[1., 2., 3.])

tests/unit/test_event.py

@@ -1,7 +1,10 @@
 import numpy as np
+from datetime import datetime
+from dateutil.tz import tzlocal
 
 from hdmf.common import MeaningsTable, VectorData
 
+from pynwb import NWBFile
 from pynwb.event import TimestampVectorData, DurationVectorData, EventsTable
 from pynwb.testing import TestCase
 
@@ -51,8 +54,15 @@ def test_init(self):
         table = EventsTable(name='events', description='test events')
         self.assertEqual(table.name, 'events')
         self.assertEqual(table.description, 'test events')
+        self.assertIsNone(table.source_description)
         self.assertEqual(len(table), 0)
 
+    def test_init_with_source_description(self):
+        """Test initialization with source_description"""
+        table = EventsTable(name='events', description='test events',
+                            source_description='Acquisition system')
+        self.assertEqual(table.source_description, 'Acquisition system')
+
     def test_add_event_timestamp_only(self):
         """Test adding event with only timestamp"""
         table = EventsTable(name='events', description='test events')
@@ -151,3 +161,90 @@ def test_init_with_meanings_tables(self):
         )
         self.assertIn('annotation_meanings', table.meanings_tables)
         self.assertIs(table.meanings_tables['annotation_meanings'], meanings)
+
+
+class TestNWBFileMergeEvents(TestCase):
+    """Unit tests for NWBFile.merge_events_tables and NWBFile.get_all_events"""
+
+    def _make_nwbfile(self):
+        return NWBFile(
+            session_description='test',
+            identifier='test_merge_events',
+            session_start_time=datetime(2021, 1, 1, tzinfo=tzlocal()),
+        )
+
+    def _make_table(self, name, timestamps, **extra_cols):
+        table = EventsTable(name=name, description=f'{name} events')
+        for col_name in extra_cols:
+            table.add_column(name=col_name, description=col_name)
+        for i, ts in enumerate(timestamps):
+            row = {'timestamp': ts}
+            for col_name, values in extra_cols.items():
+                row[col_name] = values[i]
+            table.add_event(**row)
+        return table
+
+    def test_merge_events_tables_row_count_and_index(self):
+        """merge_events_tables returns a DataFrame indexed by timestamp with all rows."""
+        t1 = self._make_table('licks', [1.0, 2.0])
+        t2 = self._make_table('rewards', [3.0, 4.0])
+        nwbfile = self._make_nwbfile()
+        result = nwbfile.merge_events_tables([t1, t2])
+        self.assertEqual(result.index.name, 'timestamp')
+        np.testing.assert_array_equal(result.index, [1.0, 2.0, 3.0, 4.0])
+
+    def test_merge_events_tables_sorted_by_timestamp(self):
+        """merge_events_tables sorts rows by timestamp across tables."""
+        t1 = self._make_table('licks', [1.0, 4.0])
+        t2 = self._make_table('rewards', [2.0, 3.0])
+        nwbfile = self._make_nwbfile()
+        result = nwbfile.merge_events_tables([t1, t2])
+        np.testing.assert_array_equal(result.index, [1.0, 2.0, 3.0, 4.0])
+
+    def test_merge_events_tables_source_column(self):
+        """merge_events_tables adds a source_events_table column with the table name."""
+        t1 = self._make_table('licks', [1.0, 2.0])
+        t2 = self._make_table('rewards', [3.0])
+        nwbfile = self._make_nwbfile()
+        result = nwbfile.merge_events_tables([t1, t2])
+        self.assertIn('source_events_table', result.columns)
+        self.assertEqual(result.iloc[0]['source_events_table'], 'licks')
+        self.assertEqual(result.iloc[1]['source_events_table'], 'licks')
+        self.assertEqual(result.iloc[2]['source_events_table'], 'rewards')
+
+    def test_merge_events_tables_fills_missing_columns_with_nan(self):
+        """Columns absent from some tables are filled with NaN."""
+        t1 = self._make_table('licks', [1.0], annotation=['lick'])
+        t2 = self._make_table('rewards', [2.0])
+        nwbfile = self._make_nwbfile()
+        result = nwbfile.merge_events_tables([t1, t2])
+        self.assertIn('annotation', result.columns)
+        self.assertEqual(result.loc[1.0, 'annotation'], 'lick')
+        self.assertTrue(np.isnan(result.loc[2.0, 'annotation']))
+
+    def test_get_all_events(self):
+        """get_all_events merges all tables in NWBFile.events, sorted by timestamp."""
+        t1 = self._make_table('licks', [1.0, 4.0])
+        t2 = self._make_table('rewards', [2.0, 3.0])
+        nwbfile = self._make_nwbfile()
+        nwbfile.add_events_table(t1)
+        nwbfile.add_events_table(t2)
+        result = nwbfile.get_all_events()
+        self.assertEqual(result.index.name, 'timestamp')
+        self.assertEqual(len(result), 4)
+        np.testing.assert_array_equal(result.index, [1.0, 2.0, 3.0, 4.0])
+        self.assertIn('source_events_table', result.columns)
+
+    def test_get_all_events_empty(self):
+        """get_all_events returns an empty DataFrame when no events tables exist."""
+        nwbfile = self._make_nwbfile()
+        result = nwbfile.get_all_events()
+        self.assertEqual(len(result), 0)
+
+    def test_create_events_table(self):
+        """NWBFile.create_events_table instantiates and registers an EventsTable."""
+        nwbfile = self._make_nwbfile()
+        table = nwbfile.create_events_table(name='licks', description='Lick times')
+        self.assertIsInstance(table, EventsTable)
+        self.assertIn('licks', nwbfile.events)
+        self.assertIs(nwbfile.events['licks'], table)

tests/unit/test_misc.py

@@ -17,7 +17,8 @@ class AnnotationSeriesConstructor(TestCase):
     def test_init_deprecated(self):
         """Test that creating an AnnotationSeries warns about deprecation."""
         msg = (
-            "AnnotationSeries is deprecated. Use an EventsTable with an 'annotation' column instead. "
+            "AnnotationSeries is deprecated. Use an EventsTable with an 'annotation' column instead, "
+            "added to the NWBFile via nwbfile.add_events_table() or nwbfile.create_events_table(). "
             "Creating a new AnnotationSeries will not be allowed in a future version of PyNWB."
         )
         with self.assertWarnsWith(UserWarning, msg):