Merge branch 'dev' into release/5.0.0

Author: rly

CHANGELOG.md

@@ -12,18 +12,22 @@
 - Deprecated `BaseStorageSpec.add_attribute`, `GroupSpec.add_group`, `GroupSpec.add_dataset`, and `GroupSpec.add_link`. Use `set_attribute`, `set_group`, `set_dataset`, and `set_link` instead. @rly [#1333](https://github.com/hdmf-dev/hdmf/pull/1333)
 - Deprecated unused `BaseStorageSpec.get_data_type_spec` and `BaseStorageSpec.get_namespace_spec`. @rly [#1333](https://github.com/hdmf-dev/hdmf/pull/1333)
 
-### Added
-- Added support for HDMF Common Schema 1.9.0.
-  - Introduced a new data type `MeaningsTable` and changes to `DynamicTable` to support included `MeaningsTable` objects. @rly [#1376](https://github.com/hdmf-dev/hdmf/pull/1376)
-- Warning when `data_type_def` and `data_type_inc` are the same in a spec. @rly [#1312](https://github.com/hdmf-dev/hdmf/pull/1312)
-- Added abstract methods `HDMFIO.load_namespaces` and `HDMFIO.load_namespaces_io`. @rly [#1299](https://github.com/hdmf-dev/hdmf/pull/1299)
-
 ### Removed
 - Removed unused and undocumented `hdmf.monitor` module. @rly [#1327](https://github.com/hdmf-dev/hdmf/pull/1327)
 - Removed deprecated `Data.set_data_io` usage and `HERDManager` methods. @rly [#1328](https://github.com/hdmf-dev/hdmf/pull/1328)
 - Removed deprecated `HDF5IO.copy_file` method. Use the `HDF5IO.export` or the `h5py.File.copy` method instead. @stephprince [#1332](https://github.com/hdmf-dev/hdmf/pull/1332)
 - Removed deprecated `extensions` kwarg for `get_type_map` function. @stephprince [#1332](https://github.com/hdmf-dev/hdmf/pull/1332)
 
+### Added
+- Added support for HDMF Common Schema 1.9.0.
+  - Introduced a new data type `MeaningsTable` and changes to `DynamicTable` to support included `MeaningsTable` objects. @rly [#1376](https://github.com/hdmf-dev/hdmf/pull/1376)
+  - Promoted `HERD` from the hdmf-experimental namespace to the HDMF Common namespace. @rly [#1387](https://github.com/hdmf-dev/hdmf/pull/1387)
+- Added warning when `data_type_def` and `data_type_inc` are the same in a spec. @rly [#1312](https://github.com/hdmf-dev/hdmf/pull/1312)
+- Added abstract methods `HDMFIO.load_namespaces` and `HDMFIO.load_namespaces_io`. @rly [#1299](https://github.com/hdmf-dev/hdmf/pull/1299)
+
+### Fixed
+- Fixed a broken test and refactored `VectorIndex.get`. @rly, @mavaylon1 [#1293](https://github.com/hdmf-dev/hdmf/pull/1293)
+
 
 ## HDMF 4.3.1 (January 28, 2026)
 

docs/gallery/plot_external_resources.py

@@ -3,10 +3,7 @@
 ==============================================
 
 This is a user guide to interacting with the
-:py:class:`~hdmf.common.resources.HERD` class. The HERD type
-is experimental and is subject to change in future releases. If you use this type,
-please provide feedback to the HDMF team so that we can improve the structure and
-access of data stored with this type for your use cases.
+:py:class:`~hdmf.common.resources.HERD` class.
 
 Introduction
 -------------
@@ -96,9 +93,6 @@
 from hdmf import Data
 import numpy as np
 import os
-# Ignore experimental feature warnings in the tutorial to improve rendering
-import warnings
-warnings.filterwarnings("ignore", category=UserWarning, message="HERD is experimental*")
 
 try:
     import linkml_runtime  # noqa: F401

pyproject.toml

@@ -174,7 +174,9 @@ line-length = 120
 [tool.ruff.lint.per-file-ignores]
 "docs/gallery/*" = ["E402", "T201"]
 "src/*/__init__.py" = ["F401"]
+"src/hdmf/testing/make_test_files.py" = ["T201"]
 "test_gallery.py" = ["T201"]
 
+
 [tool.ruff.lint.mccabe]
 max-complexity = 17

src/hdmf/common/__init__.py

@@ -272,6 +272,6 @@ def get_hdf5io(**kwargs):
 MeaningsTable = get_class('MeaningsTable', CORE_NAMESPACE)
 EnumData = get_class('EnumData', EXP_NAMESPACE)
 CSRMatrix = get_class('CSRMatrix', CORE_NAMESPACE)
-HERD = get_class('HERD', EXP_NAMESPACE)
+HERD = get_class('HERD', CORE_NAMESPACE)
 SimpleMultiContainer = get_class('SimpleMultiContainer', CORE_NAMESPACE)
 AlignedDynamicTable = get_class('AlignedDynamicTable', CORE_NAMESPACE)

src/hdmf/common/resources.py

@@ -1,6 +1,6 @@
 import pandas as pd
 import numpy as np
-from . import register_class, EXP_NAMESPACE
+from . import register_class
 from . import get_type_map
 from ..container import Table, Row, Container, Data, AbstractContainer, HERDManager
 from ..term_set import TermSet
@@ -157,7 +157,7 @@ class ObjectKey(Row):
     __table__ = ObjectKeyTable
 
 
-@register_class('HERD', EXP_NAMESPACE)
+@register_class('HERD')
 class HERD(Container):
     """
     HDMF External Resources Data Structure.

src/hdmf/common/table.py

@@ -233,21 +233,8 @@ def get(self, arg, **kwargs):
                 indices = arg
             ret = list()
             if len(indices) > 0:
-                # Note: len(indices) == 0 for test_to_hierarchical_dataframe_empty_tables.
-                # This is an edge case test for to_hierarchical_dataframe() on empty tables.
-                # When len(indices) == 0, ret is expected to be an empty list, defined above.
-                try:
-                    data = self.target.get(slice(None),  **kwargs)
-                except IndexError:
-                    """
-                    Note: TODO: test_to_hierarchical_dataframe_indexed_dtr_on_last_level.
-                    This is the old way to get the data and not an untested feature.
-                    """
-                    for i in indices:
-                        ret.append(self.__getitem_helper(i, **kwargs))
-
-                    return ret
-
+                # Load the entire target table at once to avoid multiple I/O calls
+                data = self.target.get(slice(None),  **kwargs)
                 slices = [self.__get_slice(i) for i in indices]
                 if isinstance(data, pd.DataFrame):
                     ret = [data.iloc[s] for s in slices]

src/hdmf/testing/make_test_files.py

@@ -0,0 +1,61 @@
+"""Script to generate test files for backward compatibility testing."""
+import os
+from pathlib import Path
+
+from hdmf import Data, HERDManager, __version__
+from hdmf.common import SimpleMultiContainer, get_hdf5io
+from hdmf.common.resources import HERD
+
+
+class _HERDManagerContainer(SimpleMultiContainer, HERDManager):
+    """A SimpleMultiContainer that also implements HERDManager, for testing."""
+
+    pass
+
+
+def make_herd_1_8_0_file(outdir):
+    """Generate an HDF5 file using hdmf-common-schema 1.8.0.
+
+    In schema 1.8.0, HERD was defined in the hdmf-experimental namespace.
+    This creates test files to verify backward compatibility when reading
+    files created with the old namespace.
+
+    Before running this, pip install hdmf==4.3.1 which includes hdmf-common-schema 1.8.0.
+
+    Parameters
+    ----------
+    outdir : pathlib.Path
+        Directory where the test files will be written.
+    """
+    species = Data(name="species", data=["Homo sapiens", "Mus musculus"])
+    container = _HERDManagerContainer(name="root", containers=[species])
+
+    er = HERD()
+    er.add_ref(
+        file=container,
+        container=species,
+        key="Homo sapiens",
+        entity_id="NCBI_TAXON:9606",
+        entity_uri="https://www.ncbi.nlm.nih.gov/Taxonomy/Browser/wwwtax.cgi?mode=Info&id=9606",
+    )
+
+    h5_path = outdir / "herd_1_8_0.h5"
+    with get_hdf5io(path=str(h5_path), mode="w") as io:
+        io.write(container)
+
+    old_cwd = os.getcwd()
+    os.chdir(str(outdir))
+    er.to_zip(path="herd_1_8_0.zip")
+    os.chdir(old_cwd)
+
+    print(f"Created {h5_path}")
+
+
+if __name__ == "__main__":
+    # Install these versions of HDMF and run this script to generate new files:
+    #   python src/hdmf/testing/make_test_files.py
+    # Files will be made in tests/back_compat/
+
+    if __version__ == '4.3.1':
+        outdir = Path(__file__).resolve().parent.parent.parent.parent / "tests" / "unit" / "back_compat_tests"
+        make_herd_1_8_0_file(outdir)

tests/unit/back_compat_tests/herd_1_8_0.h5

@@ -0,0 +1,52 @@
+"""Test reading HERD from files created with hdmf-common-schema 1.8.0.
+
+In schema 1.8.0, HERD was defined in the hdmf-experimental namespace.
+These tests verify that HERD is read correctly and is resolved to the
+hdmf-common namespace when using the current schema.
+"""
+
+from hdmf.common import get_hdf5io, get_type_map, CORE_NAMESPACE, SimpleMultiContainer
+from hdmf.common.resources import HERD
+from hdmf.testing import TestCase
+
+
+class TestHERD_1_8_0(TestCase):
+    """Test reading HERD from files written with hdmf-common-schema 1.8.0.
+
+    The test files were generated by make_test_files.py using hdmf 4.3.1
+    (hdmf-common-schema 1.8.0) where HERD was in the hdmf-experimental
+    namespace. The HDF5 file contains a SimpleMultiContainer with a Data
+    object, and the ZIP file contains the HERD tables.
+    """
+
+    def setUp(self):
+        self.path_h5 = "tests/unit/back_compat_tests/herd_1_8_0.h5"
+        self.path_zip = "tests/unit/back_compat_tests/herd_1_8_0.zip"
+
+    def test_read_container(self):
+        """Test that the SimpleMultiContainer with data is read correctly."""
+        with get_hdf5io(path=self.path_h5, mode="r") as io:
+            container = io.read()
+            self.assertIsInstance(container, SimpleMultiContainer)
+            self.assertIn("species", container.containers)
+
+    def test_read_herd_from_zip(self):
+        """Test that HERD written under hdmf-experimental is read correctly from ZIP."""
+        herd = HERD.from_zip(path=self.path_zip)
+        self.assertIsInstance(herd, HERD)
+        self.assertEqual(len(herd.keys.data), 1)
+        self.assertEqual(herd.keys.data[0][0], "Homo sapiens")
+        self.assertEqual(len(herd.entities.data), 1)
+        self.assertEqual(herd.entities.data[0][0], "NCBI_TAXON:9606")
+
+    def test_read_herd_with_hdf5io(self):
+        """Test that HERD is loaded when reading HDF5 with herd_path."""
+        with get_hdf5io(path=self.path_h5, mode="r", herd_path=self.path_zip) as io:
+            io.read()
+            self.assertIsInstance(io.herd, HERD)
+
+    def test_herd_namespace_is_common(self):
+        """Test that HERD is resolved in the hdmf-common namespace."""
+        tm = get_type_map()
+        dt = tm.namespace_catalog.get_spec(CORE_NAMESPACE, "HERD")
+        self.assertIsNotNone(dt)