enh: internal basin features now count as basins_innate

paulmueller · paulmueller · commit f5087dd1db2b · 2026-04-08T15:53:13.000+02:00
diff --git a/CHANGELOG b/CHANGELOG
@@ -2,6 +2,7 @@
  - fix: dataset `identifier` is session-unique for `RTDC_HTTP`
  - fix: dataset `hash` must be globally unique for `RTDC_HTTP` and `RTDC_S3`
  - fix: strip quotes from ETag in `S3File`
+ - enh: internal basin features now count as `basins_innate`
  - enh: add unified `logger` property for all instances of `RTDCBase`
 0.71.3
  - enh: disable HDF5 file locking for basic reading operations
diff --git a/dclab/cli/task_condense.py b/dclab/cli/task_condense.py
@@ -101,10 +101,27 @@ def condense_dataset(
         ancillaries: bool = None,
         store_ancillary_features: bool = True,
         store_basin_features: bool = True,
-        warnings_list: List = None):
+        warnings_list: List[str] | None = None):
     """Condense a dataset using low-level HDF5 methods
 
     For ancillary and basin features, high-level dclab methods are used.
+
+    Parameters
+    ----------
+    ds:
+        dataset from which to export
+    h5_cond:
+        HDF5 file to which data are written
+    ancillaries: bool
+        DEPRECATED, use `store_ancillary_features` instead
+    store_ancillary_features: bool
+        compute and store ancillary features in the output file
+    store_basin_features: bool
+        copy basin features from the input path to the output file;
+        Note that the basin information (including any internal
+        basin dataset) are always copied over to the new dataset.
+    warngins_list:
+        List of warnings that should be written to the output file
     """
     if ancillaries is not None:
         warnings.warn("Please use `store_ancillary_features` instead of "
@@ -182,7 +199,8 @@ def condense_dataset(
         # Write all remaining scalar features to the file
         # (these are *all* scalar features in the case of .tdms data).
         for feat in features:
-            if feat not in h5_cond["events"]:
+            if (feat not in h5_cond.get("events", {})
+                    and feat not in h5_cond.get("basin_events", {})):
                 hw.store_feature(feat=feat, data=ds[feat])
 
         # collect warnings log
diff --git a/dclab/rtdc_dataset/copier.py b/dclab/rtdc_dataset/copier.py
@@ -2,9 +2,10 @@
 from __future__ import annotations
 
 import json
+import multiprocessing as mp
 from multiprocessing.sharedctypes import Synchronized
 import re
-from typing import List, Literal
+from typing import Literal
 
 import h5py
 import h5py.h5o
@@ -21,13 +22,13 @@
 
 def rtdc_copy(src_h5file: h5py.Group,
               dst_h5file: h5py.Group,
-              features: List[str] | Literal['all', 'scalar', 'none'] = "all",
+              features: list[str] | Literal['all', 'scalar', 'none'] = "all",
               include_basins: bool = True,
               include_logs: bool = True,
               include_tables: bool = True,
               meta_prefix: str = "",
-              bytes_total: Synchronized[int] = None,
-              bytes_written: Synchronized[int] = None,
+              bytes_total: Synchronized[int] | None = None,
+              bytes_written: Synchronized[int] | None = None,
               ):
     """Create a compressed copy of an RT-DC file
 
@@ -63,14 +64,11 @@ def rtdc_copy(src_h5file: h5py.Group,
 
     # identify features in source file
     if "events" in src_h5file:
-        events_src = list(src_h5file["events"].keys())
+        events_src = (list(src_h5file.get("events", {}).keys())
+                      + list(src_h5file.get("basin_events", {}).keys()))
     else:
         events_src = []
 
-    if include_basins and "basin_events" in src_h5file:
-        events_src += list(src_h5file["basin_events"].keys())
-        events_src = sorted(set(events_src))
-
     # actual features to copy
     if isinstance(features, list):
         feature_iter = features
@@ -86,17 +84,27 @@ def rtdc_copy(src_h5file: h5py.Group,
                          f"or one of 'all', 'scalar' or 'none', got "
                          f"'{features}'")
 
-    # additional check for basin features.
+    # copy internal basins
+    basin_feat, basin_bytes = internal_basin_events_copy(
+        src_h5file=src_h5file,
+        dst_h5file=dst_h5file,
+        features=feature_iter,
+    )
+
+    # remove features that are already written to the output file
+    for feat in basin_feat:
+        if feat in feature_iter:
+            feature_iter.remove(feat)
+
     bn_regexp = re.compile("^basinmap[0-9]*$")  # future-proof regexp
     src_basin_feats = [f for f in events_src if bn_regexp.match(f)]
     if include_basins:
-        # Make sure all 'basinmap?' features are included in the output file.
+        # Explicitly include all remaining basinmap features.
         for feat in src_basin_feats:
-            if feat not in feature_iter:
+            if feat not in feature_iter and feat not in basin_feat:
                 feature_iter.append(feat)
     else:
-        # We do not need the basinmap features, because basins are
-        # stripped from the output file.
+        # Remove all remaining basinmap features from the list.
         for feat in src_basin_feats:
             if feat in feature_iter:
                 feature_iter.remove(feat)
@@ -106,12 +114,15 @@ def rtdc_copy(src_h5file: h5py.Group,
         if include_tables and "tables" in src_h5file:
             bytes_total.value += get_size(src_h5file["tables"])
 
-        for feat in feature_iter:
+        for feat in feature_iter + basin_feat:
             if feat in src_h5file["events"]:
                 bytes_total.value += get_size(src_h5file["events"][feat])
-            elif include_basins and f"basin_events/{feat}" in src_h5file:
+            elif f"basin_events/{feat}" in src_h5file:
                 bytes_total.value += get_size(src_h5file["basin_events"][feat])
 
+        if bytes_written is not None:
+            bytes_written.value += basin_bytes
+
     # copy logs
     if include_logs and "logs" in src_h5file:
         dst_h5file.require_group("logs")
@@ -148,7 +159,7 @@ def rtdc_copy(src_h5file: h5py.Group,
                               dst_h5file=dst_h5file,
                               features_iter=feature_iter)
 
-    # copy features
+    # copy regular event features
     if feature_iter:
         dst_h5file.require_group("events")
         for feat in feature_iter:
@@ -185,16 +196,57 @@ def rtdc_copy(src_h5file: h5py.Group,
                         if attr not in dst.attrs:
                             dst.attrs[attr] = ufunc(dst)
 
-            elif include_basins and f"basin_events/{feat}" in src_h5file:
-                # Also copy internal basins which should have been defined
-                # in the "basin_events" group.
-                if feat in src_h5file["basin_events"]:
-                    h5ds_copy(src_loc=src_h5file["basin_events"],
-                              src_name=feat,
-                              dst_loc=dst_h5file.require_group("basin_events"),
-                              dst_name=feat,
-                              bytes_written=bytes_written,
-                              )
+
+def internal_basin_events_copy(
+        src_h5file: h5py.Group,
+        dst_h5file: h5py.Group,
+        features: list[str],
+        ) -> tuple[list[str], int]:
+    """Copy internal basin data from the input to the output file
+
+    The basin dictionaries are read and only the `basinmap` features
+    that are required are copied to the output file.
+    """
+    basin_feat = []
+    basin_bytes_mp = mp.Value("L")
+
+    bn_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
+
+    for bn in bn_dicts:
+        if bn["type"] == "internal":
+            bn_feats = []
+            for feat in bn["features"]:
+                if feat in features and f"basin_events/{feat}" in src_h5file:
+                    bn_feats.append(feat)
+                    h5ds_copy(
+                        src_loc=src_h5file["basin_events"],
+                        src_name=feat,
+                        dst_loc=dst_h5file.require_group("basin_events"),
+                        dst_name=feat,
+                        bytes_written=basin_bytes_mp,
+                        )
+            if bn_feats:
+                # Note down features that we added
+                basin_feat += bn_feats
+                # Write basinmap feature
+                if bn["mapping"].startswith("basinmap"):
+                    basin_feat.append(bn["mapping"])
+                    h5ds_copy(
+                        src_loc=src_h5file["events"],
+                        src_name=bn["mapping"],
+                        dst_loc=dst_h5file.require_group("events"),
+                        dst_name=bn["mapping"],
+                        bytes_written=basin_bytes_mp,
+                    )
+                # Rewrite basin definition
+                bn["features"] = bn_feats
+                # Convert edited `bn` to JSON and write feature data
+                b_lines = json.dumps(bn, indent=2).split("\n")
+                key = hashobj(b_lines)
+                if key not in dst_h5file.require_group("basins"):
+                    with RTDCWriter(dst_h5file) as hw:
+                        hw.write_text(dst_h5file["basins"], key, b_lines)
+    return list(set(basin_feat)), basin_bytes_mp.value
 
 
 def basin_definition_copy(src_h5file, dst_h5file, features_iter):
@@ -207,11 +259,17 @@ def basin_definition_copy(src_h5file, dst_h5file, features_iter):
 
     The `features_iter` list of features defines which features are
     relevant for the internal basin.
+
+    To copy internal basins, use :func:`internal_basin_events_copy`.
     """
     dst_h5file.require_group("basins")
     # Load the basin information
     basin_dicts = RTDC_HDF5.basin_get_dicts_from_h5file(src_h5file)
     for bn in basin_dicts:
+        if bn["type"] == "internal":
+            # handled in `internal_basin_events_copy`
+            continue
+
         b_key = bn["key"]
 
         if b_key in dst_h5file["basins"]:
@@ -225,44 +283,20 @@ def basin_definition_copy(src_h5file, dst_h5file, features_iter):
                 f"{src_h5file} does not contain basin {b_key} which I got "
                 f"from `RTDC_HDF5.basin_get_dicts_from_h5file`.")
 
-        if bn["type"] == "internal":
-            # Make sure we define the internal features selected
-            feat_used = [f for f in bn["features"] if f in features_iter]
-            if len(feat_used) == 0:
-                # We don't have any internal features, don't write anything
-                continue
-            elif feat_used != bn["features"]:
-                bn["features"] = feat_used
-                rewrite = True
-            else:
-                rewrite = False
-        else:
-            # We do not have an internal basin, just copy everything
-            rewrite = False
-
-        if rewrite:
-            # Convert edited `bn` to JSON and write feature data
-            b_lines = json.dumps(bn, indent=2).split("\n")
-            key = hashobj(b_lines)
-            if key not in dst_h5file["basins"]:
-                with RTDCWriter(dst_h5file) as hw:
-                    hw.write_text(dst_h5file["basins"], key, b_lines)
-        else:
-            # copy only
-            h5ds_copy(src_loc=src_h5file["basins"],
-                      src_name=b_key,
-                      dst_loc=dst_h5file["basins"],
-                      dst_name=b_key,
-                      recursive=False)
+        h5ds_copy(src_loc=src_h5file["basins"],
+                  src_name=b_key,
+                  dst_loc=dst_h5file["basins"],
+                  dst_name=b_key,
+                  recursive=False)
 
 
 def h5ds_copy(src_loc: h5py.Group,
               src_name: str,
               dst_loc: h5py.Group,
-              dst_name: str = None,
+              dst_name: str | None = None,
               ensure_compression: bool = True,
               recursive: bool = True,
-              bytes_written: Synchronized[int] = None,
+              bytes_written: Synchronized[int] | None = None,
               ):
     """Copy an HDF5 Dataset from one group to another
 
diff --git a/dclab/rtdc_dataset/core.py b/dclab/rtdc_dataset/core.py
@@ -454,8 +454,16 @@ def features_basin(self):
 
     @property
     def features_innate(self):
-        """All features excluding ancillary, basin, or temporary features"""
+        """All features excluding ancillary, basin, or temporary features
+
+        Internal basin features are included since version 0.71.4.
+        """
         innate = [ft for ft in self.features if ft in self._events]
+        for bn_dict in self.basins_get_dicts():
+            if bn_dict.get("type") == "internal":
+                for feat in bn_dict.get("features", []):
+                    if feat in self.features_basin:
+                        innate.append(feat)
         return innate
 
     @property
diff --git a/tests/test_cli_compress.py b/tests/test_cli_compress.py
@@ -133,7 +133,7 @@ def test_compress_basin_internal():
     # sanity check
     with new_dataset(h5path_small) as ds:
         assert "userdef1" in ds.features_basin
-        assert "userdef1" not in ds.features_innate
+        assert "userdef1" in ds.features_innate
 
     # compress the basin-based dataset
     cli.compress(path_in=h5path_small, path_out=h5path_out)
@@ -146,6 +146,7 @@ def test_compress_basin_internal():
 
     with new_dataset(h5path_out) as ds:
         assert "userdef1" in ds.features_basin
+        assert "userdef1" in ds.features_innate
 
 
 def test_compress_basin_no_data_from_basins():
diff --git a/tests/test_cli_condense.py b/tests/test_cli_condense.py
@@ -165,9 +165,9 @@ def test_condense_basins_internal(store_basins):
     # sanity check
     with new_dataset(h5path_small) as ds:
         assert "userdef1" in ds.features_basin
-        assert "userdef1" not in ds.features_innate
+        assert "userdef1" in ds.features_innate
         assert "image_bg" in ds.features_basin
-        assert "image_bg" not in ds.features_innate
+        assert "image_bg" in ds.features_innate
 
     # condense the basin-based dataset
     cli.condense(path_in=h5path_small,
@@ -189,8 +189,10 @@ def test_condense_basins_internal(store_basins):
     with new_dataset(h5path_out) as ds:
         # userdef1 is an internal, scalar basin
         assert "userdef1" in ds.features_basin
+        assert "userdef1" in ds.features_innate
         # image_bg is not scalar, so it should not be here
         assert "image_bg" not in ds.features_basin
+        assert "image_bg" not in ds.features_innate
 
 
 @pytest.mark.filterwarnings(
diff --git a/tests/test_cli_repack.py b/tests/test_cli_repack.py
@@ -69,27 +69,23 @@ def test_repack_basin_internal(use_basins):
     # sanity check
     with new_dataset(h5path_small) as ds:
         assert "userdef1" in ds.features_basin
-        assert "userdef1" not in ds.features_innate
+        assert "userdef1" in ds.features_innate
 
-    # compress the basin-based dataset
+    # repack the basin-based dataset
     cli.repack(path_in=h5path_small, path_out=h5path_out,
                strip_basins=not use_basins)
 
     with h5py.File(h5path_out) as h5:
         assert "deform" in h5["events"], "sanity check"
         # userdef1 should not be in "events" in any case
         assert "userdef1" not in h5["events"]
-        if use_basins:
-            assert "userdef1" in h5["basin_events"]
-            assert np.all(h5["basin_events"]["userdef1"] == np.arange(2))
-        else:
-            assert "basin_events" not in h5
+        # we are repacking the entire dataset, preserving internal basins
+        assert "userdef1" in h5["basin_events"]
+        assert np.all(h5["basin_events"]["userdef1"] == np.arange(2))
 
     with new_dataset(h5path_out) as ds:
-        if use_basins:
-            assert "userdef1" in ds.features_basin
-        else:
-            assert "userdef1" not in ds.features_basin
+        assert "userdef1" in ds.features_basin
+        assert "userdef1" in ds.features_innate
 
 
 @pytest.mark.filterwarnings(
diff --git a/tests/test_rtdc_copier.py b/tests/test_rtdc_copier.py
diff --git a/tests/test_rtdc_export_hdf5.py b/tests/test_rtdc_export_hdf5.py
