Containers: batch container data-model column tag retrieval to avoid subtree fan-out (open-metadata#27836)

* Containers with deep nesting causing performance issues due to tag fetch

* Batch derived-tag fetch across data-model columns

populateDataModelColumnTags previously called addDerivedTagsGracefully
once per flattened column, which internally batches across that column's
own tags but issues a separate derived-tag DB lookup for every column.
On data models with many columns (or struct types with deep nesting)
this becomes an N+1 pattern.

Refactor:
- Pre-compute Map<String, Column> hashToColumn once (LinkedHashMap to
  preserve column order) so we no longer hash each FQN twice — once
  for the target-hash list and again on lookup.
- After fetching tags by target hash, flatten all returned TagLabels
  into a single list and call TagLabelUtil.batchFetchDerivedTags(...)
  once for the whole data model.
- Per column, use addDerivedTagsWithPreFetched(columnTags, derivedMap)
  to avoid further DB lookups.
- Fall back to the per-column addDerivedTagsGracefully path if the
  batch derived-tag fetch raises, preserving existing semantics.

Net effect: total derived-tag DB queries drop from O(N) to 1 regardless
of column count or nesting depth.


Co-authored-by: sonika-shah <58761340+sonika-shah@users.noreply.github.com>

Author: harshach

conf/openmetadata.yaml

@@ -194,22 +194,6 @@ logging:
           archivedFileCount: 7
           timeZone: UTC
           maxFileSize: 50MB
-    org.openmetadata.slowrequest:
-      level: ${SLOW_REQUEST_LOG_LEVEL:-OFF}
-      additive: false
-      appenders:
-        - type: file
-          layout:
-            type: om-event-layout
-            format: ${LOG_FORMAT:-text}
-            pattern: "%level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
-            appendLineSeparator: true
-          threshold: WARN
-          currentLogFilename: ./logs/slow-requests.log
-          archivedLogFilenamePattern: ./logs/slow-requests-%d{yyyy-MM-dd}-%i.log.gz
-          archivedFileCount: 7
-          timeZone: UTC
-          maxFileSize: 50MB
     org.openmetadata.service.util.OpenMetadataSetup:
       level: INFO
       appenders:

ingestion/src/metadata/ingestion/ometa/mixins/container_mixin.py

@@ -21,6 +21,8 @@
 from metadata.generated.schema.entity.data.container import Container
 from metadata.generated.schema.entity.data.table import TableData
 from metadata.ingestion.ometa.client import REST
+from metadata.ingestion.ometa.models import EntityList
+from metadata.ingestion.ometa.utils import quote
 from metadata.utils.logger import ometa_logger
 
 logger = ometa_logger()
@@ -105,6 +107,38 @@ def ingest_container_sample_data(self, container: Container, sample_data: TableD
             logger.warning(f"Error trying to PUT sample data for {container.fullyQualifiedName.root}: {exc}")
             return None
 
+    def list_container_children(
+        self,
+        container_fqn: str,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> EntityList[Container]:
+        """
+        Page through the immediate children of a Container via the dedicated
+        ``/v1/containers/name/{fqn}/children`` endpoint. Use this instead of
+        fetching the parent with ``fields=children`` — that field is no longer
+        served because the inline payload is unbounded for buckets with many
+        objects.
+
+        Each row is a slim projection (id, name, displayName, fqn, description,
+        service); ``dataModel``, ``tags``, ``owners``, ``extension`` are not
+        populated. Re-fetch the specific child via :meth:`get_by_name` when
+        full details are needed.
+        """
+        path = f"/containers/name/{quote(container_fqn)}/children?limit={limit}&offset={offset}"
+        resp = self.client.get(path)
+        if not isinstance(resp, dict):
+            return EntityList(entities=[], total=0)
+
+        entities = [Container(**elmt) for elmt in resp.get("data") or []]
+        paging = resp.get("paging") or {}
+        return EntityList(
+            entities=entities,
+            total=paging.get("total", len(entities)),
+            after=paging.get("after"),
+            before=paging.get("before"),
+        )
+
     def get_container_sample_data(self, container: Container) -> Optional[Container]:  # noqa: UP045
         """
         GET call for the /sampleData endpoint for a given Container

ingestion/src/metadata/sdk/entities/containers.py

@@ -2,11 +2,12 @@
 Containers entity SDK with fluent API
 """
 
-from typing import Type  # noqa: UP035
+from typing import List, Type  # noqa: UP035
 
 from metadata.generated.schema.api.data.createContainer import CreateContainerRequest
 from metadata.generated.schema.entity.data.container import Container
-from metadata.sdk.entities.base import BaseEntity
+from metadata.generated.schema.type.entityReference import EntityReference
+from metadata.sdk.entities.base import BaseEntity, EntityList
 
 
 class Containers(BaseEntity[Container, CreateContainerRequest]):
@@ -16,3 +17,53 @@ class Containers(BaseEntity[Container, CreateContainerRequest]):
     def entity_type(cls) -> Type[Container]:  # noqa: UP006
         """Return the Container entity type"""
         return Container
+
+    @classmethod
+    def list_children(
+        cls,
+        container_fqn: str,
+        *,
+        limit: int = 100,
+        offset: int = 0,
+    ) -> EntityList[Container]:
+        """
+        Page through the immediate children of a Container via the dedicated
+        ``/v1/containers/name/{fqn}/children`` endpoint. Use this instead of
+        fetching the parent with ``fields=["children"]`` — that field is no longer
+        served because the inline payload is unbounded for buckets with many
+        objects.
+
+        Each row is a slim projection (id, name, displayName, fqn, description,
+        service); ``dataModel``, ``tags``, ``owners``, ``extension`` are not
+        populated. Re-fetch the specific child via :meth:`retrieve_by_name`
+        when full details are needed.
+        """
+        client = cls._get_client()
+        page = client.list_container_children(container_fqn, limit=limit, offset=offset)
+        entities = [cls._coerce_entity(item) for item in page.entities]
+        return EntityList(
+            entities=entities,
+            after=getattr(page, "after", None),
+            before=getattr(page, "before", None),
+        )
+
+    @classmethod
+    def list_ancestors(cls, container_fqn: str) -> List[EntityReference]:  # noqa: UP006
+        """
+        Resolve the full ancestor chain for a container in a single call.
+        Returns ``EntityReference``s ordered from the root container (immediate
+        child of the storage service) down to the immediate parent of
+        ``container_fqn``. Empty list when the container is at the top level.
+        """
+        client = cls._get_client()
+        rest_client = cls._get_rest_client(client)
+        endpoint = cls._get_endpoint_path(client)
+        from metadata.ingestion.ometa.utils import (  # noqa: PLC0415
+            quote,
+        )
+
+        path = f"{endpoint}/name/{quote(container_fqn)}/ancestors"
+        resp = rest_client.get(path)
+        if not isinstance(resp, list):
+            return []
+        return [EntityReference(**ref) for ref in resp]

ingestion/tests/integration/auto_classification/containers/test_container_classification.py

@@ -30,8 +30,11 @@ def test_containers_ingested(metadata: OpenMetadata, ingest_storage_metadata, se
     """Verify containers were ingested with data models"""
     bucket = metadata.get_by_name(entity=Container, fqn=f"{service_name}.{bucket_name}", fields=["*"])
     assert bucket is not None
-    assert bucket.children is not None
-    assert len(bucket.children.root) >= 3
+
+    # `children` is no longer inlined into the parent payload — it's an unbounded
+    # collection for object stores. Use the dedicated paginated endpoint.
+    children = metadata.list_container_children(f"{service_name}.{bucket_name}")
+    assert len(children.entities) >= 3
 
     customers_container = metadata.get_by_name(
         entity=Container, fqn=f"{service_name}.{bucket_name}.customers", fields=["*"]

ingestion/tests/integration/s3/test_s3_storage.py

@@ -22,9 +22,11 @@ def test_s3_ingestion(metadata, ingest_s3_storage, service_name):
 
     # We should have the bucket and all its structured children
     bucket: Container = metadata.get_by_name(entity=Container, fqn=f"{service_name}.test-bucket", fields=["*"])
-    # The bucket has children and no dataModel
-    assert 7 == len(bucket.children.root)  # noqa: SIM300
+    # The bucket has children (via the dedicated paginated endpoint, not inlined
+    # into the parent payload) and no dataModel
     assert not bucket.dataModel
+    children = metadata.list_container_children(f"{service_name}.test-bucket")
+    assert 7 == len(children.entities)  # noqa: SIM300
 
     # We can validate the children
     cities: Container = metadata.get_by_name(entity=Container, fqn=f"{service_name}.test-bucket.cities", fields=["*"])