Merge pull request #2024 from kili-technology/feature/sup-2586-enabled-us-cloud-extra-label-data-being-returned-for-video

fix(SUP-2586): remove annotations as we now have it in the jsonResponseUrl

Author: FannyGaudin

src/kili/adapters/kili_api_gateway/asset/operations.py

@@ -30,9 +30,3 @@ def get_assets_query(fragment: str) -> str:
   external_ids: filterExistingAssets(projectID: $projectID, externalIDs: $externalIDs)
 }
 """
-
-GQL_COUNT_ASSET_ANNOTATIONS = """
-query countAssetAnnotations($where: AssetWhere!) {
-    data: countAssetAnnotations(where: $where)
-}
-"""

src/kili/adapters/kili_api_gateway/asset/operations_mixin.py

@@ -2,16 +2,11 @@
 
 from collections.abc import Generator
 
-from kili_formats.tool.annotations_to_json_response import (
-    AnnotationsToJsonResponseConverter,
-)
-
 from kili.adapters.kili_api_gateway.asset.formatters import (
     load_asset_json_fields,
 )
 from kili.adapters.kili_api_gateway.asset.mappers import asset_where_mapper
 from kili.adapters.kili_api_gateway.asset.operations import (
-    GQL_COUNT_ASSET_ANNOTATIONS,
     GQL_COUNT_ASSETS,
     GQL_CREATE_UPLOAD_BUCKET_SIGNED_URLS,
     GQL_FILTER_EXISTING_ASSETS,
@@ -23,18 +18,11 @@
     QueryOptions,
     fragment_builder,
 )
-from kili.adapters.kili_api_gateway.label.common import get_annotation_fragment
 from kili.adapters.kili_api_gateway.project.common import get_project
 from kili.core.graphql.operations.asset.mutations import GQL_SET_ASSET_CONSENSUS
 from kili.domain.asset import AssetFilters
 from kili.domain.types import ListOrTuple
 
-# Threshold for batching based on number of annotations
-# This is used to determine whether to use a single batch or multiple batches
-# when fetching assets. If the number of annotations counted exceeds this threshold,
-# the asset fetch will be done in multiple smaller batches to avoid performance issues.
-THRESHOLD_FOR_BATCHING = 200
-
 
 class AssetOperationMixin(BaseOperationMixin):
     """Mixin extending Kili API Gateway class with Assets related operations."""
@@ -46,9 +34,6 @@ def list_assets(
         options: QueryOptions,
     ) -> Generator[dict, None, None]:
         """List assets with given options."""
-        has_labels_url = "labels.jsonResponseUrl" in fields
-        has_latest_label_url = "latestLabel.jsonResponseUrl" in fields
-
         if "labels.jsonResponse" in fields or "latestLabel.jsonResponse" in fields:
             # Check if we can get the jsonResponse of if we need to rebuild it.
             project_info = get_project(
@@ -61,10 +46,7 @@ def list_assets(
                 "LLM_STATIC",
                 "GEOSPATIAL",
             }:
-                fetch_annotations = not (has_labels_url or has_latest_label_url)
-                yield from self.list_assets_split(
-                    filters, fields, options, project_info, fetch_annotations
-                )
+                yield from self.list_assets_split(filters, fields, options, project_info)
                 return
 
         fragment = fragment_builder(fields)
@@ -90,35 +72,24 @@ def list_assets_split(
         fields: ListOrTuple[str],
         options: QueryOptions,
         project_info,
-        fetch_annotations: bool,
     ) -> Generator[dict, None, None]:
         """List assets with given options."""
-        nb_annotations = self.count_assets_annotations(filters)
         assets_batch_max_amount = 10 if project_info["inputType"] == "VIDEO" else 50
         batch_size_to_use = min(options.batch_size, assets_batch_max_amount)
-        batch_size = (
-            1 if nb_annotations / batch_size_to_use > THRESHOLD_FOR_BATCHING else batch_size_to_use
-        )
 
-        options = QueryOptions(options.disable_tqdm, options.first, options.skip, batch_size)
-
-        static_fragments = {}
-        if fetch_annotations:
-            inner_annotation_fragment = get_annotation_fragment()
-            annotation_fragment = f"""
-                annotations {{
-                    {inner_annotation_fragment}
-                }}
-            """
-            static_fragments = {"labels": annotation_fragment, "latestLabel": annotation_fragment}
-
-            required_fields = {"content", "jsonContent", "resolution.width", "resolution.height"}
-            fields = list(fields)
-            for field in required_fields:
-                if field not in fields:
-                    fields.append(field)
-
-        fragment = fragment_builder(fields, static_fragments if static_fragments else None)
+        options = QueryOptions(options.disable_tqdm, options.first, options.skip, batch_size_to_use)
+
+        required_fields = {"content", "jsonContent", "resolution.width", "resolution.height"}
+        if "labels.jsonResponse" in fields:
+            required_fields.add("labels.jsonResponseUrl")
+        if "latestLabel.jsonResponse" in fields:
+            required_fields.add("latestLabel.jsonResponseUrl")
+        fields = list(fields)
+        for field in required_fields:
+            if field not in fields:
+                fields.append(field)
+
+        fragment = fragment_builder(fields)
         query = get_assets_query(fragment)
         where = asset_where_mapper(filters)
         assets_gen = PaginatedGraphQLQuery(self.graphql_client).execute_query_from_paginated_call(
@@ -128,28 +99,7 @@ def list_assets_split(
             load_asset_json_fields(asset, fields, self.http_client) for asset in assets_gen
         )
 
-        if fetch_annotations:
-            converter = AnnotationsToJsonResponseConverter(
-                json_interface=project_info["jsonInterface"],
-                project_input_type=project_info["inputType"],
-            )
-            is_requesting_annotations = any("annotations." in element for element in fields)
-            for asset in assets_gen:
-                if "latestLabel.jsonResponse" in fields and asset.get("latestLabel"):
-                    converter.patch_label_json_response(
-                        asset, asset["latestLabel"], asset["latestLabel"]["annotations"]
-                    )
-                    if not is_requesting_annotations:
-                        asset["latestLabel"].pop("annotations")
-
-                if "labels.jsonResponse" in fields:
-                    for label in asset.get("labels", []):
-                        converter.patch_label_json_response(asset, label, label["annotations"])
-                        if not is_requesting_annotations:
-                            label.pop("annotations")
-                yield asset
-        else:
-            yield from assets_gen
+        yield from assets_gen
 
     def count_assets(self, filters: AssetFilters) -> int:
         """Send a GraphQL request calling countIssues resolver."""
@@ -176,14 +126,6 @@ def filter_existing_assets(self, project_id: str, assets_external_ids: ListOrTup
         external_id_response = self.graphql_client.execute(GQL_FILTER_EXISTING_ASSETS, payload)
         return external_id_response["external_ids"]
 
-    def count_assets_annotations(self, filters: AssetFilters) -> int:
-        """Count the number of annotations for assets matching the filters."""
-        where = asset_where_mapper(filters)
-        payload = {"where": where}
-        count_result = self.graphql_client.execute(GQL_COUNT_ASSET_ANNOTATIONS, payload)
-        count: int = count_result["data"]
-        return count
-
     def update_asset_consensus(
         self,
         project_id: str,

src/kili/adapters/kili_api_gateway/label/common.py

@@ -4,9 +4,6 @@
 from kili.adapters.kili_api_gateway.asset.formatters import load_asset_json_fields
 from kili.adapters.kili_api_gateway.asset.operations import get_assets_query
 from kili.adapters.kili_api_gateway.helpers.queries import fragment_builder
-from kili.adapters.kili_api_gateway.label.operations import (
-    get_annotations_partial_query,
-)
 from kili.core.graphql.graphql_client import GraphQLClient
 from kili.domain.asset.asset import AssetId
 from kili.domain.types import ListOrTuple
@@ -32,47 +29,3 @@ def get_asset(
             f"asset ID: {asset_id}. The asset does not exist or you do not have access to it."
         )
     return load_asset_json_fields(assets[0], fields, http_client=http_client)
-
-
-def get_annotation_fragment():
-    """Generates a fragment to get all annotations and their values."""
-    return get_annotations_partial_query(
-        annotation_fragment=fragment_builder(("__typename", "id", "job", "path", "labelId")),
-        classification_annotation_fragment=fragment_builder(
-            ("annotationValue.categories", "chatItemId")
-        ),
-        comparison_annotation_fragment=fragment_builder(
-            (
-                "annotationValue.choice.code",
-                "annotationValue.choice.firstId",
-                "annotationValue.choice.secondId",
-                "chatItemId",
-            )
-        ),
-        transcription_annotation_fragment=fragment_builder(("annotationValue.text", "chatItemId")),
-        video_annotation_fragment=fragment_builder(
-            (
-                "frames.start",
-                "frames.end",
-                "keyAnnotations.id",
-                "keyAnnotations.frame",
-            )
-        ),
-        video_classification_annotation_fragment=fragment_builder(
-            ("keyAnnotations.annotationValue.categories",)
-        ),
-        video_object_detection_annotation_fragment=fragment_builder(
-            (
-                "keyAnnotations.annotationValue.vertices: verticesScalar",
-                "name",
-                "mid",
-                "category",
-            )
-        ),
-        video_transcription_annotation_fragment=fragment_builder(
-            ("keyAnnotations.annotationValue.text",)
-        ),
-        object_detection_annotation_fragment=fragment_builder(
-            ("category", "mid", "name", "annotationValue.vertices: verticesScalar")
-        ),
-    )

src/kili/adapters/kili_api_gateway/label/operations.py

@@ -71,82 +71,6 @@ def get_append_to_labels_mutation(fragment: str) -> str:
 """
 
 
-def get_annotations_partial_query(
-    annotation_fragment: str,
-    classification_annotation_fragment: str,
-    comparison_annotation_fragment: str,
-    transcription_annotation_fragment: str,
-    video_annotation_fragment: str,
-    video_object_detection_annotation_fragment: str,
-    video_classification_annotation_fragment: str,
-    video_transcription_annotation_fragment: str,
-    object_detection_annotation_fragment: str,
-) -> str:
-    """Get the gql annotations query."""
-    inline_fragments = ""
-
-    if classification_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on ClassificationAnnotation {{
-                {classification_annotation_fragment}
-            }}
-        """
-
-    if comparison_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on ComparisonAnnotation {{
-                {comparison_annotation_fragment}
-            }}
-        """
-
-    if transcription_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on TranscriptionAnnotation {{
-                {transcription_annotation_fragment}
-            }}
-        """
-
-    if video_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on VideoAnnotation {{
-                    {video_annotation_fragment}
-            }}
-        """
-
-    if video_object_detection_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on VideoObjectDetectionAnnotation {{
-                {video_object_detection_annotation_fragment}
-            }}
-        """
-
-    if video_classification_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on VideoClassificationAnnotation {{
-                {video_classification_annotation_fragment}
-            }}
-        """
-
-    if video_transcription_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on VideoTranscriptionAnnotation {{
-                {video_transcription_annotation_fragment}
-            }}
-        """
-
-    if object_detection_annotation_fragment.strip():
-        inline_fragments += f"""
-            ... on ObjectDetectionAnnotation {{
-                {object_detection_annotation_fragment}
-            }}
-        """
-
-    return f"""
-            {annotation_fragment}
-            {inline_fragments}
-    """
-
-
 GQL_COPY_LABELS = """
   mutation CopyLabels($data: CopyLabelsInput!) {
     copyLabels(data: $data)