[lts] fix(LAB-3755): add width and height in patch_label_json_respons… (#1914)

Author: FannyGaudin

src/kili/adapters/kili_api_gateway/asset/operations_mixin.py

@@ -21,7 +21,9 @@
 from kili.adapters.kili_api_gateway.label.annotation_to_json_response import (
     AnnotationsToJsonResponseConverter,
 )
-from kili.adapters.kili_api_gateway.label.common import get_annotation_fragment
+from kili.adapters.kili_api_gateway.label.common import (
+    get_annotation_fragment,
+)
 from kili.adapters.kili_api_gateway.project.common import get_project
 from kili.domain.asset import AssetFilters
 from kili.domain.types import ListOrTuple
@@ -104,14 +106,16 @@ def list_assets_split(
         for asset in assets_gen:
             if "latestLabel.jsonResponse" in fields and asset.get("latestLabel"):
                 converter.patch_label_json_response(
-                    asset["latestLabel"], asset["latestLabel"]["annotations"]
+                    asset,
+                    asset["latestLabel"],
+                    asset["latestLabel"]["annotations"],
                 )
                 if not is_requesting_annotations:
                     asset["latestLabel"].pop("annotations")
 
             if "labels.jsonResponse" in fields:
                 for label in asset.get("labels", []):
-                    converter.patch_label_json_response(label, label["annotations"])
+                    converter.patch_label_json_response(asset, label, label["annotations"])
                     if not is_requesting_annotations:
                         label.pop("annotations")
             yield asset

src/kili/adapters/kili_api_gateway/label/annotation_to_json_response.py

@@ -4,6 +4,7 @@
 from dataclasses import dataclass
 from typing import Dict, Generator, List, Optional, Tuple, TypeVar, Union, cast, overload
 
+from kili.core.helpers import get_video_and_frame_dimensions
 from kili.domain.annotation import (
     ClassicAnnotation,
     ClassificationAnnotation,
@@ -23,6 +24,32 @@
 ASSET_LEVEL_KEY = "assetLevel"
 
 
+def convert_from_normalized_to_absolute(
+    vertices: List[Vertice], width: int, height: int
+) -> List[Vertice]:
+    """Convert normalized vertices to absolute coordinates."""
+    return [
+        Vertice(
+            x=vertex["x"] * width,
+            y=vertex["y"] * height,
+        )
+        for vertex in vertices
+    ]
+
+
+def convert_from_absolute_to_normalized(
+    vertices: List[Vertice], width: int, height: int
+) -> List[Vertice]:
+    """Convert absolute vertices to normalized coordinates."""
+    return [
+        Vertice(
+            x=vertex["x"] / width,
+            y=vertex["y"] / height,
+        )
+        for vertex in vertices
+    ]
+
+
 class AnnotationsToJsonResponseConverter:
     """Convert annotations to JSON response."""
 
@@ -47,7 +74,10 @@ def _label_has_json_response_data(self, label: Dict) -> bool:
         return False
 
     def patch_label_json_response(
-        self, label: Dict, annotations: Union[List[VideoAnnotation], List[ClassicAnnotation]]
+        self,
+        asset: Optional[Dict],
+        label: Dict,
+        annotations: Union[List[VideoAnnotation], List[ClassicAnnotation]],
     ) -> None:
         """Patch the label json response using the annotations.
 
@@ -58,9 +88,17 @@ def patch_label_json_response(
                 return
 
             if self._project_input_type == "VIDEO":
+                if not asset:
+                    raise ValueError(
+                        "Asset is required for video annotations to compute dimensions."
+                    )
+                width, height = get_video_and_frame_dimensions(asset)
                 annotations = cast(List[VideoAnnotation], annotations)
                 converted_json_resp = _video_annotations_to_json_response(
-                    annotations=annotations, json_interface=self._project_json_interface
+                    annotations=annotations,
+                    json_interface=self._project_json_interface,
+                    width=width,
+                    height=height,
                 )
             else:
                 annotations = cast(List[ClassicAnnotation], annotations)
@@ -91,7 +129,7 @@ def _fill_empty_frames(json_response: Dict) -> None:
 
 
 def _video_annotations_to_json_response(
-    annotations: List[VideoAnnotation], json_interface: Dict
+    annotations: List[VideoAnnotation], json_interface: Dict, width: int, height: int
 ) -> Dict[str, Dict[JobName, Dict]]:
     """Convert video label annotations to a video json response."""
     json_resp = defaultdict(dict)
@@ -110,7 +148,7 @@ def _video_annotations_to_json_response(
         elif ann["__typename"] == "VideoObjectDetectionAnnotation":
             ann = cast(VideoObjectDetectionAnnotation, ann)
             ann_json_resp = _video_object_detection_annotation_to_json_response(
-                ann, other_annotations, json_interface=json_interface
+                ann, other_annotations, json_interface=json_interface, width=width, height=height
             )
             for frame_id, frame_json_resp in ann_json_resp.items():
                 for job_name, job_resp in frame_json_resp.items():
@@ -532,6 +570,8 @@ def _video_object_detection_annotation_to_json_response(
     annotation: VideoObjectDetectionAnnotation,
     other_annotations: List[VideoAnnotation],
     json_interface: Dict,
+    width: int,
+    height: int,
 ) -> Dict[str, Dict[JobName, Dict]]:
     # get the child annotations of the current annotation
     # and compute the json response of those child jobs
@@ -569,6 +609,8 @@ def _video_object_detection_annotation_to_json_response(
                     object_final_state=object_final_state,
                     final_state_frame_index=next_key_ann["frame"],
                     at_frame=frame_id,
+                    width=width,
+                    height=height,
                 )
 
             if json_interface["jobs"][annotation["job"]]["tools"][0] == "marker":
@@ -616,6 +658,8 @@ def _interpolate_object(
     object_final_state: List[List[List[Vertice]]],
     final_state_frame_index: int,
     at_frame: int,
+    width: int,
+    height: int,
 ) -> List[List[List[Vertice]]]:
     """Interpolate an object between two key frames."""
     # if the two frames are consecutive, we do not interpolate
@@ -648,6 +692,8 @@ def _interpolate_object(
                     next_vertices=object_final_state[0][0],
                     weight=(at_frame - initial_state_frame_index)
                     / (final_state_frame_index - initial_state_frame_index),
+                    width=width,
+                    height=height,
                 )
             ]
         ]
@@ -674,6 +720,8 @@ def _interpolate_rectangle(
     previous_vertices: List[Vertice],
     next_vertices: List[Vertice],
     weight: float,
+    width: int,
+    height: int,
 ) -> List[Vertice]:
     """Interpolate a rectangle.
 
@@ -683,9 +731,18 @@ def _interpolate_rectangle(
     The interpolated properties are used to reconstruct the vertices of the interpolated rectangle,
     which are then converted back to normalized coordinates.
     """
-    permuted_new_vertices = _find_rectangle_vertices_bijection(previous_vertices, next_vertices)
+    previous_absolute_vertices = convert_from_normalized_to_absolute(
+        previous_vertices, height=height, width=width
+    )
+    next_absolute_vertices = convert_from_normalized_to_absolute(
+        next_vertices, height=height, width=width
+    )
+
+    permuted_new_vertices = _find_rectangle_vertices_bijection(
+        previous_absolute_vertices, next_absolute_vertices
+    )
 
-    previous_rectangle_properties = _find_rectangle_properties(previous_vertices)
+    previous_rectangle_properties = _find_rectangle_properties(previous_absolute_vertices)
     next_rectangle_properties = _find_rectangle_properties(permuted_new_vertices)
 
     interpolated_angle = _interpolate_angle(
@@ -712,7 +769,7 @@ def _interpolate_rectangle(
         interpolated_rectangle_properties
     )
 
-    return interpolated_rectangle
+    return convert_from_absolute_to_normalized(interpolated_rectangle, height=height, width=width)
 
 
 def _find_rectangle_vertices_bijection(

src/kili/adapters/kili_api_gateway/label/common.py

@@ -1,8 +1,38 @@
 """Label gateway common."""
+from typing import Dict
+
+from kili.adapters.http_client import HttpClient
+from kili.adapters.kili_api_gateway.asset.formatters import load_asset_json_fields
+from kili.adapters.kili_api_gateway.asset.operations import get_assets_query
 from kili.adapters.kili_api_gateway.helpers.queries import fragment_builder
 from kili.adapters.kili_api_gateway.label.operations import (
     get_annotations_partial_query,
 )
+from kili.core.graphql.graphql_client import GraphQLClient
+from kili.domain.asset.asset import AssetId
+from kili.domain.types import ListOrTuple
+from kili.exceptions import NotFound
+
+
+def get_asset(
+    graphql_client: GraphQLClient,
+    http_client: HttpClient,
+    asset_id: AssetId,
+    fields: ListOrTuple[str],
+) -> Dict:
+    """Get asset."""
+    fragment = fragment_builder(fields)
+    query = get_assets_query(fragment)
+    result = graphql_client.execute(
+        query=query, variables={"where": {"id": asset_id}, "first": 1, "skip": 0}
+    )
+    assets = result["data"]
+
+    if len(assets) == 0:
+        raise NotFound(
+            f"asset ID: {asset_id}. The asset does not exist or you do not have access to it."
+        )
+    return load_asset_json_fields(assets[0], fields, http_client=http_client)
 
 
 def get_annotation_fragment():

src/kili/adapters/kili_api_gateway/label/operations_mixin.py

@@ -21,7 +21,7 @@
 from .annotation_to_json_response import (
     AnnotationsToJsonResponseConverter,
 )
-from .common import get_annotation_fragment
+from .common import get_annotation_fragment, get_asset
 from .formatters import load_label_json_fields
 from .mappers import (
     append_label_data_mapper,
@@ -56,6 +56,8 @@ def list_labels(
     ) -> Generator[Dict, None, None]:
         """List labels."""
         if "jsonResponse" in fields:
+            if "labelOf" not in fields:
+                fields = [*list(fields), "assetId"]
             project_info = get_project(
                 self.graphql_client, filters.project_id, ("inputType", "jsonInterface")
             )
@@ -107,7 +109,16 @@ def list_labels_split(
                 project_input_type=project_info["inputType"],
             )
             for label in labels_gen:
-                converter.patch_label_json_response(label, label["annotations"])
+                asset = None
+                if project_info["inputType"] == "VIDEO":
+                    asset = get_asset(
+                        self.graphql_client,
+                        self.http_client,
+                        label["assetId"],
+                        ["content", "jsonContent", "resolution.width", "resolution.height"],
+                    )
+
+                converter.patch_label_json_response(asset, label, label["annotations"])
                 if "annotations" not in fields:
                     label.pop("annotations")
                 yield label

src/kili/core/helpers.py

@@ -8,15 +8,19 @@
 import re
 import warnings
 from json import dumps, loads
-from typing import Any, Callable, Dict, List, Optional, Type, TypeVar, Union
+from pathlib import Path
+from typing import Any, Callable, Dict, List, Optional, Tuple, Type, TypeVar, Union
 
+import ffmpeg
 import pyparsing as pp
 import requests
 import tenacity
+from PIL import Image
 from typing_extensions import get_args, get_origin
 
 from kili.adapters.http_client import HttpClient
 from kili.core.constants import mime_extensions_for_IV2
+from kili.exceptions import NotFound
 from kili.log.logging import logger
 
 T = TypeVar("T")
@@ -368,3 +372,69 @@ def get_response_json(response: requests.Response) -> dict:
     except json.JSONDecodeError:
         logger.exception("An error occurred while decoding the json response")
         return {}
+
+
+def _get_image_dimensions(filepath: str) -> Tuple:
+    """Get an image width and height."""
+    image = Image.open(filepath)
+    return image.size
+
+
+def get_frame_dimensions(asset: Dict) -> Tuple:
+    """Get a video asset frame width and height."""
+    if "resolution" in asset and asset["resolution"] is not None:
+        return (asset["resolution"]["width"], asset["resolution"]["height"])
+
+    if (
+        isinstance(asset["jsonContent"], list)
+        and len(asset["jsonContent"]) > 0
+        and Path(asset["jsonContent"][0]).is_file()
+    ):
+        return _get_image_dimensions(asset["jsonContent"][0])
+
+    raise NotFound(
+        f"Could not find dimensions for asset with externalId '{asset['externalId']}'. Please"
+        " use `kili.update_properties_in_assets()` to update the resolution of your asset. Or use"
+        " `kili.export_labels(with_assets=True).`"
+    )
+
+
+def get_video_and_frame_dimensions(
+    asset: Dict,
+) -> Tuple[int, int]:
+    """Get a video width and height, and a frame width and height."""
+    width = height = 0
+    if "resolution" in asset and asset["resolution"] is not None:
+        width = asset["resolution"]["width"]
+        height = asset["resolution"]["height"]
+        return width, height
+    if isinstance(asset["jsonContent"], list) and Path(asset["jsonContent"][0]).is_file():
+        width, height = get_frame_dimensions(asset)
+    elif Path(asset["content"]).is_file():
+        width, height = get_video_dimensions(asset)
+    else:
+        raise FileNotFoundError(
+            f"Could not find video dimensions for asset with externalId '{asset['externalId']}'. Please"
+            " use `kili.update_properties_in_assets()` to update the resolution of your asset. Or use"
+            " `kili.export_labels(with_assets=True).`"
+        )
+    return width, height
+
+
+def get_video_dimensions(asset: Dict) -> Tuple:
+    """Get a video width and height."""
+    if "resolution" in asset and asset["resolution"] is not None:
+        return (asset["resolution"]["width"], asset["resolution"]["height"])
+
+    if Path(asset["content"]).is_file():
+        probe = ffmpeg.probe(str(asset["content"]))
+        video_info = next(s for s in probe["streams"] if s["codec_type"] == "video")
+        width = video_info["width"]
+        height = video_info["height"]
+        return width, height
+
+    raise NotFound(
+        f"Could not find video dimensions for asset with externalId '{asset['externalId']}'. Please"
+        " use `kili.update_properties_in_assets()` to update the resolution of your asset. Or use"
+        " `kili.export_labels(with_assets=True).`"
+    )