feat: add support for input overrides for attachments in conversational evals (#1383)

Author: norman-le

src/uipath/_cli/cli_eval.py

@@ -274,7 +274,9 @@ def eval(
 
         # Load eval set to resolve the path
         eval_set_path = eval_set or EvalHelpers.auto_discover_eval_set()
-        _, resolved_eval_set_path = EvalHelpers.load_eval_set(eval_set_path, eval_ids)
+        _, resolved_eval_set_path = EvalHelpers.load_eval_set(
+            eval_set_path, eval_ids, input_overrides=input_overrides
+        )
 
         eval_context.report_coverage = report_coverage
         eval_context.input_overrides = input_overrides
@@ -346,7 +348,9 @@ async def execute_eval():
 
                     # Load eval set (path is already resolved in cli_eval.py)
                     eval_context.evaluation_set, _ = EvalHelpers.load_eval_set(
-                        resolved_eval_set_path, eval_ids
+                        resolved_eval_set_path,
+                        eval_ids,
+                        input_overrides=input_overrides,
                     )
 
                     # Resolve model settings override from eval set

src/uipath/eval/helpers.py

@@ -24,6 +24,51 @@
 EVAL_SETS_DIRECTORY_NAME = "evaluations/eval-sets"
 
 
+def _apply_file_overrides_to_conversational_inputs(
+    conversational_inputs: Any,
+    overrides: dict[str, Any],
+) -> None:
+    """Apply file overrides to conversational input attachments before mapper conversion.
+
+    Extracts file objects from override values (single dict or array), matches them
+    to existing attachments by FullName, and replaces attachment fields in-place.
+    No-op if there are no file overrides or no matching attachments.
+    """
+    if not overrides:
+        return
+
+    file_overrides: list[dict[str, Any]] = []
+    for value in overrides.values():
+        if isinstance(value, list):
+            file_overrides.extend(f for f in value if isinstance(f, dict) and "ID" in f)
+        elif isinstance(value, dict) and "ID" in value:
+            file_overrides.append(value)
+
+    if not file_overrides:
+        return
+
+    override_by_name = {f["FullName"]: f for f in file_overrides if "FullName" in f}
+
+    def _override_attachments(attachments: list[Any] | None) -> None:
+        if not attachments:
+            return
+        for attachment in attachments:
+            override = override_by_name.get(attachment.full_name)
+            if override:
+                attachment.id = override["ID"]
+                if "FullName" in override:
+                    attachment.full_name = override["FullName"]
+                if "MimeType" in override:
+                    attachment.mime_type = override["MimeType"]
+
+    _override_attachments(conversational_inputs.current_user_prompt.attachments)
+
+    for exchange in conversational_inputs.conversation_history:
+        for message in exchange:
+            if hasattr(message, "attachments"):
+                _override_attachments(message.attachments)
+
+
 def discriminate_eval_set(data: dict[str, Any]) -> EvaluationSet | LegacyEvaluationSet:
     """Discriminate and parse evaluation set based on version field.
 
@@ -90,13 +135,19 @@ def auto_discover_eval_set() -> str:
 
     @staticmethod
     def load_eval_set(
-        eval_set_path: str, eval_ids: list[str] | None = None
+        eval_set_path: str,
+        eval_ids: list[str] | None = None,
+        input_overrides: dict[str, Any] | None = None,
     ) -> tuple[EvaluationSet, str]:
         """Load the evaluation set from file.
 
         Args:
             eval_set_path: Path to the evaluation set file
             eval_ids: Optional list of evaluation IDs to filter
+            input_overrides: Optional input field overrides per evaluation ID.
+                For conversational agents, file overrides are applied to attachments
+                before the legacy-to-messages conversion so that overridden IDs
+                are baked into the messages before mapping.
 
         Returns:
             Tuple of (EvaluationSet, resolved_path)
@@ -147,6 +198,16 @@ def migrate_evaluation_item(
                         )
 
                     if evaluation.conversational_inputs:
+                        overrides_for_eval = (
+                            input_overrides.get(evaluation.id, {})
+                            if input_overrides
+                            else {}
+                        )
+                        _apply_file_overrides_to_conversational_inputs(
+                            evaluation.conversational_inputs,
+                            overrides_for_eval,
+                        )
+
                         conversational_messages_input = UiPathLegacyEvalChatMessagesMapper.legacy_conversational_eval_input_to_uipath_message_list(
                             evaluation.conversational_inputs
                         )

src/uipath/eval/models/_conversational_utils.py

@@ -12,6 +12,7 @@
     UiPathConversationToolCall,
     UiPathConversationToolCallData,
     UiPathConversationToolCallResult,
+    UiPathExternalValue,
     UiPathInlineValue,
 )
 
@@ -146,13 +147,21 @@ def legacy_conversational_eval_input_to_uipath_message_list(
                         else []
                     )
 
-                    # TODO: Add attachments if present
-                    # if message.attachments:
-                    #     for attachment in message.attachments:
-                    #         content_parts.append(
-                    #             UiPathConversationContentPart(...)
-                    #         )
-
+                    if eval_message.attachments:
+                        for attachment in eval_message.attachments:
+                            content_parts.append(
+                                UiPathConversationContentPart(
+                                    content_part_id=str(uuid.uuid4()),
+                                    mime_type=attachment.mime_type,
+                                    data=UiPathExternalValue(
+                                        uri=f"urn:uipath:cas:file:orchestrator:{attachment.id}"
+                                    ),
+                                    name=attachment.full_name,
+                                    citations=[],
+                                    created_at=timestamp,
+                                    updated_at=timestamp,
+                                )
+                            )
                     messages.append(
                         UiPathConversationMessage(
                             message_id=str(uuid.uuid4()),
@@ -228,12 +237,21 @@ def legacy_conversational_eval_input_to_uipath_message_list(
             else []
         )
 
-        # TODO Add attachments if present
-        # if eval_input.current_user_prompt.attachments:
-        #     for attachment in eval_input.current_user_prompt.attachments:
-        #         content_parts.append(
-        #             UiPathConversationContentPart(...)
-        #         )
+        if eval_input.current_user_prompt.attachments:
+            for attachment in eval_input.current_user_prompt.attachments:
+                content_parts.append(
+                    UiPathConversationContentPart(
+                        content_part_id=str(uuid.uuid4()),
+                        mime_type=attachment.mime_type,
+                        data=UiPathExternalValue(
+                            uri=f"urn:uipath:cas:file:orchestrator:{attachment.id}"
+                        ),
+                        name=attachment.full_name,
+                        citations=[],
+                        created_at=timestamp,
+                        updated_at=timestamp,
+                    )
+                )
 
         messages.append(
             UiPathConversationMessage(