Merge branch 'main' into fix_llm_guardrails_check

Author: ctiliescuuipath

pyproject.toml

@@ -5,7 +5,7 @@ description = "Python SDK that enables developers to build and deploy LangGraph
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"
 dependencies = [
-    "uipath>=2.2.35, <2.3.0",
+    "uipath>=2.2.41, <2.3.0",
     "langgraph>=1.0.0, <2.0.0",
     "langchain-core>=1.0.0, <2.0.0",
     "aiosqlite==0.21.0",

src/uipath_langchain/agent/guardrails/guardrails_factory.py

@@ -17,6 +17,7 @@
     AgentUnknownGuardrail,
     AgentWordOperator,
     AgentWordRule,
+    StandardRecipient,
 )
 from uipath.core.guardrails import (
     BooleanRule,
@@ -273,17 +274,18 @@ def build_guardrails_with_actions(
                 )
             )
         elif isinstance(action, AgentGuardrailEscalateAction):
-            result.append(
-                (
-                    converted_guardrail,
-                    EscalateAction(
-                        app_name=action.app.name,
-                        app_folder_path=action.app.folder_name,
-                        version=action.app.version,
-                        assignee=action.recipient.value,
-                    ),
+            if isinstance(action.recipient, StandardRecipient):
+                result.append(
+                    (
+                        converted_guardrail,
+                        EscalateAction(
+                            app_name=action.app.name,
+                            app_folder_path=action.app.folder_name,
+                            version=action.app.version,
+                            assignee=action.recipient.value,
+                        ),
+                    )
                 )
-            )
         elif isinstance(action, AgentGuardrailFilterAction):
             result.append((converted_guardrail, FilterAction(fields=action.fields)))
     return result

src/uipath_langchain/agent/react/file_type_handler.py

@@ -0,0 +1,123 @@
+import base64
+from enum import StrEnum
+from typing import Any
+
+import httpx
+from uipath._utils._ssl_context import get_httpx_client_kwargs
+
+IMAGE_MIME_TYPES: set[str] = {
+    "image/png",
+    "image/jpeg",
+    "image/gif",
+    "image/webp",
+}
+
+
+class LlmProvider(StrEnum):
+    OPENAI = "openai"
+    BEDROCK = "bedrock"
+    VERTEX = "vertex"
+    UNKNOWN = "unknown"
+
+
+def is_pdf(mime_type: str) -> bool:
+    """Check if the MIME type represents a PDF document."""
+    return mime_type.lower() == "application/pdf"
+
+
+def is_image(mime_type: str) -> bool:
+    """Check if the MIME type represents a supported image format (PNG, JPEG, GIF, WebP)."""
+    return mime_type.lower() in IMAGE_MIME_TYPES
+
+
+def detect_provider(model_name: str) -> LlmProvider:
+    """Detect the LLM provider (Bedrock, OpenAI, or Vertex) based on the model name."""
+    if not model_name:
+        raise ValueError(f"Unsupported model: {model_name}")
+
+    model_lower = model_name.lower()
+
+    if "anthropic" in model_lower or "claude" in model_lower:
+        return LlmProvider.BEDROCK
+
+    if "gpt" in model_lower:
+        return LlmProvider.OPENAI
+
+    if "gemini" in model_lower:
+        return LlmProvider.VERTEX
+
+    raise ValueError(f"Unsupported model: {model_name}")
+
+
+async def _download_file(url: str) -> str:
+    """Download a file from a URL and return its content as a base64 string."""
+    async with httpx.AsyncClient(**get_httpx_client_kwargs()) as client:
+        response = await client.get(url)
+        response.raise_for_status()
+        file_content = response.content
+
+    return base64.b64encode(file_content).decode("utf-8")
+
+
+async def build_message_content_part_from_data(
+    url: str,
+    filename: str,
+    mime_type: str,
+    model: str,
+) -> dict[str, Any]:
+    """Download a file and build a provider-specific message content part.
+
+    The format varies based on the detected provider (Bedrock, OpenAI, or Vertex).
+    """
+    provider = detect_provider(model)
+
+    if provider == LlmProvider.BEDROCK:
+        raise ValueError("Anthropic models are not yet supported for file attachments")
+
+    if provider == LlmProvider.OPENAI:
+        return await _build_openai_content_part_from_data(
+            url, mime_type, filename, False
+        )
+
+    if provider == LlmProvider.VERTEX:
+        raise ValueError("Gemini models are not yet supported for file attachments")
+
+    raise ValueError(f"Unsupported provider: {provider}")
+
+
+async def _build_openai_content_part_from_data(
+    url: str,
+    mime_type: str,
+    filename: str,
+    download_image: bool,
+) -> dict[str, Any]:
+    """Build a content part for OpenAI models (base64-encoded or URL reference)."""
+    if download_image:
+        base64_content = await _download_file(url)
+        if is_image(mime_type):
+            data_url = f"data:{mime_type};base64,{base64_content}"
+            return {
+                "type": "input_image",
+                "image_url": data_url,
+            }
+
+        if is_pdf(mime_type):
+            return {
+                "type": "input_file",
+                "filename": filename,
+                "file_data": base64_content,
+            }
+
+    elif is_image(mime_type):
+        return {
+            "type": "input_image",
+            "image_url": url,
+        }
+
+    elif is_pdf(mime_type):
+        return {
+            "type": "input_file",
+            "file_url": url,
+        }
+
+    raise ValueError(f"Unsupported mime_type: {mime_type}")

src/uipath_langchain/agent/react/llm_with_files.py

@@ -0,0 +1,76 @@
+"""LLM invocation with file attachments support."""
+
+from dataclasses import dataclass
+from typing import Any
+
+from langchain_core.language_models import BaseChatModel
+from langchain_core.messages import AIMessage, AnyMessage, HumanMessage
+
+from .file_type_handler import build_message_content_part_from_data
+
+
+@dataclass
+class FileInfo:
+    """File information for LLM file attachments."""
+
+    url: str
+    name: str
+    mime_type: str
+
+
+def _get_model_name(model: BaseChatModel) -> str:
+    """Extract model name from a BaseChatModel instance."""
+    for attr in ["model_name", "_model_name", "model", "model_id"]:
+        value = getattr(model, attr, None)
+        if value and isinstance(value, str):
+            return value
+    raise ValueError(f"Model name not found in model {model}")
+
+
+async def create_part_for_file(
+    file_info: FileInfo,
+    model: BaseChatModel,
+) -> dict[str, Any]:
+    """Create a provider-specific message content part for a file attachment.
+
+    Downloads the file from file_info.url and formats it for the model's provider.
+    """
+    model_name = _get_model_name(model)
+    return await build_message_content_part_from_data(
+        url=file_info.url,
+        filename=file_info.name,
+        mime_type=file_info.mime_type,
+        model=model_name,
+    )
+
+
+async def llm_call_with_files(
+    messages: list[AnyMessage],
+    files: list[FileInfo],
+    model: BaseChatModel,
+) -> AIMessage:
+    """Invoke an LLM with file attachments.
+
+    Downloads files, creates provider-specific content parts, and appends them
+    as a HumanMessage. If no files are provided, equivalent to model.ainvoke().
+    """
+    if not files:
+        response = await model.ainvoke(messages)
+        if not isinstance(response, AIMessage):
+            raise TypeError(
+                f"LLM returned {type(response).__name__} instead of AIMessage"
+            )
+        return response
+
+    content_parts: list[str | dict[Any, Any]] = []
+    for file_info in files:
+        content_part = await create_part_for_file(file_info, model)
+        content_parts.append(content_part)
+
+    file_message = HumanMessage(content=content_parts)
+    all_messages = list(messages) + [file_message]
+
+    response = await model.ainvoke(all_messages)
+    if not isinstance(response, AIMessage):
+        raise TypeError(f"LLM returned {type(response).__name__} instead of AIMessage")
+    return response

tests/agent/guardrails/test_guardrails_factory.py

@@ -9,7 +9,6 @@
     AgentBooleanOperator,
     AgentBooleanRule,
     AgentCustomGuardrail,
-    AgentEscalationRecipient,
     AgentEscalationRecipientType,
     AgentGuardrailActionType,
     AgentGuardrailBlockAction,
@@ -24,6 +23,7 @@
     AgentWordOperator,
     AgentWordRule,
     FieldReference,
+    StandardRecipient,
 )
 from uipath.agent.models.agent import (
     AgentGuardrail as AgentGuardrailModel,
@@ -139,7 +139,7 @@ def test_escalate_action_is_mapped_with_app_and_recipient(self) -> None:
             folder_name="/TestFolder",
             version=2,
         )
-        recipient = AgentEscalationRecipient(
+        recipient = StandardRecipient(
             type=AgentEscalationRecipientType.USER_EMAIL,
             value="admin@example.com",
         )