LCORE-2917: add multimodal image support for query endpoints

- prepare_input() skips image attachments (text-only for moderation)
- build_multimodal_input() converts text + images into pydantic-ai
  UserContent parts with ImageUrl data URLs
- ResponsesApiParams carries image_attachments (excluded from API body)
- prepare_responses_params() extracts image attachments from request
- Agent runners build multimodal prompts when images are present
- Text-only requests are completely unchanged

Author: JslYoon

src/models/common/responses/responses_api_params.py

@@ -20,6 +20,7 @@
 )
 from pydantic import BaseModel, Field
 
+from models.common.query import Attachment
 from models.common.responses.types import IncludeParameter, InputTool, ResponseInput
 from utils.tool_formatter import translate_vector_store_ids_to_user_facing
 
@@ -130,6 +131,12 @@ class ResponsesApiParams(BaseModel):
         "compacted, lightspeed-stack supplies explicit input and must not let "
         "Llama Stack reload the full history via the conversation parameter.",
     )
+    image_attachments: Optional[list[Attachment]] = Field(
+        default=None,
+        exclude=True,
+        description="Image attachments from the query request. Excluded from the "
+        "API request body — used by agent runners to build multimodal prompts.",
+    )
 
     def model_dump(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
         """Serialize to a request body dict.

src/utils/agents/query.py

@@ -44,6 +44,7 @@
 from utils.conversations import append_turn_items_to_conversation
 from utils.pydantic_ai import build_agent
 from utils.query import (
+    build_multimodal_input,
     extract_provider_and_model_from_model_id,
     handle_known_apistatus_errors,
     is_context_length_error,
@@ -318,7 +319,14 @@ async def retrieve_agent_response(
             client, responses_params, configuration.skills, no_tools=no_tools
         )
         logger.debug("Starting agent non-streaming response processing")
-        run_result = await agent.run(cast(str, responses_params.input))
+        if responses_params.image_attachments:
+            prompt = build_multimodal_input(
+                cast(str, responses_params.input),
+                responses_params.image_attachments,
+            )
+        else:
+            prompt = cast(str, responses_params.input)
+        run_result = await agent.run(prompt)
     except (AgentRunError, APIStatusError, APIConnectionError, RuntimeError) as exc:
         response = map_agent_inference_error(exc, responses_params.model)
         raise HTTPException(**response.model_dump()) from exc

src/utils/agents/streaming.py

@@ -58,7 +58,11 @@
 )
 from utils.conversations import append_turn_items_to_conversation
 from utils.pydantic_ai import build_agent
-from utils.query import consume_query_tokens, store_query_results
+from utils.query import (
+    build_multimodal_input,
+    consume_query_tokens,
+    store_query_results,
+)
 from utils.quota_utils import get_available_quotas
 from utils.responses import (
     deduplicate_referenced_documents,
@@ -316,7 +320,13 @@ async def agent_response_generator(
         rag_id_mapping=context.rag_id_mapping,
         turn_summary=turn_summary,
     )
-    prompt = cast(str, responses_params.input)  # query is always a string
+    if responses_params.image_attachments:
+        prompt = build_multimodal_input(
+            cast(str, responses_params.input),
+            responses_params.image_attachments,
+        )
+    else:
+        prompt = cast(str, responses_params.input)
 
     logger.debug("Starting agent streaming response processing")
     async with agent.run_stream_events(prompt) as stream:

src/utils/query.py

@@ -11,6 +11,7 @@
 )
 from llama_stack_client.types import Shield
 from openai._exceptions import APIStatusError as OpenAIAPIStatusError
+from pydantic_ai.messages import ImageUrl, UserContent
 from sqlalchemy import func
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -169,11 +170,11 @@ def is_input_shield(shield: Shield) -> bool:
 def prepare_input(
     query_request: QueryRequest, inline_rag_context: Optional[str] = None
 ) -> str:
-    """
-    Prepare input text for Responses API by appending RAG context and attachments.
+    """Prepare text input for moderation and Responses API.
 
     Takes the query text, appends any inline RAG context for the LLM call, then
-    appends any attachment content with type labels.
+    appends any text attachment content with type labels. Image attachments are
+    skipped — they are handled separately as structured multimodal input.
 
     Args:
         query_request: The query request containing the query and optional attachments
@@ -182,20 +183,43 @@ def prepare_input(
             API model.
 
     Returns:
-        str: The input text with RAG context and attachments appended (if any)
+        str: The input text with RAG context and text attachments appended (if any)
     """
     input_text: str = query_request.query
     if inline_rag_context:
         input_text += f"\n\n{inline_rag_context}"
     if query_request.attachments:
         for attachment in query_request.attachments:
-            # Append attachment content with type label
+            if attachment.content_type in constants.IMAGE_CONTENT_TYPES:
+                continue
             input_text += (
                 f"\n\n[Attachment: {attachment.attachment_type}]\n{attachment.content}"
             )
     return input_text
 
 
+def build_multimodal_input(
+    text: str, image_attachments: list[Attachment]
+) -> list[UserContent]:
+    """Build a pydantic-ai multimodal prompt from text and image attachments.
+
+    Constructs a list of UserContent items containing the text prompt followed
+    by ImageUrl entries for each image attachment, using base64 data URLs.
+
+    Args:
+        text: The text portion of the input (query + RAG context + text attachments).
+        image_attachments: Image attachments with base64-encoded content.
+
+    Returns:
+        List of UserContent items: the text string followed by ImageUrl objects.
+    """
+    parts: list[UserContent] = [text]
+    for attachment in image_attachments:
+        data_url = f"data:{attachment.content_type};base64,{attachment.content}"
+        parts.append(ImageUrl(url=data_url, media_type=attachment.content_type))
+    return parts
+
+
 def store_query_results(  # pylint: disable=too-many-arguments
     user_id: str,
     conversation_id: str,

src/utils/responses.py

@@ -93,6 +93,7 @@
     NotFoundResponse,
     ServiceUnavailableResponse,
 )
+from models.common.query import Attachment
 from models.common.responses.responses_api_params import ResponsesApiParams
 from models.common.responses.types import (
     InputTool,
@@ -384,9 +385,20 @@ async def prepare_responses_params(  # pylint: disable=too-many-arguments,too-ma
     )
 
     # Prepare input for Responses API
-    # Adds inline RAG context and attachments
+    # Adds inline RAG context and text attachments (images are excluded)
     input_text = prepare_input(query_request, inline_rag_context)
 
+    # Extract image attachments for multimodal support
+    image_attachments: Optional[list[Attachment]] = None
+    if query_request.attachments:
+        images = [
+            a
+            for a in query_request.attachments
+            if a.content_type in constants.IMAGE_CONTENT_TYPES
+        ]
+        if images:
+            image_attachments = images
+
     # Handle conversation ID for Responses API
     conversation_id = query_request.conversation_id
     if conversation_id:
@@ -431,6 +443,7 @@ async def prepare_responses_params(  # pylint: disable=too-many-arguments,too-ma
         extra_headers=extra_headers,
         max_infer_iters=configuration.inference.max_infer_iters,
         max_tool_calls=configuration.inference.max_tool_calls,
+        image_attachments=image_attachments,
     )
 
 

tests/unit/utils/test_query.py

@@ -2,13 +2,15 @@
 
 # pylint: disable=too-many-lines
 
+import base64
 import sqlite3
 from typing import Any
 
 import psycopg2
 import pytest
 from fastapi import HTTPException
 from llama_stack_client.types import ModelListResponse
+from pydantic_ai.messages import ImageUrl
 from pytest_mock import MockerFixture
 from sqlalchemy.exc import SQLAlchemyError
 
@@ -27,6 +29,7 @@
 from models.database.conversations import UserConversation, UserTurn
 from tests.unit import config_dict
 from utils.query import (
+    build_multimodal_input,
     consume_query_tokens,
     extract_provider_and_model_from_model_id,
     handle_known_apistatus_errors,
@@ -255,6 +258,96 @@ def test_prepare_input_with_attachments(self) -> None:
         assert "[Attachment: text]" in result
         assert "attachment content" in result
 
+    def test_prepare_input_skips_image_attachments(self) -> None:
+        """Test that image attachments are excluded from text input."""
+        text_attachment = Attachment(
+            attachment_type="log",
+            content="log output",
+            content_type="text/plain",
+        )
+        image_attachment = Attachment(
+            attachment_type="image",
+            content=base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode(),
+            content_type="image/jpeg",
+        )
+        query_request = QueryRequest(
+            query="describe this image",
+            attachments=[text_attachment, image_attachment],
+        )  # pyright: ignore[reportCallIssue]
+        result = prepare_input(query_request)
+        assert "describe this image" in result
+        assert "[Attachment: log]" in result
+        assert "log output" in result
+        assert result.count("[Attachment:") == 1
+
+    def test_prepare_input_only_image_attachments(self) -> None:
+        """Test prepare_input with only image attachments returns just the query."""
+        image_attachment = Attachment(
+            attachment_type="image",
+            content=base64.b64encode(b"\x89PNG" + b"\x00" * 10).decode(),
+            content_type="image/png",
+        )
+        query_request = QueryRequest(
+            query="what is in this image",
+            attachments=[image_attachment],
+        )  # pyright: ignore[reportCallIssue]
+        result = prepare_input(query_request)
+        assert result == "what is in this image"
+
+
+class TestBuildMultimodalInput:
+    """Tests for build_multimodal_input function."""
+
+    def test_single_image(self) -> None:
+        """Test building multimodal input with one image."""
+        image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
+        image_attachment = Attachment(
+            attachment_type="image",
+            content=image_data,
+            content_type="image/jpeg",
+        )
+        result = build_multimodal_input("describe this", [image_attachment])
+        assert len(result) == 2
+        assert result[0] == "describe this"
+        assert isinstance(result[1], ImageUrl)
+        assert result[1].url == f"data:image/jpeg;base64,{image_data}"
+
+    def test_multiple_images(self) -> None:
+        """Test building multimodal input with multiple images."""
+        jpeg_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
+        png_data = base64.b64encode(b"\x89PNG" + b"\x00" * 10).decode()
+        attachments = [
+            Attachment(
+                attachment_type="image",
+                content=jpeg_data,
+                content_type="image/jpeg",
+            ),
+            Attachment(
+                attachment_type="image",
+                content=png_data,
+                content_type="image/png",
+            ),
+        ]
+        result = build_multimodal_input("compare these", attachments)
+        assert len(result) == 3
+        assert result[0] == "compare these"
+        assert isinstance(result[1], ImageUrl)
+        assert result[1].url == f"data:image/jpeg;base64,{jpeg_data}"
+        assert isinstance(result[2], ImageUrl)
+        assert result[2].url == f"data:image/png;base64,{png_data}"
+
+    def test_image_url_media_type(self) -> None:
+        """Test that ImageUrl has correct media_type set."""
+        image_data = base64.b64encode(b"\x89PNG" + b"\x00" * 10).decode()
+        attachment = Attachment(
+            attachment_type="image",
+            content=image_data,
+            content_type="image/png",
+        )
+        result = build_multimodal_input("test", [attachment])
+        assert isinstance(result[1], ImageUrl)
+        assert result[1].media_type == "image/png"
+
 
 class TestExtractProviderAndModelFromModelId:
     """Tests for extract_provider_and_model_from_model_id function."""