Skip to content

Commit 824ed3f

Browse files
committed
LCORE-2917: regenerate OpenAPI schema and add multimodal image tests
Regenerate docs/openapi.json to reflect Attachment model changes (image type, content_type examples, description updates). Add tests for image attachment validation, prepare_responses_params image extraction, and multimodal prompt construction in both blocking and streaming agent runners. Signed-off-by: Lucas <lyoon@redhat.com>
1 parent ae7c339 commit 824ed3f

6 files changed

Lines changed: 197 additions & 5 deletions

File tree

docs/openapi.json

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -11487,23 +11487,26 @@
1148711487
"attachment_type": {
1148811488
"type": "string",
1148911489
"title": "Attachment Type",
11490-
"description": "The attachment type, like 'log', 'configuration' etc.",
11490+
"description": "The attachment type, like 'log', 'configuration', 'image' etc.",
1149111491
"examples": [
11492-
"log"
11492+
"log",
11493+
"image"
1149311494
]
1149411495
},
1149511496
"content_type": {
1149611497
"type": "string",
1149711498
"title": "Content Type",
1149811499
"description": "The content type as defined in MIME standard",
1149911500
"examples": [
11500-
"text/plain"
11501+
"text/plain",
11502+
"image/jpeg",
11503+
"image/png"
1150111504
]
1150211505
},
1150311506
"content": {
1150411507
"type": "string",
1150511508
"title": "Content",
11506-
"description": "The actual attachment content",
11509+
"description": "The actual attachment content (text or base64-encoded image data)",
1150711510
"examples": [
1150811511
"warning: quota exceeded"
1150911512
]
@@ -11517,7 +11520,7 @@
1151711520
"content"
1151811521
],
1151911522
"title": "Attachment",
11520-
"description": "Model representing an attachment that can be sent from the UI as part of query.\n\nA list of attachments can be an optional part of 'query' request.\n\nAttributes:\n attachment_type: The attachment type, like \"log\", \"configuration\" etc.\n content_type: The content type as defined in MIME standard\n content: The actual attachment content",
11523+
"description": "Model representing an attachment that can be sent from the UI as part of query.\n\nA list of attachments can be an optional part of 'query' request.\n\nAttributes:\n attachment_type: The attachment type, like \"log\", \"configuration\", \"image\" etc.\n content_type: The content type as defined in MIME standard\n content: The actual attachment content (text or base64-encoded image data)",
1152111524
"examples": [
1152211525
{
1152311526
"attachment_type": "log",
@@ -11533,6 +11536,11 @@
1153311536
"attachment_type": "configuration",
1153411537
"content": "foo: bar",
1153511538
"content_type": "application/yaml"
11539+
},
11540+
{
11541+
"attachment_type": "image",
11542+
"content": "<base64-encoded image data>",
11543+
"content_type": "image/png"
1153611544
}
1153711545
]
1153811546
},

src/models/common/query.py

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ class Attachment(BaseModel):
4444
def validate_image_attachment(self) -> "Attachment":
4545
"""Validate consistency between attachment_type and content_type for images.
4646
47+
Returns:
48+
Self: The validated Attachment instance.
49+
4750
Raises:
4851
ValueError: If image content_type is used without attachment_type='image',
4952
if attachment_type='image' is used without an image content_type,

tests/unit/utils/agents/test_query.py

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
"""Unit tests for utils.agents.query module."""
22

3+
import base64
34
from collections.abc import Callable
45
from typing import Any, Optional
56

@@ -11,6 +12,7 @@
1112
from llama_stack_client import APIConnectionError, APIStatusError
1213
from pydantic_ai.messages import (
1314
FinishReason,
15+
ImageUrl,
1416
ModelRequest,
1517
ModelResponse,
1618
NativeToolCallPart,
@@ -25,6 +27,7 @@
2527

2628
from constants import ENDPOINT_PATH_QUERY
2729
from models.common.moderation import ShieldModerationBlocked, ShieldModerationPassed
30+
from models.common.query import Attachment
2831
from models.common.responses.responses_api_params import ResponsesApiParams
2932
from models.common.responses.types import ResponseInput
3033
from models.common.turn_summary import TurnSummary
@@ -431,6 +434,49 @@ async def test_success_returns_turn_summary(
431434
assert summary.llm_response == "Hello!"
432435
assert summary.id == "resp-success"
433436

437+
@pytest.mark.asyncio
438+
async def test_success_with_image_attachments_sends_multimodal_prompt(
439+
self,
440+
mocker: MockerFixture,
441+
make_agent_run_result: Callable[..., Any],
442+
make_responses_params: Callable[..., ResponsesApiParams],
443+
patch_recording_metrics: None,
444+
) -> None:
445+
"""Test that image attachments produce a multimodal prompt."""
446+
run_result = make_agent_run_result(
447+
content="I see a screenshot.",
448+
response_id="resp-image",
449+
)
450+
mock_agent = mocker.AsyncMock()
451+
mock_agent.run = mocker.AsyncMock(return_value=run_result)
452+
mocker.patch(
453+
"utils.agents.query.build_agent",
454+
return_value=mock_agent,
455+
)
456+
457+
image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
458+
image_attachment = Attachment(
459+
attachment_type="image",
460+
content=image_data,
461+
content_type="image/jpeg",
462+
)
463+
params = make_responses_params(input_text="describe this")
464+
params.image_attachments = [image_attachment]
465+
466+
summary = await retrieve_agent_response(
467+
client=mocker.AsyncMock(),
468+
responses_params=params,
469+
moderation_result=ShieldModerationPassed(),
470+
endpoint_path=ENDPOINT_PATH_QUERY,
471+
)
472+
473+
prompt_arg = mock_agent.run.call_args[0][0]
474+
assert isinstance(prompt_arg, list)
475+
assert prompt_arg[0] == "describe this"
476+
assert isinstance(prompt_arg[1], ImageUrl)
477+
assert prompt_arg[1].url == f"data:image/jpeg;base64,{image_data}"
478+
assert summary.llm_response == "I see a screenshot."
479+
434480
@pytest.mark.asyncio
435481
@pytest.mark.usefixtures("patch_query_configuration")
436482
async def test_agent_connection_error_raises_http_exception(

tests/unit/utils/agents/test_streaming.py

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
# pylint: disable=too-many-lines
44

55
import asyncio
6+
import base64
67
import json
78
from collections.abc import AsyncIterator, Callable
89
from typing import Any, Optional
@@ -19,6 +20,7 @@
1920
FinishReason,
2021
FunctionToolCallEvent,
2122
FunctionToolResultEvent,
23+
ImageUrl,
2224
ModelResponse,
2325
NativeToolCallPart,
2426
NativeToolReturnPart,
@@ -50,6 +52,7 @@
5052
TurnCompleteStreamPayload,
5153
)
5254
from models.common.moderation import ShieldModerationBlocked, ShieldModerationPassed
55+
from models.common.query import Attachment as QueryAttachment
5356
from models.common.responses.contexts import ResponseGeneratorContext
5457
from models.common.responses.responses_api_params import ResponsesApiParams
5558
from models.common.turn_summary import RAGContext, TurnSummary
@@ -889,6 +892,68 @@ async def test_streams_token_events_and_updates_summary(
889892
assert turn_summary.token_usage.input_tokens == 4
890893
assert turn_summary.token_usage.output_tokens == 2
891894

895+
@pytest.mark.asyncio
896+
async def test_streams_with_image_attachments_passes_multimodal_prompt(
897+
self,
898+
mocker: MockerFixture,
899+
make_generator_context: Callable[..., ResponseGeneratorContext],
900+
make_responses_params: Callable[..., ResponsesApiParams],
901+
make_agent_run_result: Callable[..., Any],
902+
patch_recording_metrics: None,
903+
) -> None:
904+
"""Test that image attachments produce a multimodal prompt for streaming."""
905+
context = make_generator_context()
906+
turn_summary = TurnSummary()
907+
run_result = make_agent_run_result(
908+
content="I see a screenshot.",
909+
response_id="resp-image-stream",
910+
input_tokens=5,
911+
output_tokens=3,
912+
)
913+
events = [
914+
PartStartEvent(index=0, part=TextPart(content="I see")),
915+
PartDeltaEvent(
916+
index=0, delta=TextPartDelta(content_delta=" a screenshot.")
917+
),
918+
AgentRunResultEvent(result=run_result),
919+
]
920+
mock_agent = mocker.Mock()
921+
mock_agent.run_stream_events.return_value = _mock_run_stream(events)
922+
mocker.patch(
923+
"utils.agents.streaming.get_agent_finish_reason",
924+
return_value=AgentFinishReason.SUCCESS,
925+
)
926+
mocker.patch(
927+
"utils.agents.streaming.deduplicate_referenced_documents",
928+
side_effect=lambda docs: docs,
929+
)
930+
931+
image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
932+
image_attachment = QueryAttachment(
933+
attachment_type="image",
934+
content=image_data,
935+
content_type="image/jpeg",
936+
)
937+
params = make_responses_params(input_text="describe this")
938+
params.image_attachments = [image_attachment]
939+
940+
_ = [
941+
event
942+
async for event in agent_response_generator(
943+
mock_agent,
944+
params,
945+
context,
946+
turn_summary,
947+
ENDPOINT_PATH_STREAMING_QUERY,
948+
)
949+
]
950+
951+
prompt_arg = mock_agent.run_stream_events.call_args[0][0]
952+
assert isinstance(prompt_arg, list)
953+
assert prompt_arg[0] == "describe this"
954+
assert isinstance(prompt_arg[1], ImageUrl)
955+
assert prompt_arg[1].url == f"data:image/jpeg;base64,{image_data}"
956+
892957
@pytest.mark.asyncio
893958
async def test_non_success_finish_reason_yields_error_event(
894959
self,

tests/unit/utils/test_query.py

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,26 @@ def test_invalid_content_type(self) -> None:
441441
assert exc_info.value.status_code == 422
442442
assert "Invalid attachment content type" in str(exc_info.value.detail)
443443

444+
def test_valid_image_attachment_jpeg(self) -> None:
445+
"""Test validation passes for JPEG image attachment."""
446+
image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
447+
attachment = Attachment(
448+
attachment_type="image",
449+
content=image_data,
450+
content_type="image/jpeg",
451+
)
452+
validate_attachments_metadata([attachment])
453+
454+
def test_valid_image_attachment_png(self) -> None:
455+
"""Test validation passes for PNG image attachment."""
456+
image_data = base64.b64encode(b"\x89PNG" + b"\x00" * 10).decode()
457+
attachment = Attachment(
458+
attachment_type="image",
459+
content=image_data,
460+
content_type="image/png",
461+
)
462+
validate_attachments_metadata([attachment])
463+
444464

445465
class TestIsTranscriptsEnabled:
446466
"""Tests for is_transcripts_enabled function."""

tests/unit/utils/test_responses.py

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@
22

33
# pylint: disable=line-too-long,too-many-lines
44

5+
import base64
56
import json
67
from pathlib import Path
78
from typing import Any, Optional, cast
@@ -55,6 +56,7 @@
5556

5657
import constants
5758
from models.api.requests import QueryRequest
59+
from models.common.query import Attachment
5860
from models.common.responses.types import InputTool, InputToolMCP
5961
from models.config import (
6062
ApprovalFilter,
@@ -2226,6 +2228,54 @@ async def test_prepare_responses_params_api_status_error_on_conversation(
22262228
await prepare_responses_params(mock_client, query_request, None, "token")
22272229
assert exc_info.value.status_code == 500
22282230

2231+
@pytest.mark.asyncio
2232+
async def test_image_attachments_extracted(self, mocker: MockerFixture) -> None:
2233+
"""Test that image attachments are extracted into ResponsesApiParams."""
2234+
mock_client = mocker.AsyncMock()
2235+
mock_conversation = mocker.Mock()
2236+
mock_conversation.id = "new_conv_id"
2237+
mock_client.conversations.create = mocker.AsyncMock(
2238+
return_value=mock_conversation
2239+
)
2240+
2241+
image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode()
2242+
text_attachment = Attachment(
2243+
attachment_type="log",
2244+
content="log output",
2245+
content_type="text/plain",
2246+
)
2247+
image_attachment = Attachment(
2248+
attachment_type="image",
2249+
content=image_data,
2250+
content_type="image/jpeg",
2251+
)
2252+
query_request = QueryRequest(
2253+
query="describe this",
2254+
attachments=[text_attachment, image_attachment],
2255+
) # pyright: ignore[reportCallIssue]
2256+
2257+
mock_config = mocker.Mock()
2258+
mock_config.inference = InferenceConfiguration()
2259+
mocker.patch("utils.responses.configuration", mock_config)
2260+
mocker.patch("utils.responses.get_system_prompt", return_value="System prompt")
2261+
mocker.patch("utils.responses.prepare_tools", return_value=None)
2262+
mocker.patch(
2263+
"utils.responses.select_model_for_responses",
2264+
return_value="provider1/model1",
2265+
)
2266+
mocker.patch("utils.responses.check_model_configured", return_value=True)
2267+
2268+
result = await prepare_responses_params(
2269+
mock_client, query_request, None, "token"
2270+
)
2271+
2272+
assert isinstance(result.input, str)
2273+
assert result.image_attachments is not None
2274+
assert len(result.image_attachments) == 1
2275+
assert result.image_attachments[0].content_type == "image/jpeg"
2276+
dumped = result.model_dump(exclude_none=True)
2277+
assert "image_attachments" not in dumped
2278+
22292279

22302280
class TestParseReferencedDocuments:
22312281
"""Tests for parse_referenced_documents function."""

0 commit comments

Comments
 (0)