|
3 | 3 | # pylint: disable=too-many-lines |
4 | 4 |
|
5 | 5 | import asyncio |
| 6 | +import base64 |
6 | 7 | import json |
7 | 8 | from collections.abc import AsyncIterator, Callable |
8 | 9 | from typing import Any, Optional |
|
19 | 20 | FinishReason, |
20 | 21 | FunctionToolCallEvent, |
21 | 22 | FunctionToolResultEvent, |
| 23 | + ImageUrl, |
22 | 24 | ModelResponse, |
23 | 25 | NativeToolCallPart, |
24 | 26 | NativeToolReturnPart, |
|
50 | 52 | TurnCompleteStreamPayload, |
51 | 53 | ) |
52 | 54 | from models.common.moderation import ShieldModerationBlocked, ShieldModerationPassed |
| 55 | +from models.common.query import Attachment as QueryAttachment |
53 | 56 | from models.common.responses.contexts import ResponseGeneratorContext |
54 | 57 | from models.common.responses.responses_api_params import ResponsesApiParams |
55 | 58 | from models.common.turn_summary import RAGContext, TurnSummary |
@@ -889,6 +892,68 @@ async def test_streams_token_events_and_updates_summary( |
889 | 892 | assert turn_summary.token_usage.input_tokens == 4 |
890 | 893 | assert turn_summary.token_usage.output_tokens == 2 |
891 | 894 |
|
| 895 | + @pytest.mark.asyncio |
| 896 | + async def test_streams_with_image_attachments_passes_multimodal_prompt( |
| 897 | + self, |
| 898 | + mocker: MockerFixture, |
| 899 | + make_generator_context: Callable[..., ResponseGeneratorContext], |
| 900 | + make_responses_params: Callable[..., ResponsesApiParams], |
| 901 | + make_agent_run_result: Callable[..., Any], |
| 902 | + patch_recording_metrics: None, |
| 903 | + ) -> None: |
| 904 | + """Test that image attachments produce a multimodal prompt for streaming.""" |
| 905 | + context = make_generator_context() |
| 906 | + turn_summary = TurnSummary() |
| 907 | + run_result = make_agent_run_result( |
| 908 | + content="I see a screenshot.", |
| 909 | + response_id="resp-image-stream", |
| 910 | + input_tokens=5, |
| 911 | + output_tokens=3, |
| 912 | + ) |
| 913 | + events = [ |
| 914 | + PartStartEvent(index=0, part=TextPart(content="I see")), |
| 915 | + PartDeltaEvent( |
| 916 | + index=0, delta=TextPartDelta(content_delta=" a screenshot.") |
| 917 | + ), |
| 918 | + AgentRunResultEvent(result=run_result), |
| 919 | + ] |
| 920 | + mock_agent = mocker.Mock() |
| 921 | + mock_agent.run_stream_events.return_value = _mock_run_stream(events) |
| 922 | + mocker.patch( |
| 923 | + "utils.agents.streaming.get_agent_finish_reason", |
| 924 | + return_value=AgentFinishReason.SUCCESS, |
| 925 | + ) |
| 926 | + mocker.patch( |
| 927 | + "utils.agents.streaming.deduplicate_referenced_documents", |
| 928 | + side_effect=lambda docs: docs, |
| 929 | + ) |
| 930 | + |
| 931 | + image_data = base64.b64encode(b"\xff\xd8\xff\xe0" + b"\x00" * 10).decode() |
| 932 | + image_attachment = QueryAttachment( |
| 933 | + attachment_type="image", |
| 934 | + content=image_data, |
| 935 | + content_type="image/jpeg", |
| 936 | + ) |
| 937 | + params = make_responses_params(input_text="describe this") |
| 938 | + params.image_attachments = [image_attachment] |
| 939 | + |
| 940 | + _ = [ |
| 941 | + event |
| 942 | + async for event in agent_response_generator( |
| 943 | + mock_agent, |
| 944 | + params, |
| 945 | + context, |
| 946 | + turn_summary, |
| 947 | + ENDPOINT_PATH_STREAMING_QUERY, |
| 948 | + ) |
| 949 | + ] |
| 950 | + |
| 951 | + prompt_arg = mock_agent.run_stream_events.call_args[0][0] |
| 952 | + assert isinstance(prompt_arg, list) |
| 953 | + assert prompt_arg[0] == "describe this" |
| 954 | + assert isinstance(prompt_arg[1], ImageUrl) |
| 955 | + assert prompt_arg[1].url == f"data:image/jpeg;base64,{image_data}" |
| 956 | + |
892 | 957 | @pytest.mark.asyncio |
893 | 958 | async def test_non_success_finish_reason_yields_error_event( |
894 | 959 | self, |
|
0 commit comments