Skip to content

Commit bd50263

Browse files
ZhiXiao-LinZhiXiao-Lin
committed
feat(multimodal): add multi-modal vision support for image attachments
- Add `Attachment` struct with jpeg/png/gif/webp/from_file constructors - Add `ContentBlock::Image` variant with `ImageSource` for Anthropic format - Add `ToolResultContentField` enum (Text | Blocks) for multi-modal tool results - Add `Message::user_with_attachments()` and `Message::tool_result_with_images()` - Add `ToolOutput::with_images()` for tools returning screenshots/diagrams - Add `AgentSession::send_with_attachments()` and `stream_with_attachments()` - Add `AgentLoop::execute_from_messages()` for pre-built multi-modal messages - Update `OpenAiClient::convert_messages()` with image_url block support - Anthropic serialization works natively via serde (no changes needed) - Agent loop carries image attachments through all permission/HITL paths - Backward-compatible: plain string tool results serialize identically - 35 new tests: Attachment, ImageSource, ContentBlock::Image, ToolResultContentField, Message constructors, OpenAI conversion, backward compat, tool output images - Docs: sessions.mdx vision section, README multi-modal section - Test count 1179 → 1214
1 parent 9721488 commit bd50263

13 files changed

Lines changed: 907 additions & 30 deletions

File tree

‎README.md‎

Lines changed: 30 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ let result = session.send("Refactor auth to use JWT").await?;
1111
[![Crates.io](https://img.shields.io/crates/v/a3s-code-core.svg)](https://crates.io/crates/a3s-code-core)
1212
[![Documentation](https://docs.rs/a3s-code-core/badge.svg)](https://docs.rs/a3s-code-core)
1313
[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](./LICENSE)
14-
[![Tests](https://img.shields.io/badge/tests-1179%20passing-brightgreen.svg)](./core/tests)
14+
[![Tests](https://img.shields.io/badge/tests-1214%20passing-brightgreen.svg)](./core/tests)
1515

1616
---
1717

@@ -210,6 +210,30 @@ The planner creates steps with dependencies. Independent steps execute in parall
210210

211211
---
212212

213+
### 🖼️ Multi-Modal Support (Vision)
214+
215+
Send image attachments alongside text prompts. Requires a vision-capable model (Claude Sonnet, GPT-4o).
216+
217+
```rust
218+
use a3s_code_core::Attachment;
219+
220+
// Send a prompt with an image
221+
let image = Attachment::from_file("screenshot.png")?;
222+
let result = session.send_with_attachments(
223+
"What's in this screenshot?",
224+
&[image],
225+
None,
226+
).await?;
227+
228+
// Tools can return images too
229+
let output = ToolOutput::success("Screenshot captured")
230+
.with_images(vec![Attachment::png(screenshot_bytes)]);
231+
```
232+
233+
Supported formats: JPEG, PNG, GIF, WebP. Image data is base64-encoded for both Anthropic and OpenAI providers.
234+
235+
---
236+
213237
### 🚦 Lane-Based Priority Queue
214238

215239
Tool execution is routed through a priority queue backed by [a3s-lane](../lane):
@@ -440,6 +464,10 @@ let session = agent.session(".", Some(options))?; // With options
440464
let result = session.send("prompt", None).await?;
441465
let (rx, handle) = session.stream("prompt", None).await?;
442466

467+
// Multi-modal (vision)
468+
let result = session.send_with_attachments("Describe", &[image], None).await?;
469+
let (rx, handle) = session.stream_with_attachments("Describe", &[image], None).await?;
470+
443471
// Direct tool access
444472
let content = session.read_file("src/main.rs").await?;
445473
let output = session.bash("cargo test").await?;
@@ -504,7 +532,7 @@ cargo test # All tests
504532
cargo test --lib # Unit tests only
505533
```
506534

507-
**Test Coverage:** 1179 tests, 100% pass rate
535+
**Test Coverage:** 1214 tests, 100% pass rate
508536

509537
---
510538

‎core/examples/sdk_chat.rs‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -118,10 +118,12 @@ async fn main() -> anyhow::Result<()> {
118118
}
119119
}
120120
ContentBlock::ToolUse { name, .. } => Some(format!("[tool_use: {}]", name)),
121-
ContentBlock::ToolResult { content, .. } => Some(format!(
122-
"[tool_result: {}]",
123-
&content[..content.len().min(40)]
124-
)),
121+
ContentBlock::ToolResult { content, .. } => {
122+
let text = content.as_text();
123+
let truncated = &text[..text.len().min(40)];
124+
Some(format!("[tool_result: {}]", truncated))
125+
}
126+
ContentBlock::Image { .. } => Some("[image]".to_string()),
125127
})
126128
.collect::<Vec<_>>()
127129
.join(" | ");

‎core/src/agent.rs‎

Lines changed: 79 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -888,6 +888,43 @@ impl AgentLoop {
888888
.await
889889
}
890890

891+
/// Execute the agent loop with pre-built messages (user message already included).
892+
///
893+
/// Used by `send_with_attachments` / `stream_with_attachments` where the
894+
/// user message contains multi-modal content and is already appended to
895+
/// the messages vec.
896+
pub async fn execute_from_messages(
897+
&self,
898+
messages: Vec<Message>,
899+
session_id: Option<&str>,
900+
event_tx: Option<mpsc::Sender<AgentEvent>>,
901+
) -> Result<AgentResult> {
902+
tracing::info!(
903+
a3s.session.id = session_id.unwrap_or("none"),
904+
a3s.agent.max_turns = self.config.max_tool_rounds,
905+
"a3s.agent.execute_from_messages started"
906+
);
907+
908+
// Pass empty prompt so execute_loop skips adding a user message
909+
let result = self
910+
.execute_loop(&messages, "", session_id, event_tx)
911+
.await;
912+
913+
match &result {
914+
Ok(r) => tracing::info!(
915+
a3s.agent.tool_calls_count = r.tool_calls_count,
916+
a3s.llm.total_tokens = r.usage.total_tokens,
917+
"a3s.agent.execute_from_messages completed"
918+
),
919+
Err(e) => tracing::warn!(
920+
error = %e,
921+
"a3s.agent.execute_from_messages failed"
922+
),
923+
}
924+
925+
result
926+
}
927+
891928
/// Execute the agent loop for a prompt with session context
892929
///
893930
/// Takes the conversation history, user prompt, and optional session ID.
@@ -1004,7 +1041,9 @@ impl AgentLoop {
10041041
};
10051042

10061043
// Add user message
1007-
messages.push(Message::user(prompt));
1044+
if !prompt.is_empty() {
1045+
messages.push(Message::user(prompt));
1046+
}
10081047

10091048
loop {
10101049
turn += 1;
@@ -1285,7 +1324,7 @@ impl AgentLoop {
12851324
PermissionDecision::Ask
12861325
};
12871326

1288-
let (output, exit_code, is_error, _metadata) = match permission_decision {
1327+
let (output, exit_code, is_error, _metadata, images) = match permission_decision {
12891328
PermissionDecision::Deny => {
12901329
tracing::info!(
12911330
tool_name = tool_call.name.as_str(),
@@ -1310,7 +1349,7 @@ impl AgentLoop {
13101349
.ok();
13111350
}
13121351

1313-
(denial_msg, 1, true, None)
1352+
(denial_msg, 1, true, None, Vec::new())
13141353
}
13151354
PermissionDecision::Allow => {
13161355
tracing::info!(
@@ -1326,8 +1365,8 @@ impl AgentLoop {
13261365
.await;
13271366

13281367
match result {
1329-
Ok(r) => (r.output, r.exit_code, r.exit_code != 0, r.metadata),
1330-
Err(e) => (format!("Tool execution error: {}", e), 1, true, None),
1368+
Ok(r) => (r.output, r.exit_code, r.exit_code != 0, r.metadata, r.images),
1369+
Err(e) => (format!("Tool execution error: {}", e), 1, true, None, Vec::new()),
13311370
}
13321371
}
13331372
PermissionDecision::Ask => {
@@ -1353,19 +1392,28 @@ impl AgentLoop {
13531392
)
13541393
.await;
13551394

1356-
let (output, exit_code, is_error, _metadata) = match result {
1357-
Ok(r) => (r.output, r.exit_code, r.exit_code != 0, r.metadata),
1395+
let (output, exit_code, is_error, _metadata, images) = match result {
1396+
Ok(r) => (r.output, r.exit_code, r.exit_code != 0, r.metadata, r.images),
13581397
Err(e) => {
1359-
(format!("Tool execution error: {}", e), 1, true, None)
1398+
(format!("Tool execution error: {}", e), 1, true, None, Vec::new())
13601399
}
13611400
};
13621401

13631402
// Add tool result to messages
1364-
messages.push(Message::tool_result(
1365-
&tool_call.id,
1366-
&output,
1367-
is_error,
1368-
));
1403+
if images.is_empty() {
1404+
messages.push(Message::tool_result(
1405+
&tool_call.id,
1406+
&output,
1407+
is_error,
1408+
));
1409+
} else {
1410+
messages.push(Message::tool_result_with_images(
1411+
&tool_call.id,
1412+
&output,
1413+
&images,
1414+
is_error,
1415+
));
1416+
}
13691417

13701418
// Record tool result on the tool span for early exit
13711419
let tool_duration = tool_start.elapsed();
@@ -1425,12 +1473,14 @@ impl AgentLoop {
14251473
r.exit_code,
14261474
r.exit_code != 0,
14271475
r.metadata,
1476+
r.images,
14281477
),
14291478
Err(e) => (
14301479
format!("Tool execution error: {}", e),
14311480
1,
14321481
true,
14331482
None,
1483+
Vec::new(),
14341484
),
14351485
}
14361486
} else {
@@ -1439,15 +1489,15 @@ impl AgentLoop {
14391489
tool_call.name,
14401490
response.reason.unwrap_or_else(|| "No reason provided".to_string())
14411491
);
1442-
(rejection_msg, 1, true, None)
1492+
(rejection_msg, 1, true, None, Vec::new())
14431493
}
14441494
}
14451495
Ok(Err(_)) => {
14461496
let msg = format!(
14471497
"Tool '{}' confirmation failed: confirmation channel closed",
14481498
tool_call.name
14491499
);
1450-
(msg, 1, true, None)
1500+
(msg, 1, true, None, Vec::new())
14511501
}
14521502
Err(_) => {
14531503
cm.check_timeouts().await;
@@ -1458,7 +1508,7 @@ impl AgentLoop {
14581508
"Tool '{}' execution timed out waiting for confirmation ({}ms). Execution rejected.",
14591509
tool_call.name, timeout_ms
14601510
);
1461-
(msg, 1, true, None)
1511+
(msg, 1, true, None, Vec::new())
14621512
}
14631513
crate::hitl::TimeoutAction::AutoApprove => {
14641514
let stream_ctx = self.streaming_tool_context(
@@ -1480,12 +1530,14 @@ impl AgentLoop {
14801530
r.exit_code,
14811531
r.exit_code != 0,
14821532
r.metadata,
1533+
r.images,
14831534
),
14841535
Err(e) => (
14851536
format!("Tool execution error: {}", e),
14861537
1,
14871538
true,
14881539
None,
1540+
Vec::new(),
14891541
),
14901542
}
14911543
}
@@ -1503,7 +1555,7 @@ impl AgentLoop {
15031555
tool_name = tool_call.name.as_str(),
15041556
"Tool requires confirmation but no HITL manager configured"
15051557
);
1506-
(msg, 1, true, None)
1558+
(msg, 1, true, None, Vec::new())
15071559
}
15081560
}
15091561
};
@@ -1535,7 +1587,16 @@ impl AgentLoop {
15351587
}
15361588

15371589
// Add tool result to messages
1538-
messages.push(Message::tool_result(&tool_call.id, &output, is_error));
1590+
if images.is_empty() {
1591+
messages.push(Message::tool_result(&tool_call.id, &output, is_error));
1592+
} else {
1593+
messages.push(Message::tool_result_with_images(
1594+
&tool_call.id,
1595+
&output,
1596+
&images,
1597+
is_error,
1598+
));
1599+
}
15391600
}
15401601
}
15411602
}

‎core/src/agent_api.rs‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -785,6 +785,70 @@ impl AgentSession {
785785
Ok(result)
786786
}
787787

788+
/// Send a prompt with image attachments and wait for the complete response.
789+
///
790+
/// Images are included as multi-modal content blocks in the user message.
791+
/// Requires a vision-capable model (e.g., Claude Sonnet, GPT-4o).
792+
pub async fn send_with_attachments(
793+
&self,
794+
prompt: &str,
795+
attachments: &[crate::llm::Attachment],
796+
history: Option<&[Message]>,
797+
) -> Result<AgentResult> {
798+
// Build a user message with text + images, then pass it as the last
799+
// history entry. We use an empty prompt so execute_loop doesn't add
800+
// a duplicate user message.
801+
let use_internal = history.is_none();
802+
let mut effective_history = match history {
803+
Some(h) => h.to_vec(),
804+
None => self.history.read().unwrap().clone(),
805+
};
806+
effective_history.push(Message::user_with_attachments(prompt, attachments));
807+
808+
let agent_loop = self.build_agent_loop();
809+
let result = agent_loop
810+
.execute_from_messages(effective_history, None, None)
811+
.await?;
812+
813+
if use_internal {
814+
*self.history.write().unwrap() = result.messages.clone();
815+
if self.auto_save {
816+
if let Err(e) = self.save().await {
817+
tracing::warn!("Auto-save failed for session {}: {}", self.session_id, e);
818+
}
819+
}
820+
}
821+
822+
Ok(result)
823+
}
824+
825+
/// Stream a prompt with image attachments.
826+
///
827+
/// Images are included as multi-modal content blocks in the user message.
828+
/// Requires a vision-capable model (e.g., Claude Sonnet, GPT-4o).
829+
pub async fn stream_with_attachments(
830+
&self,
831+
prompt: &str,
832+
attachments: &[crate::llm::Attachment],
833+
history: Option<&[Message]>,
834+
) -> Result<(mpsc::Receiver<AgentEvent>, JoinHandle<()>)> {
835+
let (tx, rx) = mpsc::channel(256);
836+
let mut effective_history = match history {
837+
Some(h) => h.to_vec(),
838+
None => self.history.read().unwrap().clone(),
839+
};
840+
effective_history.push(Message::user_with_attachments(prompt, attachments));
841+
842+
let agent_loop = self.build_agent_loop();
843+
let handle = tokio::spawn(async move {
844+
let _ = agent_loop
845+
.execute_from_messages(effective_history, None, Some(tx))
846+
.await;
847+
});
848+
849+
Ok((rx, handle))
850+
}
851+
788852
/// Send a prompt and stream events back.
789853
///
790854
/// When `history` is `None`, uses the session's internal history

‎core/src/lib.rs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,7 +86,8 @@ pub use config::{CodeConfig, ModelConfig, ModelCost, ModelLimit, ModelModalities
8686
pub use error::{CodeError, Result};
8787
pub use hooks::HookEngine;
8888
pub use llm::{
89-
AnthropicClient, ContentBlock, LlmClient, LlmResponse, Message, OpenAiClient, TokenUsage,
89+
AnthropicClient, Attachment, ContentBlock, ImageSource, LlmClient, LlmResponse, Message,
90+
OpenAiClient, TokenUsage,
9091
};
9192
pub use queue::{
9293
ExternalTask, ExternalTaskResult, LaneHandlerConfig, SessionLane,

0 commit comments

Comments
 (0)