Improve InputMessage to support list-type content

Author: RussellLuo

README.md

@@ -567,6 +567,7 @@ reporter = AgentSpec(
 - [patterns](examples/patterns)
 - [agents-as-tools](examples/agents-as-tools)
 - [react-mcp](examples/react-mcp)
+- [react-vision](examples/react-vision)
 - [a2a](examples/a2a)
 - [mcp](examples/mcp)
 - [mcp-new](examples/mcp-new)

coagent/agents/react_agent/__init__.py

@@ -2,7 +2,15 @@
 from .agent import ReActAgent
 from .context import RunContext
 from .mcp import MCPTools
-from .messages import InputMessage, InputHistory, OutputMessage
+from .messages import (
+    InputMessage,
+    InputHistory,
+    OutputMessage,
+    InputMessageTextParam,
+    InputMessageImageParam,
+    InputMessageFileParam,
+    InputMessageAudioParam,
+)
 from .types import (
     MessageOutputItem,
     ToolCallItem,

coagent/agents/react_agent/messages.py

@@ -1,20 +1,27 @@
 from coagent.core import Message
 from pydantic import Field
 from typing import Literal
+from typing_extensions import TypeAlias
 
 from .types import (
     MessageOutputItem,
     ToolCallItem,
     ToolCallOutputItem,
     ToolCallProgressItem,
+    ResponseInputTextParam as InputMessageTextParam,
+    ResponseInputImageParam as InputMessageImageParam,
+    ResponseInputFileParam as InputMessageFileParam,
+    ResponseInputAudioParam as InputMessageAudioParam,
 )
 
+InputContentParam: TypeAlias = list[InputMessageTextParam | InputMessageImageParam | InputMessageFileParam | InputMessageAudioParam]
+
 
 class InputMessage(Message):
     role: Literal["user", "assistant"] = Field(
         ..., description="The role of the message. (e.g. `user`, `assistant`)"
     )
-    content: str = Field(
+    content: str | InputContentParam = Field(
         ...,
         description="The content of the message.",
     )

coagent/agents/react_agent/types.py

@@ -11,6 +11,12 @@
     ResponseFunctionToolCall,
     ResponseFunctionToolCallOutputItem,
 )
+from openai.types.responses.response_input_message_content_list_param import (
+    ResponseInputTextParam, # noqa: F401
+    ResponseInputImageParam, # noqa: F401
+    ResponseInputFileParam, # noqa: F401
+    ResponseInputAudioParam # noqa: F401
+)
 from openai.types.responses.response_reasoning_item import ResponseReasoningItem
 from pydantic import BaseModel
 

examples/react-vision/README.md

@@ -0,0 +1,25 @@
+# ReAct Vision
+
+This example demonstrates how to handle images using a vision model (e.g. gemini-2.5-flash) in a ReAct agent.
+
+
+## Prerequisites
+
+- Install `coagent` (see [Installation](../../README.md#installation)).
+
+
+## Quick Start
+
+Set the environment variables for your vision model:
+
+```bash
+export MODEL_ID="your-model-id"
+export MODEL_BASE_URL="your-base-url"
+export MODEL_API_KEY="your-api-key"
+```
+
+Run the agent:
+
+```bash
+python examples/react-vision/agent.py
+```

examples/react-vision/agent.py

@@ -0,0 +1,77 @@
+import asyncio
+from coagent.agents.react_agent import (
+    ReActAgent,
+    InputMessage,
+    InputMessageTextParam,
+    InputMessageImageParam,
+    InputHistory,
+    OutputMessage,
+    MessageOutputItem,
+    ToolCallItem,
+    ToolCallOutputItem,
+    ToolCallProgressItem,
+)
+from coagent.core import AgentSpec, new, init_logger
+from coagent.runtimes import LocalRuntime
+
+
+analyzer = AgentSpec(
+    "analyzer",
+    new(
+        ReActAgent,
+        name="reporter",
+        system="You are a helpful image analyzer",
+    ),
+)
+
+
+async def main():
+    async with LocalRuntime() as runtime:
+        await runtime.register(analyzer)
+
+        result = await analyzer.run(
+            InputHistory(
+                messages=[
+                    InputMessage(
+                        role="user",
+                        content=[
+                            InputMessageTextParam(
+                                type="input_text", text="What’s in this image?"
+                            ),
+                            InputMessageImageParam(
+                                detail="auto",
+                                type="input_image",
+                                image_url="https://raw.githubusercontent.com/OpenCSGs/coagent/main/assets/coagent-overview.png",
+                            ),
+                        ],
+                    )
+                ]
+            ).encode(),
+            stream=True,
+        )
+        async for chunk in result:
+            msg = OutputMessage.decode(chunk)
+            i = msg.item
+            match i:
+                case MessageOutputItem():
+                    print(i.raw_item.content[0].text, end="", flush=True)
+                case ToolCallItem():
+                    print(
+                        f"\n[tool#{i.raw_item.call_id} call: {i.raw_item.name}]",
+                        flush=True,
+                    )
+                case ToolCallProgressItem():
+                    print(
+                        f"\n[tool#{i.raw_item.call_id} progress: {i.raw_item.message}]",
+                        flush=True,
+                    )
+                case ToolCallOutputItem():
+                    print(
+                        f"\n[tool#{i.raw_item.call_id} output: {i.raw_item.output}]",
+                        flush=True,
+                    )
+
+
+if __name__ == "__main__":
+    init_logger()
+    asyncio.run(main())