Merge pull request #443 from Mng-dev-ai/feat/bridge-inline-image-attachments

Inline image attachments for bridge-backed models

Author: Mng-dev-ai

backend/app/constants.py

@@ -2,6 +2,8 @@
 from pathlib import Path
 from typing import Final
 
+from app.models.schemas.settings import ProviderType
+
 CLAUDE_DIR: Final[Path] = (
     Path(d) if (d := os.environ.get("CLAUDE_CONFIG_DIR")) else Path.home() / ".claude"
 )
@@ -173,3 +175,9 @@
 DOCKER_STATUS_RUNNING: Final[str] = "running"
 
 SANDBOX_BASHRC_PATH: Final[str] = "/home/user/.bashrc"
+
+BRIDGE_PROVIDER_TYPES: Final[set[str]] = {
+    ProviderType.OPENROUTER.value,
+    ProviderType.OPENAI.value,
+    ProviderType.COPILOT.value,
+}

backend/app/services/chat.py

@@ -10,7 +10,7 @@
 from sqlalchemy import exists, func, select, update
 from sqlalchemy.orm import aliased, selectinload
 
-from app.constants import REDIS_KEY_CHAT_STREAM_LIVE
+from app.constants import BRIDGE_PROVIDER_TYPES, REDIS_KEY_CHAT_STREAM_LIVE
 from app.core.config import get_settings
 from app.models.db_models.chat import Chat, Message
 from app.models.db_models.enums import MessageRole, MessageStreamStatus, StreamEventKind
@@ -852,6 +852,14 @@ async def initiate_chat_completion(
         context_window = self._provider_service.get_model_context_window(
             user_settings, request.model_id
         )
+        provider, _ = self._provider_service.get_provider_for_model(
+            user_settings, request.model_id
+        )
+        uses_bridge = (
+            provider.get("provider_type") in BRIDGE_PROVIDER_TYPES
+            if provider
+            else False
+        )
 
         try:
             await self._enqueue_chat_task(
@@ -865,6 +873,7 @@ async def initiate_chat_completion(
                 assistant_message_id=str(assistant_message.id),
                 thinking_mode=request.thinking_mode,
                 worktree=request.worktree,
+                uses_bridge=uses_bridge,
                 attachments=attachments,
                 context_window=context_window,
                 selected_persona_name=request.selected_persona_name,
@@ -896,6 +905,7 @@ async def _enqueue_chat_task(
         assistant_message_id: str,
         thinking_mode: str | None,
         worktree: bool = False,
+        uses_bridge: bool = False,
         attachments: list[MessageAttachmentDict] | None,
         context_window: int | None = None,
         selected_persona_name: str = DEFAULT_PERSONA_NAME,
@@ -925,6 +935,7 @@ async def _enqueue_chat_task(
             assistant_message_id=assistant_message_id,
             thinking_mode=thinking_mode,
             worktree=worktree,
+            uses_bridge=uses_bridge,
             attachments=stream_attachments,
             selected_persona_name=selected_persona_name,
         )

backend/app/services/claude_agent.py

@@ -1,4 +1,6 @@
+import base64
 import logging
+import mimetypes
 import sys
 from collections.abc import AsyncIterator, Callable
 from functools import partial
@@ -13,6 +15,7 @@
 )
 
 from app.constants import (
+    BRIDGE_PROVIDER_TYPES,
     SANDBOX_GIT_ASKPASS_PATH,
     SANDBOX_HOME_DIR,
     SANDBOX_WORKSPACE_DIR,
@@ -188,16 +191,21 @@ async def stream_response(
         session_callback: Callable[[str, str | None], None] | None = None,
         attachments: list[dict[str, Any]] | None = None,
         attachment_base_dir: str = SANDBOX_HOME_DIR,
+        uses_bridge: bool = False,
     ) -> AsyncIterator[StreamEvent]:
         # Send a prompt to the Claude SDK client and yield processed stream
         # events, handling plan mode transitions on tool success/failure.
-        user_prompt = self.prepare_user_prompt(
-            prompt, custom_instructions, attachments, attachment_base_dir
+        user_content = self.build_user_message_content(
+            prompt,
+            custom_instructions,
+            attachments,
+            attachment_base_dir,
+            uses_bridge=uses_bridge,
         )
 
         prompt_message = {
             "type": "user",
-            "message": {"role": MessageRole.USER.value, "content": user_prompt},
+            "message": {"role": MessageRole.USER.value, "content": user_content},
             "parent_tool_use_id": None,
             "session_id": session_id,
         }
@@ -252,11 +260,7 @@ def _build_auth_env(
             # existing login; in Docker mode an explicit token is needed.
             if auth_token:
                 env["CLAUDE_CODE_OAUTH_TOKEN"] = auth_token
-        elif provider_type in (
-            ProviderType.OPENROUTER.value,
-            ProviderType.OPENAI.value,
-            ProviderType.COPILOT.value,
-        ):
+        elif provider_type in BRIDGE_PROVIDER_TYPES:
             # Non-Anthropic providers route through our local bridge
             # (https://github.com/Mng-dev-ai/anthropic-bridge) that translates
             # Anthropic API calls to the provider's format.
@@ -625,6 +629,80 @@ def prepare_user_prompt(
         parts.append(f"<user_prompt>{prompt}</user_prompt>")
         return "".join(parts)
 
+    @classmethod
+    def build_user_message_content(
+        cls,
+        prompt: str,
+        custom_instructions: str | None,
+        attachments: list[dict[str, Any]] | None = None,
+        attachment_base_dir: str = SANDBOX_HOME_DIR,
+        uses_bridge: bool = False,
+    ) -> str | list[dict[str, Any]]:
+        # Anthropic's API handles image attachments natively, but bridge-backed
+        # providers (OpenAI, OpenRouter, Copilot) need images inlined as base64
+        # content blocks since the bridge translates to their format.
+        if not uses_bridge:
+            return cls.prepare_user_prompt(
+                prompt,
+                custom_instructions,
+                attachments,
+                attachment_base_dir,
+            )
+
+        # Slash commands must stay as bare strings for the SDK to handle them.
+        if any(prompt.startswith(cmd) for cmd in ALLOWED_SLASH_COMMANDS):
+            return prompt
+
+        image_blocks: list[dict[str, Any]] = []
+        for attachment in attachments or []:
+            if attachment.get("file_type") == "image":
+                image_block = cls._attachment_to_image_block(attachment)
+                if image_block:
+                    image_blocks.append(image_block)
+
+        # Keep all attachments (including images) in the text block so the
+        # model sees filenames and can map them to the inline image blocks.
+        prompt_text = cls.prepare_user_prompt(
+            prompt,
+            custom_instructions,
+            attachments,
+            attachment_base_dir,
+        )
+        if not image_blocks:
+            return prompt_text
+
+        return [{"type": "text", "text": prompt_text}, *image_blocks]
+
+    @staticmethod
+    def _attachment_to_image_block(attachment: dict[str, Any]) -> dict[str, Any] | None:
+        relative_path = attachment.get("file_path")
+        if not isinstance(relative_path, str) or not relative_path:
+            return None
+
+        file_path = Path(settings.STORAGE_PATH) / relative_path
+        try:
+            raw = file_path.read_bytes()
+        except OSError:
+            logger.warning(
+                "Image attachment not found for inline bridge send: %s",
+                file_path,
+            )
+            return None
+
+        mime_type, _ = mimetypes.guess_type(str(file_path))
+        media_type = (
+            mime_type if mime_type and mime_type.startswith("image/") else "image/png"
+        )
+        encoded = base64.b64encode(raw).decode("ascii")
+        return {
+            "type": "image",
+            "source": {
+                "type": "base64",
+                "media_type": media_type,
+                "data": encoded,
+            },
+        }
+
     @staticmethod
     async def _create_prompt_iterable(
         prompt_message: dict[str, Any],

backend/app/services/sandbox.py

@@ -14,6 +14,7 @@
 from app.constants import (
     ANTHROPIC_BRIDGE_HOST,
     ANTHROPIC_BRIDGE_PORT,
+    BRIDGE_PROVIDER_TYPES,
     SANDBOX_CLAUDE_DIR,
     SANDBOX_CLAUDE_JSON_PATH,
     SANDBOX_GIT_ASKPASS_PATH,
@@ -626,13 +627,8 @@ def _has_bridge_provider(
     ) -> bool:
         if not custom_providers:
             return False
-        bridge_types = {
-            ProviderType.OPENROUTER.value,
-            ProviderType.OPENAI.value,
-            ProviderType.COPILOT.value,
-        }
         return any(
-            provider.get("provider_type") in bridge_types
+            provider.get("provider_type") in BRIDGE_PROVIDER_TYPES
             and provider.get("enabled", True)
             for provider in custom_providers
         )

backend/app/services/scheduler.py

@@ -16,6 +16,7 @@
 from sqlalchemy.exc import SQLAlchemyError
 from sqlalchemy.ext.asyncio import AsyncSession
 
+from app.constants import BRIDGE_PROVIDER_TYPES
 from app.core.config import get_settings
 from app.models.db_models.chat import Chat, Message
 from app.models.db_models.workspace import Workspace
@@ -730,9 +731,18 @@ async def run_scheduled_task(
 
             system_prompt = build_system_prompt_for_chat(user_settings)
 
-            context_window = ProviderService().get_model_context_window(
+            provider_service = ProviderService()
+            context_window = provider_service.get_model_context_window(
                 user_settings, model_id
             )
+            provider, _ = provider_service.get_provider_for_model(
+                user_settings, model_id
+            )
+            uses_bridge = (
+                provider.get("provider_type") in BRIDGE_PROVIDER_TYPES
+                if provider
+                else False
+            )
             stream_request = ChatStreamRequest(
                 prompt=prompt_message,
                 system_prompt=system_prompt,
@@ -744,6 +754,7 @@ async def run_scheduled_task(
                 session_id=None,
                 assistant_message_id=str(assistant_message_id),
                 thinking_mode="ultra",
+                uses_bridge=uses_bridge,
                 attachments=None,
                 selected_persona_name=DEFAULT_PERSONA_NAME,
             )

backend/app/services/streaming/runtime.py

@@ -16,6 +16,7 @@
 from sqlalchemy.orm import selectinload
 
 from app.constants import (
+    BRIDGE_PROVIDER_TYPES,
     REDIS_KEY_CHAT_CONTEXT_USAGE,
     REDIS_KEY_CHAT_STREAM_LIVE,
     SANDBOX_HOME_DIR,
@@ -147,6 +148,7 @@ def __init__(
         self.session_container: dict[str, Any] = {"session_id": request.session_id}
         self.assistant_message_id = request.assistant_message_id
         self.model_id = request.model_id
+        self.uses_bridge = request.uses_bridge
         self.context_window = request.context_window
         self.prompt = request.prompt
         self._is_new_chat = request.session_id is None
@@ -168,6 +170,20 @@ def __init__(
         self._cancelled: bool = False
         self._send_now_pending: bool = False
 
+    @property
+    def attachment_base_dir(self) -> str:
+        # Files are written to the sandbox home dir
+        # (/home/user/ → {host_base}/{sandbox_id}/), so in host mode
+        # point Claude at the real host path, not the workspace path.
+        if self.chat.sandbox_provider == SandboxProviderType.HOST:
+            sid = self.chat.sandbox_id
+            if sid:
+                host_base = (
+                    Path(settings.get_host_sandbox_base_dir()).expanduser().resolve()
+                )
+                return str(host_base / sid)
+        return SANDBOX_HOME_DIR
+
     async def run(
         self,
         ai_service: ClaudeAgentService,
@@ -495,21 +511,34 @@ async def _write_send_now(self, ai_service: ClaudeAgentService) -> bool:
                         self.model_id = resolved_model
                         if self._session:
                             self._session.current_model = resolved_model
+                    user_settings = await UserService(
+                        session_factory=self.session_factory
+                    ).get_user_settings(self.chat.user_id, db=None)
+                    provider, _ = ProviderService().get_provider_for_model(
+                        user_settings, queued_model
+                    )
+                    self.uses_bridge = (
+                        provider.get("provider_type") in BRIDGE_PROVIDER_TYPES
+                        if provider
+                        else False
+                    )
                 queued_permission = queued_msg.get("permission_mode")
                 if queued_permission:
                     sdk_permission = SDK_PERMISSION_MODE_MAP.get(
                         queued_permission, "acceptEdits"
                     )
                     await self.client.set_permission_mode(sdk_permission)
 
-            prompt = ai_service.prepare_user_prompt(
+            user_content = ai_service.build_user_message_content(
                 queued_msg["content"],
                 self.custom_instructions,
                 queued_msg.get("attachments"),
+                self.attachment_base_dir,
+                uses_bridge=self.uses_bridge,
             )
             injection = {
                 "type": "user",
-                "message": {"role": "user", "content": prompt},
+                "message": {"role": "user", "content": user_content},
                 "parent_tool_use_id": None,
                 "session_id": self.session_container.get("session_id"),
             }
@@ -848,9 +877,18 @@ def _build_queued_stream_request(
             user_settings,
             selected_persona_name=selected_persona_name,
         )
-        context_window = ProviderService().get_model_context_window(
+        provider_service = ProviderService()
+        context_window = provider_service.get_model_context_window(
             user_settings, queued_msg["model_id"]
         )
+        provider, _ = provider_service.get_provider_for_model(
+            user_settings, queued_msg["model_id"]
+        )
+        uses_bridge = (
+            provider.get("provider_type") in BRIDGE_PROVIDER_TYPES
+            if provider
+            else False
+        )
         return ChatStreamRequest(
             prompt=queued_msg["content"],
             system_prompt=system_prompt,
@@ -872,6 +910,7 @@ def _build_queued_stream_request(
             assistant_message_id=assistant_message_id,
             thinking_mode=queued_msg.get("thinking_mode"),
             worktree=queued_msg.get("worktree", False),
+            uses_bridge=uses_bridge,
             attachments=queued_msg.get("attachments"),
             selected_persona_name=selected_persona_name,
         )
@@ -1117,20 +1156,6 @@ async def execute_chat(
                         params.options.permission_mode
                     )
                 stream_result = StreamResult()
-                attachment_base_dir = SANDBOX_HOME_DIR
-                if runtime.chat.sandbox_provider == SandboxProviderType.HOST:
-                    # Files are written to the sandbox home dir
-                    # (/home/user/ → {host_base}/{sandbox_id}/), so
-                    # point Claude at the real host path, not the
-                    # workspace path.
-                    host_base = (
-                        Path(settings.get_host_sandbox_base_dir())
-                        .expanduser()
-                        .resolve()
-                    )
-                    sid = runtime.chat.sandbox_id
-                    assert sid is not None
-                    attachment_base_dir = str(host_base / sid)
                 stream = ai_service.stream_response(
                     client=session.client,
                     prompt=request.prompt,
@@ -1139,7 +1164,8 @@ async def execute_chat(
                     result=stream_result,
                     session_callback=session_callback,
                     attachments=request.attachments,
-                    attachment_base_dir=attachment_base_dir,
+                    attachment_base_dir=runtime.attachment_base_dir,
+                    uses_bridge=request.uses_bridge,
                 )
                 return await runtime.run(ai_service, stream_result, stream)
             except (