feat(sandbox): accept PromptInputPart[] messages in prompt dispatch (#187)

Author: vutuanlinh2k2

package.json

@@ -358,7 +358,7 @@
     "@tangle-network/agent-knowledge": "^1.7.0",
     "@tangle-network/agent-runtime": "^0.79.3",
     "@tangle-network/brand": "^1.0.0",
-    "@tangle-network/sandbox": "^0.9.7",
+    "@tangle-network/sandbox": "^0.10.5",
     "@tangle-network/sandbox-ui": "^0.72.0",
     "@tangle-network/ui": "^11.0.0",
     "@testing-library/dom": "^10.4.1",

pnpm-lock.yaml

@@ -41,6 +41,7 @@ import {
   streamSandboxPrompt,
   resolveModel,
   flattenHistory,
+  mergeHistoryIntoParts,
   mergeExtraMcp,
   attachReasoningEffort,
   syncSandboxMemberAdd,
@@ -65,6 +66,7 @@ import {
   SandboxRuntimeAuthRefreshError,
   type SandboxRuntimeConfig,
   type SecretStore,
+  type PromptInputPart,
 } from './index'
 import { resolveTangleExecutionEnvironment } from '../runtime/model'
 import type {
@@ -502,13 +504,68 @@ describe('streamSandboxPrompt seam', () => {
     const [, opts] = (box.streamPrompt as ReturnType<typeof vi.fn>).mock.calls[0]!
     expect(opts.backend.interactions).toBeUndefined()
   })
+
+  it('forwards a PromptInputPart[] message to box.streamPrompt with history folded into the text part', async () => {
+    async function* events() {
+      yield { type: 'result' }
+    }
+    const box = fakeBox({ streamPrompt: vi.fn().mockReturnValue(events()) })
+    const parts: PromptInputPart[] = [
+      { type: 'image', url: 'https://img/1.png' },
+      { type: 'text', text: 'describe this' },
+    ]
+    for await (const _ of streamSandboxPrompt(shell(), box, parts, {
+      history: [{ role: 'assistant', content: 'prior' }],
+    })) {
+      void _
+    }
+    const [prompt] = (box.streamPrompt as ReturnType<typeof vi.fn>).mock.calls[0]!
+    expect(Array.isArray(prompt)).toBe(true)
+    expect(prompt).toEqual([
+      { type: 'image', url: 'https://img/1.png' },
+      { type: 'text', text: 'Assistant: prior\n\nUser: describe this' },
+    ])
+  })
 })
 
 describe('pure seam helpers', () => {
   it('flattenHistory returns the bare message when no history', () => {
     expect(flattenHistory('x')).toBe('x')
   })
 
+  it('mergeHistoryIntoParts returns the same parts when there is no history', () => {
+    const parts: PromptInputPart[] = [{ type: 'text', text: 'hi' }]
+    expect(mergeHistoryIntoParts(parts)).toBe(parts)
+    expect(mergeHistoryIntoParts(parts, [])).toBe(parts)
+  })
+
+  it('mergeHistoryIntoParts folds the transcript into the first text part, preserving other parts and order', () => {
+    const parts: PromptInputPart[] = [
+      { type: 'image', url: 'https://img/1.png' },
+      { type: 'text', text: 'describe this' },
+      { type: 'file', filename: 'notes.txt' },
+    ]
+    const merged = mergeHistoryIntoParts(parts, [
+      { role: 'user', content: 'earlier question' },
+      { role: 'assistant', content: 'earlier answer' },
+    ])
+    expect(merged).toEqual([
+      { type: 'image', url: 'https://img/1.png' },
+      {
+        type: 'text',
+        text: 'User: earlier question\n\nAssistant: earlier answer\n\nUser: describe this',
+      },
+      { type: 'file', filename: 'notes.txt' },
+    ])
+  })
+
+  it('mergeHistoryIntoParts throws fail-loud when parts contain no text part', () => {
+    const parts: PromptInputPart[] = [{ type: 'image', url: 'https://img/1.png' }]
+    expect(() => mergeHistoryIntoParts(parts, [{ role: 'user', content: 'prior' }])).toThrow(
+      /text part/,
+    )
+  })
+
   it('mergeExtraMcp throws on a collision with app-tool or base profile servers', () => {
     const appTool: Record<string, AgentProfileMcpServer> = {
       'app-propose': {} as AgentProfileMcpServer,

src/sandbox/index.test.ts

@@ -1508,15 +1508,49 @@ export function resolveModel(
   }
 }
 
+// The SDK's SandboxInstance.streamPrompt/.prompt accept `string | PromptInputPart[]`
+// but the published package does not re-export the PromptInputPart type by name from
+// any of its entry points, so it's derived structurally off the method signature
+// itself — this stays in lockstep with the SDK's actual accepted shape.
+export type PromptInputPart = Extract<
+  Parameters<SandboxInstance['streamPrompt']>[0],
+  readonly unknown[]
+>[number]
+
+function historyTranscript(
+  history: Array<{ role: 'user' | 'assistant'; content: string }>,
+): string {
+  return history
+    .map((entry) => `${entry.role === 'assistant' ? 'Assistant' : 'User'}: ${entry.content}`)
+    .join('\n\n')
+}
+
 export function flattenHistory(
   message: string,
   history?: Array<{ role: 'user' | 'assistant'; content: string }>,
 ): string {
   if (!history?.length) return message
-  const transcript = history
-    .map((entry) => `${entry.role === 'assistant' ? 'Assistant' : 'User'}: ${entry.content}`)
-    .join('\n\n')
-  return `${transcript}\n\nUser: ${message}`
+  return `${historyTranscript(history)}\n\nUser: ${message}`
+}
+
+/**
+ * History-aware equivalent of flattenHistory for multimodal prompt parts: the
+ * transcript is folded into the first text part (image/file parts carry no
+ * text to prepend to) rather than replacing the message wholesale.
+ */
+export function mergeHistoryIntoParts(
+  parts: PromptInputPart[],
+  history?: Array<{ role: 'user' | 'assistant'; content: string }>,
+): PromptInputPart[] {
+  if (!history?.length) return parts
+  const textIndex = parts.findIndex((part) => part.type === 'text')
+  if (textIndex === -1) {
+    throw new Error('mergeHistoryIntoParts requires at least one text part to carry the history')
+  }
+  const textPart = parts[textIndex] as Extract<PromptInputPart, { type: 'text' }>
+  const merged = [...parts]
+  merged[textIndex] = { ...textPart, text: `${historyTranscript(history)}\n\nUser: ${textPart.text}` }
+  return merged
 }
 
 export function mergeExtraMcp(
@@ -1581,7 +1615,7 @@ type StreamPromptOptions = Parameters<SandboxInstance['streamPrompt']>[1]
 export async function* streamSandboxPrompt(
   shell: SandboxRuntimeConfig,
   box: SandboxInstance,
-  message: string,
+  message: string | PromptInputPart[],
   options?: StreamSandboxPromptOptions,
 ): AsyncGenerator<unknown> {
   const harness = options?.harness ?? 'opencode'
@@ -1595,7 +1629,10 @@ export async function* streamSandboxPrompt(
   // if the UI snap was bypassed. Provider-less ids pass (session's own config).
   if (model?.model) assertHarnessModelCompatible(harness, model.model)
 
-  const prompt = flattenHistory(message, options?.history)
+  const prompt =
+    typeof message === 'string'
+      ? flattenHistory(message, options?.history)
+      : mergeHistoryIntoParts(message, options?.history)
 
   const appToolMcp = options?.appToolMcp ?? {}
   const extraMcp = mergeExtraMcp(appToolMcp, options?.baseProfileMcp ?? {}, options?.extraMcp)
@@ -1645,7 +1682,7 @@ export async function* streamSandboxPrompt(
 export async function runSandboxPrompt(
   shell: SandboxRuntimeConfig,
   box: SandboxInstance,
-  message: string,
+  message: string | PromptInputPart[],
   options?: StreamSandboxPromptOptions,
 ): Promise<string> {
   let fullText = ''
@@ -1816,7 +1853,7 @@ export async function mintSandboxScopedToken(
 export async function driveSandboxTurn(
   shell: SandboxRuntimeConfig,
   box: SandboxInstance,
-  message: string,
+  message: string | PromptInputPart[],
   options: StreamSandboxPromptOptions & { sessionId: string },
 ): Promise<Outcome<PromptResult>> {
   const harness = options.harness ?? 'opencode'
@@ -1825,7 +1862,10 @@ export async function driveSandboxTurn(
     modelApiKey: options.modelApiKey,
   })
   if (model?.model) assertHarnessModelCompatible(harness, model.model)
-  const prompt = flattenHistory(message, options.history)
+  const prompt =
+    typeof message === 'string'
+      ? flattenHistory(message, options.history)
+      : mergeHistoryIntoParts(message, options.history)
   const appToolMcp = options.appToolMcp ?? {}
   const extraMcp = mergeExtraMcp(appToolMcp, options.baseProfileMcp ?? {}, options.extraMcp)
   const profile = attachReasoningEffort(