|
| 1 | +import { expect, test } from "bun:test"; |
| 2 | +import { createOpenAIChatAdapter } from "../src/adapters/openai-chat"; |
| 3 | +import type { OcxContentPart, OcxMessage, OcxParsedRequest, OcxProviderConfig } from "../src/types"; |
| 4 | + |
| 5 | +// Issue #888: role:"tool" content is text-only on chat-completions providers, so images inside a |
| 6 | +// tool result were flattened to an "[image]" marker and vision-capable routed models hallucinated |
| 7 | +// what they never saw. Tool-result images now ride in a follow-up user vision message released when |
| 8 | +// the tool round closes, without splitting the round (strict providers reject interleaved users). |
| 9 | + |
| 10 | +const provider: OcxProviderConfig = { |
| 11 | + adapter: "openai-chat", |
| 12 | + baseUrl: "https://example.test/v1", |
| 13 | + apiKey: "sk-test", |
| 14 | + authMode: "key", |
| 15 | +}; |
| 16 | + |
| 17 | +const IMAGE_URL = "data:image/png;base64,aGVsbG8taW1hZ2UtYnl0ZXM="; |
| 18 | + |
| 19 | +interface ChatPart { |
| 20 | + type: string; |
| 21 | + text?: string; |
| 22 | + image_url?: { url: string; detail?: string }; |
| 23 | +} |
| 24 | + |
| 25 | +interface ChatMsg { |
| 26 | + role: string; |
| 27 | + content?: string | ChatPart[]; |
| 28 | + tool_calls?: { id: string; function: { name: string; arguments: string } }[]; |
| 29 | + tool_call_id?: string; |
| 30 | +} |
| 31 | + |
| 32 | +function wire(messages: OcxMessage[]): ChatMsg[] { |
| 33 | + const parsed: OcxParsedRequest = { |
| 34 | + modelId: "test-model", |
| 35 | + context: { messages }, |
| 36 | + stream: false, |
| 37 | + options: {}, |
| 38 | + }; |
| 39 | + const req = createOpenAIChatAdapter(provider).buildRequest(parsed) as { body: string }; |
| 40 | + return (JSON.parse(req.body) as { messages: ChatMsg[] }).messages; |
| 41 | +} |
| 42 | + |
| 43 | +function user(text: string): OcxMessage { |
| 44 | + return { role: "user", content: text, timestamp: 0 }; |
| 45 | +} |
| 46 | + |
| 47 | +function assistantWithCalls(calls: { id: string; name: string }[]): OcxMessage { |
| 48 | + return { |
| 49 | + role: "assistant", |
| 50 | + content: calls.map(c => ({ type: "toolCall" as const, id: c.id, name: c.name, arguments: {} })), |
| 51 | + timestamp: 0, |
| 52 | + }; |
| 53 | +} |
| 54 | + |
| 55 | +function toolResult(callId: string, name: string, content: string | OcxContentPart[]): OcxMessage { |
| 56 | + return { role: "toolResult", toolCallId: callId, toolName: name, content, isError: false, timestamp: 0 }; |
| 57 | +} |
| 58 | + |
| 59 | +/** The carrier is a user message whose parts start with an "[ocx]" text label followed by image_url parts. */ |
| 60 | +function isImageCarrier(msg: ChatMsg): boolean { |
| 61 | + if (msg.role !== "user" || !Array.isArray(msg.content)) return false; |
| 62 | + const [head, ...rest] = msg.content; |
| 63 | + return head?.type === "text" && typeof head.text === "string" && head.text.startsWith("[ocx]") |
| 64 | + && rest.length > 0 && rest.every(p => p.type === "image_url"); |
| 65 | +} |
| 66 | + |
| 67 | +/** Every role:"tool" message must sit in an unbroken block right after its assistant tool_calls message. */ |
| 68 | +function assertRoundsUnbroken(messages: ChatMsg[]): void { |
| 69 | + for (let i = 0; i < messages.length; i++) { |
| 70 | + const m = messages[i]; |
| 71 | + if (m.role !== "tool") continue; |
| 72 | + let j = i - 1; |
| 73 | + while (j >= 0 && messages[j].role === "tool") j--; |
| 74 | + expect(j).toBeGreaterThanOrEqual(0); |
| 75 | + expect(messages[j].role).toBe("assistant"); |
| 76 | + expect((messages[j].tool_calls ?? []).map(tc => tc.id)).toContain(m.tool_call_id); |
| 77 | + } |
| 78 | +} |
| 79 | + |
| 80 | +test("tool-result images ride a follow-up user message; text, detail, and https URLs survive", () => { |
| 81 | + const messages = wire([ |
| 82 | + user("read the screenshot"), |
| 83 | + assistantWithCalls([{ id: "call_1", name: "Read" }]), |
| 84 | + toolResult("call_1", "Read", [ |
| 85 | + { type: "text", text: "1 match found" }, |
| 86 | + { type: "image", imageUrl: IMAGE_URL, detail: "high" }, |
| 87 | + { type: "image", imageUrl: "https://example.test/shot.png" }, |
| 88 | + { type: "image", imageUrl: "" }, // empty file_id shape: keeps its marker, never reaches the carrier |
| 89 | + ]), |
| 90 | + ]); |
| 91 | + assertRoundsUnbroken(messages); |
| 92 | + const tool = messages.find(m => m.role === "tool")!; |
| 93 | + expect(tool.content).toBe("1 match found[image][image][image]"); |
| 94 | + const carrier = messages.find(isImageCarrier)!; |
| 95 | + expect(carrier).toBeDefined(); |
| 96 | + expect(messages.indexOf(carrier)).toBe(messages.indexOf(tool) + 1); |
| 97 | + const parts = (carrier.content as ChatPart[]).filter(p => p.type === "image_url"); |
| 98 | + expect(parts.map(p => p.image_url)).toEqual([ |
| 99 | + { url: IMAGE_URL, detail: "high" }, |
| 100 | + { url: "https://example.test/shot.png" }, |
| 101 | + ]); |
| 102 | +}); |
| 103 | + |
| 104 | +test("images from a multi-call round flush once, only after the whole round closes", () => { |
| 105 | + const messages = wire([ |
| 106 | + assistantWithCalls([{ id: "call_1", name: "shot" }, { id: "call_2", name: "list" }]), |
| 107 | + toolResult("call_1", "shot", [{ type: "image", imageUrl: IMAGE_URL }]), |
| 108 | + toolResult("call_2", "list", "file1.txt"), |
| 109 | + ]); |
| 110 | + assertRoundsUnbroken(messages); |
| 111 | + const toolIdx = messages.map((m, i) => (m.role === "tool" ? i : -1)).filter(i => i >= 0); |
| 112 | + expect(toolIdx).toEqual([toolIdx[0], toolIdx[0] + 1]); // nothing interleaves the round |
| 113 | + const carriers = messages.filter(isImageCarrier); |
| 114 | + expect(carriers.length).toBe(1); |
| 115 | + expect(messages.indexOf(carriers[0])).toBe(toolIdx[1] + 1); |
| 116 | +}); |
| 117 | + |
| 118 | +test("orphan tool result with an image still emits the carrier after its synthesized pair", () => { |
| 119 | + const messages = wire([ |
| 120 | + user("hi"), |
| 121 | + toolResult("call_orphan", "shot", [{ type: "image", imageUrl: IMAGE_URL }]), |
| 122 | + ]); |
| 123 | + assertRoundsUnbroken(messages); |
| 124 | + const tool = messages.find(m => m.role === "tool")!; |
| 125 | + expect(tool.content).toBe("[image]"); |
| 126 | + const carrier = messages.find(isImageCarrier)!; |
| 127 | + expect(messages.indexOf(carrier)).toBe(messages.indexOf(tool) + 1); |
| 128 | +}); |
| 129 | + |
| 130 | +test("interrupted round: the synthetic closure still flushes collected images", () => { |
| 131 | + const messages = wire([ |
| 132 | + assistantWithCalls([{ id: "call_1", name: "shot" }, { id: "call_2", name: "list" }]), |
| 133 | + toolResult("call_1", "shot", [{ type: "image", imageUrl: IMAGE_URL }]), |
| 134 | + ]); |
| 135 | + assertRoundsUnbroken(messages); |
| 136 | + const carrier = messages.find(isImageCarrier)!; |
| 137 | + expect(carrier).toBeDefined(); |
| 138 | + expect(messages.indexOf(carrier)).toBe(messages.length - 1); |
| 139 | +}); |
| 140 | + |
| 141 | +test("image-free tool results emit no carrier and an unchanged wire", () => { |
| 142 | + const messages = wire([ |
| 143 | + user("hi"), |
| 144 | + assistantWithCalls([{ id: "call_1", name: "list" }]), |
| 145 | + toolResult("call_1", "list", "file1.txt"), |
| 146 | + user("thanks"), |
| 147 | + ]); |
| 148 | + expect(messages.some(isImageCarrier)).toBe(false); |
| 149 | + expect(messages.map(m => m.role)).toEqual(["user", "assistant", "tool", "user"]); |
| 150 | +}); |
0 commit comments