From 6c7494f64cab4d9364c6903b0f180485e19caf34 Mon Sep 17 00:00:00 2001 From: Yuxin Qiao <104957188+Yuxin-Qiao@users.noreply.github.com> Date: Sun, 2 Aug 2026 23:26:28 +0800 Subject: [PATCH] fix(anthropic): complete AgentRouter streams that end before terminal frames (#658) --- .../ja/reference/configuration/providers.md | 1 + .../ko/reference/configuration/providers.md | 1 + .../docs/reference/configuration/providers.md | 1 + .../ru/reference/configuration/providers.md | 1 + .../reference/configuration/providers.md | 1 + src/adapters/anthropic.ts | 62 ++++++++- src/providers/free-directory.ts | 4 +- src/types.ts | 7 + tests/anthropic-eof-tolerance.test.ts | 122 ++++++++++++++++++ 9 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 tests/anthropic-eof-tolerance.test.ts diff --git a/docs-site/src/content/docs/ja/reference/configuration/providers.md b/docs-site/src/content/docs/ja/reference/configuration/providers.md index c9875cd13..606193f4d 100644 --- a/docs-site/src/content/docs/ja/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ja/reference/configuration/providers.md @@ -80,6 +80,7 @@ description: プロバイダー エントリ、認証、エンドポイント、 | `thinkingBudgetModels?` | `string[]` |整数 `thinking_budget` を使用したチャット モデル。労力は予算の一部にマッピングされます。 | | `noVisionModels?` | `string[]` |ビジョン サイドカーを通じて送信されるテキストのみのモデル。マッチングでは、Ollama `:size` タグが許容されます。 | | `escapeBuiltinToolNames?` | `boolean` | Anthropic 互換ゲートウェイの組み込みツール名をエスケープし、返された呼び出しで復元します。 | +| `anthropicEofTolerance?` | `boolean` | `message_stop` 前にストリームが終了しても、可視テキストまたは完全な JSON オブジェクトのツール入力が受信済みの場合に限り完了を許可します（Anthropic 互換ゲートウェイ向け）。デフォルトはオフ。 | | `googleMode?` | `"ai-studio" \| "vertex" \| "cloud-code-assist"` | Google トランスポート/認証モード。デフォルトは`ai-studio`です。 | | `project?` | `string` | Vertex または Antigravity Cloud Code Assist プロジェクト ID。 | | `location?` | `string` |頂点の位置。環境フォールバックは `GOOGLE_CLOUD_LOCATION` です。 | diff --git a/docs-site/src/content/docs/ko/reference/configuration/providers.md b/docs-site/src/content/docs/ko/reference/configuration/providers.md index b3adefda8..7f9b0a492 100644 --- a/docs-site/src/content/docs/ko/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ko/reference/configuration/providers.md @@ -80,6 +80,7 @@ description: 공급자 항목, 인증, 엔드포인트, 모델 카탈로그, 할 | `thinkingBudgetModels?` | `string[]` | 정수 `thinking_budget`를 쓰는 chat 모델입니다. effort는 예산 비율로 매핑됩니다. | | `noVisionModels?` | `string[]` | vision sidecar로 보내는 텍스트 전용 모델입니다. 일치 판정은 Ollama `:size` 태그도 허용합니다. | | `escapeBuiltinToolNames?` | `boolean` | Anthropic 호환 게이트웨이를 위해 내장 도구 이름을 이스케이프하고, 반환된 호출에서는 다시 복원합니다. | +| `anthropicEofTolerance?` | `boolean` | `message_stop` 전에 스트림이 끝나도 표시 텍스트 또는 완전한 JSON 객체 툴 입력을 받은 경우에만 완료를 허용합니다（Anthropic 호환 게이트웨이용）. 기본값은 꺼짐. | | `googleMode?` | `"ai-studio" \| "vertex" \| "cloud-code-assist"` | Google 전송/인증 모드입니다. 기본값은 `ai-studio`입니다. | | `project?` | `string` | Vertex 또는 Antigravity Cloud Code Assist 프로젝트 id입니다. | | `location?` | `string` | Vertex 위치입니다. 환경 변수 폴백은 `GOOGLE_CLOUD_LOCATION`입니다. | diff --git a/docs-site/src/content/docs/reference/configuration/providers.md b/docs-site/src/content/docs/reference/configuration/providers.md index 5109a9d8e..9b587e5cd 100644 --- a/docs-site/src/content/docs/reference/configuration/providers.md +++ b/docs-site/src/content/docs/reference/configuration/providers.md @@ -91,6 +91,7 @@ differing backup and rewrites known legacy namespaced selected ids to bare ids. | `thinkingBudgetModels?` | `string[]` | Chat models using integer `thinking_budget`; effort maps to a budget fraction. | | `noVisionModels?` | `string[]` | Text-only models sent through the vision sidecar; matching tolerates an Ollama `:size` tag. | | `escapeBuiltinToolNames?` | `boolean` | Escape built-in tool names for Anthropic-compatible gateways and restore them in returned calls. | +| `anthropicEofTolerance?` | `boolean` | Let an Anthropic-compatible gateway complete a stream that ends before `message_stop`, only when visible text or a complete JSON-object tool input was received. Off by default. | | `googleMode?` | `"ai-studio" \| "vertex" \| "cloud-code-assist"` | Google transport/auth mode. Default `ai-studio`. | | `project?` | `string` | Vertex or Antigravity Cloud Code Assist project id. | | `location?` | `string` | Vertex location; environment fallback is `GOOGLE_CLOUD_LOCATION`. | diff --git a/docs-site/src/content/docs/ru/reference/configuration/providers.md b/docs-site/src/content/docs/ru/reference/configuration/providers.md index 6805a6e55..28494c132 100644 --- a/docs-site/src/content/docs/ru/reference/configuration/providers.md +++ b/docs-site/src/content/docs/ru/reference/configuration/providers.md @@ -96,6 +96,7 @@ cross-route credential fallback не существует. Строки API GPT- | `thinkingBudgetModels?` | `string[]` | Chat-модели, использующие целочисленный `thinking_budget`; effort отображается в долю бюджета. | | `noVisionModels?` | `string[]` | Text-only-модели, идущие через vision sidecar; при сопоставлении tolerируется тег Ollama вида `:size`. | | `escapeBuiltinToolNames?` | `boolean` | Экранировать built-in tool name'ы для Anthropic-compatible gateway'ев и восстанавливать их в возвращаемых call'ах. | +| `anthropicEofTolerance?` | `boolean` | Позволяет Anthropic-совместимому шлюзу завершить поток до `message_stop`, только если получен видимый текст или полный JSON-объект аргументов инструмента. По умолчанию выключено. | | `googleMode?` | `"ai-studio" \| "vertex" \| "cloud-code-assist"` | Режим транспорта/аутентификации Google. По умолчанию `ai-studio`. | | `project?` | `string` | Идентификатор проекта Vertex или Antigravity Cloud Code Assist. | | `location?` | `string` | Локация Vertex; fallback через окружение — `GOOGLE_CLOUD_LOCATION`. | diff --git a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md index 71f5b9218..67da0af88 100644 --- a/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md +++ b/docs-site/src/content/docs/zh-cn/reference/configuration/providers.md @@ -80,6 +80,7 @@ description: 提供者条目、身份验证、端点、模型目录、配额、 | `thinkingBudgetModels?` | `string[]` | 使用整数 `thinking_budget` 的 chat 模型；effort 会映射为预算比例。 | | `noVisionModels?` | `string[]` | 经由视觉 sidecar 发送的纯文本模型；匹配时会容忍 Ollama 的 `:size` 标记。 | | `escapeBuiltinToolNames?` | `boolean` | 为 Anthropic 兼容网关转义内置工具名，并在返回的调用中恢复。 | +| `anthropicEofTolerance?` | `boolean` | 允许 Anthropic 兼容网关在 `message_stop` 前结束流，仅当已收到可见文本或完整的 JSON 对象工具输入时。默认关闭。 | | `googleMode?` | `"ai-studio" \| "vertex" \| "cloud-code-assist"` | Google 传输/身份验证模式。默认 `ai-studio`。 | | `project?` | `string` | Vertex 或 Antigravity Cloud Code Assist 项目 id。 | | `location?` | `string` | Vertex 位置；环境变量回退为 `GOOGLE_CLOUD_LOCATION`。 | diff --git a/src/adapters/anthropic.ts b/src/adapters/anthropic.ts index 791fbcd35..afce8cf2b 100644 --- a/src/adapters/anthropic.ts +++ b/src/adapters/anthropic.ts @@ -277,7 +277,39 @@ function usableToolUseId(id: unknown): string { return typeof id === "string" && id.trim() ? id : synthesizeToolUseId(); } -function toolUseArguments(input: unknown): string { +/** + * Bound repair for a malformed tool-arguments string under the compatibility profile (#658): + * a gateway such as AgentRouter can concatenate JSON objects (`{}{"value":42}`). Find the + * last parseable JSON object by scanning suffixes from each object-open brace and prefixes + * ending at each object-close brace, bounded so hostile input cannot cost unbounded time. + */ +function lastValidJsonObject(input: string, maxCandidates: number): string | undefined { + const opens: number[] = []; + const closes: number[] = []; + for (let i = 0; i < input.length; i++) { + if (input[i] === "{") opens.push(i); + else if (input[i] === "}") closes.push(i); + } + let tried = 0; + for (let i = opens.length - 1; i >= 0 && tried < maxCandidates; i--, tried++) { + const candidate = input.slice(opens[i]); + try { + const parsed = JSON.parse(candidate) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return candidate; + } catch { /* keep scanning */ } + } + tried = 0; + for (let i = closes.length - 1; i >= 0 && tried < maxCandidates; i--, tried++) { + const candidate = input.slice(0, closes[i] + 1); + try { + const parsed = JSON.parse(candidate) as unknown; + if (parsed && typeof parsed === "object" && !Array.isArray(parsed)) return candidate; + } catch { /* keep scanning */ } + } + return undefined; +} + +function toolUseArguments(input: unknown, lenient = false): string { if (typeof input === "string") { const trimmed = input.trim(); if (!trimmed) return "{}"; @@ -285,6 +317,10 @@ function toolUseArguments(input: unknown): string { JSON.parse(trimmed); return trimmed; } catch { + if (lenient) { + const repaired = lastValidJsonObject(trimmed, 32); + if (repaired !== undefined) return repaired; + } // A tool call's arguments must be a JSON object. Re-encoding an unparseable string as a // JSON *string* is the double-encoding #765 reports: the caller then receives // `"get weather"` where an object was required and the tool call is unusable either way. @@ -798,6 +834,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti let pendingUsage: Record | undefined; let pendingStopReason: string | undefined; let emittedDone = false; + let sawVisibleText = false; const emitDone = function* (): Generator { if (emittedDone) return; @@ -853,6 +890,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti const delta = data.delta as Record | undefined; if (!delta) break; if (delta.type === "text_delta" && typeof delta.text === "string") { + sawVisibleText = true; yield { type: "text_delta", text: delta.text }; } else if (delta.type === "thinking_delta" && typeof delta.thinking === "string") { yield { type: "thinking_delta", thinking: delta.thinking }; @@ -951,6 +989,26 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti usage: usageFromAnthropic(pendingUsage), ...(stopReason ? { stopReason } : {}), }; + } else if (provider.anthropicEofTolerance === true) { + // AgentRouter-style compatibility profile (#658): the upstream can close the stream + // after valid content without terminal frames. Complete only when visible text was + // received or an open tool call has complete JSON-object arguments; everything else + // (incomplete tool JSON, no usable content, transport failure) stays a truncation + // error, matching the strict default. + if (currentToolCallId) { + if (streamedToolArgumentsParse(currentToolCallJson)) { + budget.closeCall(currentToolCallId); + currentToolCallId = ""; + yield { type: "tool_call_end" }; + yield* emitDone(); + } else { + yield { type: "error", message: "upstream stream ended before message_stop — possible truncation" }; + } + } else if (sawVisibleText) { + yield* emitDone(); + } else { + yield { type: "error", message: "upstream stream ended before message_stop — possible truncation" }; + } } else { yield { type: "error", message: "upstream stream ended before message_stop — possible truncation" }; } @@ -980,7 +1038,7 @@ export function createAnthropicAdapter(provider: OcxProviderConfig, cacheRetenti } else if (block.type === "tool_use") { const id = usableToolUseId(block.id); events.push({ type: "tool_call_start", id, name: toolNames.fromWire(block.name ?? "") }); - events.push({ type: "tool_call_delta", arguments: toolUseArguments(block.input) }); + events.push({ type: "tool_call_delta", arguments: toolUseArguments(block.input, provider.anthropicEofTolerance === true) }); events.push({ type: "tool_call_end" }); } } diff --git a/src/providers/free-directory.ts b/src/providers/free-directory.ts index c390d00ed..9d96063dd 100644 --- a/src/providers/free-directory.ts +++ b/src/providers/free-directory.ts @@ -45,6 +45,8 @@ export interface FreeDirectoryProvider { keyOptional?: boolean; models?: string[]; liveModels: boolean; + /** Anthropic-compatible gateways that may close streams before terminal frames. */ + anthropicEofTolerance?: boolean; note?: string; googleMode?: "ai-studio" | "vertex"; } @@ -116,7 +118,7 @@ const CONNECTABLE: Record = { // `unverified` and drops the shared verification date rather than borrowing it. bytez: openAi("https://api.bytez.com/models/v2/openai/v1", "https://bytez.com", { verification: "unverified", lastVerified: undefined, documentationUrl: "https://docs.bytez.com/", discovery: "static", liveModels: false, models: ["meta-llama/Llama-3.3-70B-Instruct", "mistralai/Mistral-7B-Instruct-v0.3", "Qwen/Qwen2.5-72B-Instruct"], note: "The recurring-credit classification is retained from the requested catalog, but the current reset terms could not be independently verified." }), "nous-research": openAi("https://inference-api.nousresearch.com/v1", "https://portal.nousresearch.com", { discovery: "static", liveModels: false, models: ["Hermes-4-405B", "Hermes-4-70B"] }), - agentrouter: { baseUrl: "https://agentrouter.org", dashboardUrl: "https://agentrouter.org", adapter: "anthropic", authKind: "key", supportLevel: "experimental", verification: "primary", modelsUrl: "https://agentrouter.org/v1/models", lastVerified: LAST_VERIFIED, discovery: "live", liveModels: true }, + agentrouter: { baseUrl: "https://agentrouter.org", dashboardUrl: "https://agentrouter.org", adapter: "anthropic", authKind: "key", supportLevel: "experimental", verification: "primary", modelsUrl: "https://agentrouter.org/v1/models", lastVerified: LAST_VERIFIED, discovery: "live", liveModels: true, anthropicEofTolerance: true }, ai21: openAi("https://api.ai21.com/studio/v1", "https://studio.ai21.com/account/api-key", { supportLevel: "supported", verification: "official", documentationUrl: "https://docs.ai21.com/reference/models" }), baichuan: openAi("https://api.baichuan-ai.com/v1", "https://platform.baichuan-ai.com/console/apikey", { verification: "official" }), // Verified end-to-end 2026-07-30: /v1/models returns the OpenAI-shaped live catalog (13 models), diff --git a/src/types.ts b/src/types.ts index c4827a8fa..f14aa7d87 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1104,6 +1104,13 @@ export interface OcxProviderConfig { thinkingBudgetModels?: string[]; /** Anthropic-compatible gateways that need custom tool names escaped on the wire. */ escapeBuiltinToolNames?: boolean; + /** + * Anthropic-compatible gateways (e.g. AgentRouter) that may close the stream before + * `message_stop`. With this enabled the adapter completes an otherwise-clean EOF only when + * visible text was received or an open tool call has complete JSON-object arguments; all + * other EOFs remain truncation errors. Absent = strict default behavior. + */ + anthropicEofTolerance?: boolean; /** * Model ids that do NOT accept image inputs. The proxy gives them "eyes" via the vision sidecar: * attached images are described by a gpt vision model and replaced with text before the call. diff --git a/tests/anthropic-eof-tolerance.test.ts b/tests/anthropic-eof-tolerance.test.ts new file mode 100644 index 000000000..5c9712810 --- /dev/null +++ b/tests/anthropic-eof-tolerance.test.ts @@ -0,0 +1,122 @@ +import { describe, expect, test } from "bun:test"; +import { createAnthropicAdapter as createAnthropicAdapterProduction } from "../src/adapters/anthropic"; +import { FREE_PROVIDER_DIRECTORY } from "../src/providers/free-directory"; +import type { AdapterEvent, OcxProviderConfig } from "../src/types"; +import { withTestTranslatorBudget } from "./helpers/translator-budget"; + +/** + * #658: AgentRouter's Anthropic-compatible endpoint can close the stream before + * `content_block_stop`, `message_delta`, and `message_stop`. The default adapter treats + * that EOF as a fatal truncation; with `anthropicEofTolerance` enabled it may complete + * only when visible text was received or an open tool call has complete JSON-object + * arguments. These tests pin the wire behavior; no request reaches agentrouter.org. + */ + +const createAnthropicAdapter = (...args: Parameters) => + withTestTranslatorBudget(createAnthropicAdapterProduction(...args)); + +function providerFor(extra: Partial = {}): OcxProviderConfig { + return { + adapter: "anthropic", + baseUrl: "https://agentrouter.org", + apiKey: "test-key", + authMode: "key", + ...extra, + } as OcxProviderConfig; +} + +const strict = providerFor(); +const tolerant = providerFor({ anthropicEofTolerance: true }); + +const TRUNCATION = "upstream stream ended before message_stop — possible truncation"; + +function sseResponse(events: string[]): Response { + return new Response(events.join("\n\n"), { headers: { "content-type": "text/event-stream" } }); +} + +async function collect(provider: OcxProviderConfig, events: string[]): Promise { + const out: AdapterEvent[] = []; + for await (const event of createAnthropicAdapter(provider).parseStream(sseResponse(events))) out.push(event); + return out; +} + +const textEof = [ + 'event: message_start\ndata: {"type":"message_start","message":{"usage":{"input_tokens":2}}}', + 'event: content_block_start\ndata: {"type":"content_block_start","index":0,"content_block":{"type":"text","text":""}}', + 'event: content_block_delta\ndata: {"type":"content_block_delta","index":0,"delta":{"type":"text_delta","text":"visible"}}', +]; + +function toolEof(partialJson: string, id = "toolu_1"): string[] { + return [ + 'event: message_start\ndata: {"type":"message_start","message":{}}', + `event: content_block_start\ndata: ${JSON.stringify({ type: "content_block_start", index: 0, content_block: { type: "tool_use", id, name: "get_weather" } })}`, + `event: content_block_delta\ndata: ${JSON.stringify({ type: "content_block_delta", index: 0, delta: { type: "input_json_delta", partial_json: partialJson } })}`, + ]; +} + +describe("AgentRouter Anthropic EOF tolerance (#658)", () => { + test("text EOF completes when anthropicEofTolerance is enabled", async () => { + const events = await collect(tolerant, textEof); + + expect(events).toContainEqual({ type: "text_delta", text: "visible" }); + expect(events.at(-1)).toEqual({ type: "done", usage: { inputTokens: 2, outputTokens: 0 } }); + expect(events.some(event => event.type === "error")).toBe(false); + }); + + test("the same EOF without the capability stays a truncation error", async () => { + const events = await collect(strict, textEof); + + expect(events.at(-1)).toEqual({ type: "error", message: TRUNCATION }); + expect(events.some(event => event.type === "done")).toBe(false); + }); + + test("a complete tool call at EOF closes and completes", async () => { + const events = await collect(tolerant, toolEof('{"value":42}')); + + expect(events).toContainEqual({ type: "tool_call_start", id: "toolu_1", name: "get_weather" }); + expect(events).toContainEqual({ type: "tool_call_delta", arguments: '{"value":42}' }); + expect(events.at(-1)).toEqual({ type: "done", usage: undefined }); + expect(events.some(event => event.type === "error")).toBe(false); + }); + + test("an incomplete tool call at EOF remains a truncation error", async () => { + const events = await collect(tolerant, toolEof('{"value":')); + + expect(events.at(-1)).toEqual({ type: "error", message: TRUNCATION }); + expect(events.some(event => event.type === "done" || event.type === "tool_call_end")).toBe(false); + }); + + test("EOF before any usable content remains a truncation error", async () => { + const events = await collect(tolerant, [ + 'event: message_start\ndata: {"type":"message_start","message":{}}', + ]); + + expect(events.at(-1)).toEqual({ type: "error", message: TRUNCATION }); + }); + + test("a missing tool_use id gets a stable synthesized id on the tolerant path", async () => { + const events = await collect(tolerant, toolEof('{"value":42}', "")); + const start = events.find(event => event.type === "tool_call_start"); + + expect(start?.type).toBe("tool_call_start"); + expect((start as { id: string }).id).toMatch(/^toolu_[0-9a-f]{24}$/); + expect(events.at(-1)).toEqual({ type: "done", usage: undefined }); + }); + + test("non-stream concatenated tool input keeps the last valid object when enabled", async () => { + const payload = JSON.stringify({ + content: [{ type: "tool_use", id: "toolu_1", name: "get_weather", input: '{}{"value":42}' }], + }); + + const tolerantEvents = await createAnthropicAdapter(tolerant).parseResponse(new Response(payload)); + expect(tolerantEvents).toContainEqual({ type: "tool_call_delta", arguments: '{"value":42}' }); + + const strictEvents = await createAnthropicAdapter(strict).parseResponse(new Response(payload)); + expect(strictEvents).toContainEqual({ type: "tool_call_delta", arguments: "{}" }); + }); + + test("the AgentRouter directory row declares the EOF tolerance capability", () => { + const row = FREE_PROVIDER_DIRECTORY.find(provider => provider.id === "agentrouter"); + expect(row?.anthropicEofTolerance).toBe(true); + }); +});