From 540e5fbf46fe69c13f431dfb6d222cfb59edf3bc Mon Sep 17 00:00:00 2001 From: DavidShawa Date: Sun, 19 Jul 2026 00:32:55 -0700 Subject: [PATCH] fix(openai): scope prompt cache key to official endpoint, unify usage mapping, tier ChatGPT model routing --- packages/@ant/model-provider/src/index.ts | 4 + .../__tests__/openaiStreamAdapter.test.ts | 60 ++++++++++++++- .../src/shared/__tests__/openaiUsage.test.ts | 52 +++++++++++++ .../src/shared/openaiStreamAdapter.ts | 61 +++++++++------ .../model-provider/src/shared/openaiUsage.ts | 35 +++++++++ .../api/openai/__tests__/openaiShared.test.ts | 50 ++++++++++++ .../__tests__/queryModelOpenAI.isolated.ts | 47 +++++++++++- .../openai/__tests__/responsesAdapter.test.ts | 26 ++----- .../api/openai/__tests__/thinking.test.ts | 16 +++- src/services/api/openai/index.ts | 29 +++++-- src/services/api/openai/openaiShared.ts | 53 ++++++++----- src/services/api/openai/requestBody.ts | 7 +- src/services/api/openai/responsesAdapter.ts | 36 +++------ .../__tests__/sideQuery.chatgptAuth.test.ts | 47 +++++++++++- .../__tests__/chatgptModelRouting.test.ts | 76 +++++++++++++++++++ src/utils/model/chatgptModels.ts | 25 ++++++ src/utils/model/model.ts | 59 ++++++++------ src/utils/sideQuery.ts | 47 ++++++++++-- 18 files changed, 585 insertions(+), 145 deletions(-) create mode 100644 packages/@ant/model-provider/src/shared/__tests__/openaiUsage.test.ts create mode 100644 packages/@ant/model-provider/src/shared/openaiUsage.ts create mode 100644 src/services/api/openai/__tests__/openaiShared.test.ts create mode 100644 src/utils/model/__tests__/chatgptModelRouting.test.ts diff --git a/packages/@ant/model-provider/src/index.ts b/packages/@ant/model-provider/src/index.ts index 6f2b1a56ce..823edb3812 100644 --- a/packages/@ant/model-provider/src/index.ts +++ b/packages/@ant/model-provider/src/index.ts @@ -67,3 +67,7 @@ export { anthropicToolChoiceToOpenAI, } from './shared/openaiConvertTools.js' export { adaptOpenAIStreamToAnthropic } from './shared/openaiStreamAdapter.js' +export { + normalizeOpenAIUsage, + type AnthropicUsage, +} from './shared/openaiUsage.js' diff --git a/packages/@ant/model-provider/src/shared/__tests__/openaiStreamAdapter.test.ts b/packages/@ant/model-provider/src/shared/__tests__/openaiStreamAdapter.test.ts index 8e8e15000c..9aba0fb6c2 100644 --- a/packages/@ant/model-provider/src/shared/__tests__/openaiStreamAdapter.test.ts +++ b/packages/@ant/model-provider/src/shared/__tests__/openaiStreamAdapter.test.ts @@ -34,11 +34,15 @@ function makeChunk( } /** Collect all emitted Anthropic events from the stream adapter for assertion */ -async function collectEvents(chunks: ChatCompletionChunk[]) { +async function collectEvents( + chunks: ChatCompletionChunk[], + options?: { includeCacheWriteTokens?: boolean }, +) { const events: any[] = [] for await (const event of adaptOpenAIStreamToAnthropic( mockStream(chunks), 'gpt-4o', + options, )) { events.push(event) } @@ -521,6 +525,60 @@ describe('thinking support (reasoning_content)', () => { }) describe('prompt caching support', () => { + test('maps official OpenAI cache writes when explicitly enabled', async () => { + const events = await collectEvents( + [ + makeChunk({ + choices: [ + { index: 0, delta: { content: 'hi' }, finish_reason: null }, + ], + }), + makeChunk({ + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { + prompt_tokens: 1000, + completion_tokens: 50, + total_tokens: 1050, + prompt_tokens_details: { + cached_tokens: 600, + cache_write_tokens: 250, + }, + } as any, + }), + ], + { includeCacheWriteTokens: true }, + ) + + const msgDelta = events.find(e => e.type === 'message_delta') as any + expect(msgDelta.usage.input_tokens).toBe(150) + expect(msgDelta.usage.cache_read_input_tokens).toBe(600) + expect(msgDelta.usage.cache_creation_input_tokens).toBe(250) + }) + + test('ignores cache writes for compatible providers by default', async () => { + const events = await collectEvents([ + makeChunk({ + choices: [{ index: 0, delta: { content: 'hi' }, finish_reason: null }], + }), + makeChunk({ + choices: [{ index: 0, delta: {}, finish_reason: 'stop' }], + usage: { + prompt_tokens: 1000, + completion_tokens: 50, + total_tokens: 1050, + prompt_tokens_details: { + cached_tokens: 600, + cache_write_tokens: 250, + }, + } as any, + }), + ]) + + const msgDelta = events.find(e => e.type === 'message_delta') as any + expect(msgDelta.usage.input_tokens).toBe(400) + expect(msgDelta.usage.cache_creation_input_tokens).toBe(0) + }) + test('maps cached_tokens to cache_read_input_tokens', async () => { const events = await collectEvents([ makeChunk({ diff --git a/packages/@ant/model-provider/src/shared/__tests__/openaiUsage.test.ts b/packages/@ant/model-provider/src/shared/__tests__/openaiUsage.test.ts new file mode 100644 index 0000000000..3d28f06e4a --- /dev/null +++ b/packages/@ant/model-provider/src/shared/__tests__/openaiUsage.test.ts @@ -0,0 +1,52 @@ +import { describe, expect, test } from 'bun:test' +import { normalizeOpenAIUsage } from '../openaiUsage.js' + +describe('normalizeOpenAIUsage', () => { + test('partitions total input into ordinary, cache-read, and cache-write tokens', () => { + expect( + normalizeOpenAIUsage({ + totalInputTokens: 1000, + outputTokens: 50, + cacheReadTokens: 600, + cacheWriteTokens: 250, + }), + ).toEqual({ + input_tokens: 150, + output_tokens: 50, + cache_creation_input_tokens: 250, + cache_read_input_tokens: 600, + }) + }) + + test('clamps overlapping cache segments to the total input', () => { + expect( + normalizeOpenAIUsage({ + totalInputTokens: 5000, + outputTokens: 10, + cacheReadTokens: 4000, + cacheWriteTokens: 4000, + }), + ).toEqual({ + input_tokens: 0, + output_tokens: 10, + cache_creation_input_tokens: 1000, + cache_read_input_tokens: 4000, + }) + }) + + test('clamps negative provider values to zero', () => { + expect( + normalizeOpenAIUsage({ + totalInputTokens: -1, + outputTokens: -2, + cacheReadTokens: -3, + cacheWriteTokens: -4, + }), + ).toEqual({ + input_tokens: 0, + output_tokens: 0, + cache_creation_input_tokens: 0, + cache_read_input_tokens: 0, + }) + }) +}) diff --git a/packages/@ant/model-provider/src/shared/openaiStreamAdapter.ts b/packages/@ant/model-provider/src/shared/openaiStreamAdapter.ts index 8a14c58db9..1c9f608c03 100644 --- a/packages/@ant/model-provider/src/shared/openaiStreamAdapter.ts +++ b/packages/@ant/model-provider/src/shared/openaiStreamAdapter.ts @@ -1,6 +1,7 @@ import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' import type { ChatCompletionChunk } from 'openai/resources/chat/completions/completions.mjs' import { randomUUID } from 'crypto' +import { normalizeOpenAIUsage } from './openaiUsage.js' /** * Adapt an OpenAI streaming response into Anthropic BetaRawMessageStreamEvent. @@ -13,10 +14,10 @@ import { randomUUID } from 'crypto' * finish_reason → message_delta(stop_reason) + message_stop * * Usage field mapping (OpenAI → Anthropic): - * prompt_tokens - cached_tokens → input_tokens (non-cached input only) + * prompt_tokens - cached_tokens - cache_write_tokens → input_tokens * completion_tokens → output_tokens * prompt_tokens_details.cached_tokens → cache_read_input_tokens - * (no OpenAI equivalent) → cache_creation_input_tokens (always 0) + * prompt_tokens_details.cache_write_tokens → cache_creation_input_tokens * * All four fields are emitted in the post-loop message_delta (not message_start) * so that trailing usage chunks (sent after finish_reason by some @@ -35,6 +36,7 @@ import { randomUUID } from 'crypto' export async function* adaptOpenAIStreamToAnthropic( stream: AsyncIterable, model: string, + options?: { includeCacheWriteTokens?: boolean }, ): AsyncGenerator { const messageId = `msg_${randomUUID().replace(/-/g, '').slice(0, 24)}` @@ -53,13 +55,13 @@ export async function* adaptOpenAIStreamToAnthropic( // Track text block state let textBlockOpen = false - // Track usage — all four Anthropic fields, populated from OpenAI usage fields: - // rawInputTokens tracks the raw prompt_tokens (OpenAI total, including cached). - // inputTokens is the derived Anthropic value (non-cached only = rawInputTokens - cachedReadTokens). + // Track raw OpenAI usage across chunks. The normalized Anthropic fields are + // disjoint: ordinary input + cache reads + cache writes = total input. let rawInputTokens = 0 - let inputTokens = 0 let outputTokens = 0 - let cachedReadTokens = 0 + let rawCacheReadTokens = 0 + let rawCacheWriteTokens = 0 + let usage = normalizeOpenAIUsage({ totalInputTokens: 0, outputTokens: 0 }) // Track all open content block indices (for cleanup) const openBlockIndices = new Set() @@ -75,16 +77,32 @@ export async function* adaptOpenAIStreamToAnthropic( // Extract usage from any chunk that carries it. if (chunk.usage) { rawInputTokens = chunk.usage.prompt_tokens ?? rawInputTokens - const rawCached = - ((chunk.usage as any).prompt_tokens_details?.cached_tokens as - | number - | undefined) ?? cachedReadTokens - // Anthropic's input_tokens = non-cached input only. OpenAI's prompt_tokens - // includes cached tokens, so subtract. Clamp to 0 in case cached > total - // due to a streaming race. - inputTokens = Math.max(0, rawInputTokens - rawCached) outputTokens = chunk.usage.completion_tokens ?? outputTokens - cachedReadTokens = rawCached + + const usageRecord = chunk.usage as unknown as Record + const detailsValue = usageRecord.prompt_tokens_details + const details = + detailsValue && typeof detailsValue === 'object' + ? (detailsValue as Record) + : undefined + if (typeof details?.cached_tokens === 'number') { + rawCacheReadTokens = details.cached_tokens + } + if ( + options?.includeCacheWriteTokens && + typeof details?.cache_write_tokens === 'number' + ) { + rawCacheWriteTokens = details.cache_write_tokens + } else if (!options?.includeCacheWriteTokens) { + rawCacheWriteTokens = 0 + } + + usage = normalizeOpenAIUsage({ + totalInputTokens: rawInputTokens, + outputTokens, + cacheReadTokens: rawCacheReadTokens, + cacheWriteTokens: rawCacheWriteTokens, + }) } // Emit message_start on first chunk @@ -102,10 +120,8 @@ export async function* adaptOpenAIStreamToAnthropic( stop_reason: null, stop_sequence: null, usage: { - input_tokens: inputTokens, + ...usage, output_tokens: 0, - cache_creation_input_tokens: 0, - cache_read_input_tokens: cachedReadTokens, }, }, } as unknown as BetaRawMessageStreamEvent @@ -312,12 +328,7 @@ export async function* adaptOpenAIStreamToAnthropic( stop_reason: stopReason, stop_sequence: null, }, - usage: { - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_read_input_tokens: cachedReadTokens, - cache_creation_input_tokens: 0, - }, + usage, } as BetaRawMessageStreamEvent yield { diff --git a/packages/@ant/model-provider/src/shared/openaiUsage.ts b/packages/@ant/model-provider/src/shared/openaiUsage.ts new file mode 100644 index 0000000000..774decd4b9 --- /dev/null +++ b/packages/@ant/model-provider/src/shared/openaiUsage.ts @@ -0,0 +1,35 @@ +export type AnthropicUsage = { + input_tokens: number + output_tokens: number + cache_creation_input_tokens: number + cache_read_input_tokens: number +} + +/** + * Convert OpenAI's total-input usage into Anthropic's disjoint usage fields. + * Cache reads take priority when malformed provider data makes segments overlap. + */ +export function normalizeOpenAIUsage(params: { + totalInputTokens: number + outputTokens: number + cacheReadTokens?: number + cacheWriteTokens?: number +}): AnthropicUsage { + const totalInput = Math.max(0, params.totalInputTokens) + const cacheRead = Math.min( + Math.max(0, params.cacheReadTokens ?? 0), + totalInput, + ) + const remainingAfterRead = Math.max(0, totalInput - cacheRead) + const cacheCreation = Math.min( + Math.max(0, params.cacheWriteTokens ?? 0), + remainingAfterRead, + ) + + return { + input_tokens: Math.max(0, remainingAfterRead - cacheCreation), + output_tokens: Math.max(0, params.outputTokens), + cache_creation_input_tokens: cacheCreation, + cache_read_input_tokens: cacheRead, + } +} diff --git a/src/services/api/openai/__tests__/openaiShared.test.ts b/src/services/api/openai/__tests__/openaiShared.test.ts new file mode 100644 index 0000000000..75a1e81e78 --- /dev/null +++ b/src/services/api/openai/__tests__/openaiShared.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, test } from 'bun:test' +import { + getOfficialOpenAIPromptCacheKey, + isOfficialOpenAIBaseURL, +} from '../openaiShared.js' + +describe('isOfficialOpenAIBaseURL', () => { + test('treats the SDK default endpoint as official OpenAI', () => { + expect(isOfficialOpenAIBaseURL(undefined)).toBe(true) + expect(isOfficialOpenAIBaseURL('')).toBe(true) + }) + + test('accepts global and regional official OpenAI endpoints', () => { + expect(isOfficialOpenAIBaseURL('https://api.openai.com/v1')).toBe(true) + expect(isOfficialOpenAIBaseURL('https://eu.api.openai.com/v1')).toBe(true) + expect(isOfficialOpenAIBaseURL('https://api.openai.com:443/v1')).toBe(true) + }) + + test('rejects OpenAI-compatible and spoofed endpoints', () => { + expect(isOfficialOpenAIBaseURL('https://api.deepseek.com/v1')).toBe(false) + expect(isOfficialOpenAIBaseURL('http://api.openai.com/v1')).toBe(false) + expect(isOfficialOpenAIBaseURL('https://api.openai.com.evil.test/v1')).toBe( + false, + ) + expect(isOfficialOpenAIBaseURL('https://api.openai.com:8443/v1')).toBe( + false, + ) + expect(isOfficialOpenAIBaseURL('not-a-url')).toBe(false) + }) +}) + +describe('getOfficialOpenAIPromptCacheKey', () => { + test('returns a session key for the SDK default and official endpoint', () => { + expect(getOfficialOpenAIPromptCacheKey(undefined, 'session-1')).toBe( + 'ccb:session-1', + ) + expect( + getOfficialOpenAIPromptCacheKey('https://api.openai.com/v1', 'session-2'), + ).toBe('ccb:session-2') + }) + + test('returns undefined for compatible endpoints', () => { + expect( + getOfficialOpenAIPromptCacheKey( + 'https://api.deepseek.com/v1', + 'session-1', + ), + ).toBeUndefined() + }) +}) diff --git a/src/services/api/openai/__tests__/queryModelOpenAI.isolated.ts b/src/services/api/openai/__tests__/queryModelOpenAI.isolated.ts index 0aa42a4174..eea7565b14 100644 --- a/src/services/api/openai/__tests__/queryModelOpenAI.isolated.ts +++ b/src/services/api/openai/__tests__/queryModelOpenAI.isolated.ts @@ -220,6 +220,28 @@ mock.module('@ant/model-provider', () => ({ }, })), anthropicToolChoiceToOpenAI: () => undefined, + normalizeOpenAIUsage: (params: { + totalInputTokens: number + outputTokens: number + cacheReadTokens?: number + cacheWriteTokens?: number + }) => { + const cacheRead = Math.min( + Math.max(0, params.cacheReadTokens ?? 0), + Math.max(0, params.totalInputTokens), + ) + const remaining = Math.max(0, params.totalInputTokens - cacheRead) + const cacheCreation = Math.min( + Math.max(0, params.cacheWriteTokens ?? 0), + remaining, + ) + return { + input_tokens: Math.max(0, remaining - cacheCreation), + output_tokens: Math.max(0, params.outputTokens), + cache_creation_input_tokens: cacheCreation, + cache_read_input_tokens: cacheRead, + } + }, })) mock.module('../../../../services/analytics/growthbook.js', () => ({ @@ -347,8 +369,15 @@ mock.module('../../../../utils/modelCost.js', () => ({ getModelPricingString: () => undefined, })) -mock.module('../../../../services/langfuse/tracing.js', () => ({ +mock.module('src/services/langfuse/tracing.ts', () => ({ + createTrace: () => null, recordLLMObservation: () => {}, + recordToolObservation: () => {}, + createToolBatchSpan: () => null, + endToolBatchSpan: () => {}, + createSubagentTrace: () => null, + createChildSpan: () => null, + endTrace: () => {}, })) mock.module('../../../../services/langfuse/convert.js', () => ({ @@ -587,7 +616,7 @@ describe('queryModelOpenAI — stream_events forwarded', () => { }) describe('queryModelOpenAI — max_tokens forwarded to request', () => { - test('buildOpenAIRequestBody includes max_tokens in the request payload', async () => { + test('official OpenAI requests include max_tokens and a session cache key', async () => { _nextEvents = [ makeMessageStart(), makeContentBlockStart(0, 'text'), @@ -601,8 +630,18 @@ describe('queryModelOpenAI — max_tokens forwarded to request', () => { expect(_lastCreateArgs).not.toBeNull() expect(_lastCreateArgs!.max_tokens).toBe(8192) - // Process-sticky OpenAI cache routing (not message-derived) - expect(_lastCreateArgs!.prompt_cache_key).toMatch(/^ccb:[0-9a-f-]+$/i) + expect(_lastCreateArgs!.prompt_cache_key).toStartWith('ccb:') + }) + + test('compatible providers do not receive OpenAI cache parameters', async () => { + _nextEvents = [makeMessageStart(), makeMessageStop()] + + await runQueryModel(_nextEvents, { + OPENAI_BASE_URL: 'https://api.deepseek.com/v1', + }) + + expect(_lastCreateArgs).not.toBeNull() + expect('prompt_cache_key' in _lastCreateArgs!).toBe(false) }) }) diff --git a/src/services/api/openai/__tests__/responsesAdapter.test.ts b/src/services/api/openai/__tests__/responsesAdapter.test.ts index 5b32633e19..9bc3c6a37c 100644 --- a/src/services/api/openai/__tests__/responsesAdapter.test.ts +++ b/src/services/api/openai/__tests__/responsesAdapter.test.ts @@ -4,6 +4,8 @@ import { formatOpenAIPromptCacheKey } from '../openaiShared.js' import { calculateCacheHitRate } from '../../../../utils/cacheWarning.js' describe('buildResponsesRequest', () => { + const promptCacheKey = formatOpenAIPromptCacheKey('session-abc-123') + test('includes max reasoning effort for ChatGPT Responses requests', () => { const request = buildResponsesRequest({ model: 'gpt-5.6-sol', @@ -11,6 +13,7 @@ describe('buildResponsesRequest', () => { tools: [], toolChoice: undefined, reasoningEffort: 'max', + promptCacheKey, }) expect(request.reasoning).toEqual({ effort: 'max' }) @@ -23,6 +26,7 @@ describe('buildResponsesRequest', () => { tools: [], toolChoice: undefined, reasoningEffort: 'xhigh', + promptCacheKey, }) expect(request.reasoning).toEqual({ effort: 'xhigh' }) @@ -34,42 +38,24 @@ describe('buildResponsesRequest', () => { messages: [{ role: 'user', content: 'hello' }], tools: [], toolChoice: undefined, + promptCacheKey, }) as Record expect('max_output_tokens' in request).toBe(false) }) test('includes stable prompt_cache_key for session-sticky cache routing', () => { - const key = formatOpenAIPromptCacheKey('session-abc-123') const request = buildResponsesRequest({ model: 'gpt-5.6-sol', messages: [{ role: 'user', content: 'hello' }], tools: [], toolChoice: undefined, - promptCacheKey: key, + promptCacheKey, }) expect(request.prompt_cache_key).toBe('ccb:session-abc-123') }) - test('defaults prompt_cache_key to process-stable fallback when not overridden', () => { - const request = buildResponsesRequest({ - model: 'gpt-5.5', - messages: [{ role: 'user', content: 'hello' }], - tools: [], - toolChoice: undefined, - }) - const again = buildResponsesRequest({ - model: 'gpt-5.5', - messages: [{ role: 'user', content: 'next' }], - tools: [], - toolChoice: undefined, - }) - - expect(request.prompt_cache_key).toMatch(/^ccb:[0-9a-f-]+$/i) - expect(again.prompt_cache_key).toBe(request.prompt_cache_key) - }) - test('prompt_cache_key is stable across turns (not derived from messages)', () => { const key = formatOpenAIPromptCacheKey('same-session') const turn1 = buildResponsesRequest({ diff --git a/src/services/api/openai/__tests__/thinking.test.ts b/src/services/api/openai/__tests__/thinking.test.ts index 9ff9d92e38..a12b3f1ac8 100644 --- a/src/services/api/openai/__tests__/thinking.test.ts +++ b/src/services/api/openai/__tests__/thinking.test.ts @@ -203,8 +203,6 @@ describe('buildOpenAIRequestBody — thinking params', () => { messages: [{ role: 'user', content: 'hello' }], tools: [] as any[], toolChoice: undefined as any, - // Avoid depending on bootstrap session state in pure request-body tests. - promptCacheKey: 'ccb:test-session', } as any test('includes official DeepSeek API thinking format when enabled', () => { @@ -212,13 +210,23 @@ describe('buildOpenAIRequestBody — thinking params', () => { expect(body.thinking).toEqual({ type: 'enabled' }) }) - test('includes prompt_cache_key for sticky OpenAI cache routing', () => { + test('includes prompt_cache_key when supplied for the official OpenAI API', () => { const body = buildOpenAIRequestBody({ ...baseParams, enableThinking: false, maxTokens: 1024, + promptCacheKey: 'ccb:session-123', }) - expect(body.prompt_cache_key).toBe('ccb:test-session') + expect(body.prompt_cache_key).toBe('ccb:session-123') + }) + + test('does not send prompt_cache_key to compatible providers when omitted', () => { + const body = buildOpenAIRequestBody({ + ...baseParams, + enableThinking: false, + maxTokens: 1024, + }) + expect('prompt_cache_key' in body).toBe(false) }) test('includes vLLM/self-hosted thinking format when enabled', () => { diff --git a/src/services/api/openai/index.ts b/src/services/api/openai/index.ts index c201f12992..0aaaa763ec 100644 --- a/src/services/api/openai/index.ts +++ b/src/services/api/openai/index.ts @@ -13,8 +13,13 @@ import type { } from '../../../types/message.js' import type { AgentId } from '../../../types/ids.js' import type { Tools } from '../../../Tool.js' +import { getSessionId } from '../../../bootstrap/state.js' import { getOpenAIClient } from './client.js' -import { getOpenAIPromptCacheKey, updateOpenAIUsage } from './openaiShared.js' +import { + formatOpenAIPromptCacheKey, + getOfficialOpenAIPromptCacheKey, + updateOpenAIUsage, +} from './openaiShared.js' import { anthropicMessagesToOpenAI, resolveOpenAIModel, @@ -346,19 +351,26 @@ export async function* queryModelOpenAI( options.maxOutputTokensOverride, ) - // Sticky cache routing key: stable for this CCB process so OpenAI can - // co-locate multi-turn requests on the same cache-bearing node. Never hash - // the full message body (that changes every turn and defeats routing). - const promptCacheKey = getOpenAIPromptCacheKey() + const useChatGPTResponses = isChatGPTAuthEnabled() + // OpenAI's official OAuth and API-key routes share the same prompt-cache + // contract. Scope the key to the real conversation so resumed turns stay + // sticky while unrelated sessions do not share a routing bucket. Generic + // compatible endpoints intentionally receive no OpenAI-specific fields. + const sessionId = getSessionId() + const sessionPromptCacheKey = formatOpenAIPromptCacheKey(sessionId) + const promptCacheKey = useChatGPTResponses + ? sessionPromptCacheKey + : getOfficialOpenAIPromptCacheKey(process.env.OPENAI_BASE_URL, sessionId) + const useOfficialOpenAICache = promptCacheKey !== undefined logForDebugging( - `[OpenAI] Calling model=${openaiModel}, messages=${openaiMessages.length}, tools=${openaiTools.length}, thinking=${enableThinking}, prompt_cache_key=${promptCacheKey}`, + `[OpenAI] Calling model=${openaiModel}, messages=${openaiMessages.length}, tools=${openaiTools.length}, thinking=${enableThinking}${promptCacheKey ? `, prompt_cache_key=${promptCacheKey}` : ''}`, ) // 11. Call OpenAI API with streaming. ChatGPT subscription auth uses the // Codex Responses backend; API-key/OpenAI-compatible auth keeps the // existing Chat Completions adapter. - const adaptedStream = isChatGPTAuthEnabled() + const adaptedStream = useChatGPTResponses ? adaptResponsesStreamToAnthropic( await createChatGPTResponsesStream({ request: buildResponsesRequest({ @@ -367,7 +379,7 @@ export async function* queryModelOpenAI( tools: openaiTools, toolChoice: openaiToolChoice, reasoningEffort, - promptCacheKey, + promptCacheKey: sessionPromptCacheKey, }), signal, fetchOverride: options.fetchOverride as unknown as typeof fetch, @@ -393,6 +405,7 @@ export async function* queryModelOpenAI( { signal }, ), openaiModel, + { includeCacheWriteTokens: useOfficialOpenAICache }, ) // 12. Convert OpenAI stream to Anthropic events, then process into diff --git a/src/services/api/openai/openaiShared.ts b/src/services/api/openai/openaiShared.ts index 0521c48cdc..6a7e32cdd3 100644 --- a/src/services/api/openai/openaiShared.ts +++ b/src/services/api/openai/openaiShared.ts @@ -8,7 +8,31 @@ * Keep this module free of bootstrap/state imports so pure request-body unit * tests and isolated mocks do not need a full session runtime. */ -import { randomUUID } from 'crypto' + +/** + * Whether a configured base URL resolves directly to OpenAI's official API. + * + * An absent URL means the OpenAI SDK default (`api.openai.com`). Regional + * endpoints are subdomains of `api.openai.com`. Keep this strict so generic + * OpenAI-compatible providers never receive OpenAI-specific cache parameters. + */ +export function isOfficialOpenAIBaseURL(baseURL: string | undefined): boolean { + if (!baseURL?.trim()) return true + + try { + const url = new URL(baseURL) + const isOfficialHost = + url.hostname === 'api.openai.com' || + url.hostname.endsWith('.api.openai.com') + return ( + url.protocol === 'https:' && + isOfficialHost && + (url.port === '' || url.port === '443') + ) + } catch { + return false + } +} /** * Build a stable OpenAI `prompt_cache_key` for a session. @@ -25,25 +49,16 @@ export function formatOpenAIPromptCacheKey(sessionId: string): string { } /** - * Process-scoped sticky key. OpenAI uses this for cache-node routing, not as a - * content hash — it only needs to be stable across multi-turn requests in the - * same CCB process. Avoids a bootstrap/state import so pure unit tests and - * partial mocks stay isolated. - */ -let processPromptCacheKey: string | null = null - -/** - * Stable OpenAI `prompt_cache_key` for this process. - * Prefer an explicit override (session id) when the caller already has one. + * Return a session-sticky cache key only for OpenAI's official API endpoint. + * Compatible providers must not receive OpenAI-specific request parameters. */ -export function getOpenAIPromptCacheKey(sessionIdOverride?: string): string { - if (sessionIdOverride) { - return formatOpenAIPromptCacheKey(sessionIdOverride) - } - if (!processPromptCacheKey) { - processPromptCacheKey = formatOpenAIPromptCacheKey(randomUUID()) - } - return processPromptCacheKey +export function getOfficialOpenAIPromptCacheKey( + baseURL: string | undefined, + sessionId: string, +): string | undefined { + return isOfficialOpenAIBaseURL(baseURL) + ? formatOpenAIPromptCacheKey(sessionId) + : undefined } /** diff --git a/src/services/api/openai/requestBody.ts b/src/services/api/openai/requestBody.ts index a31e377667..16043150b4 100644 --- a/src/services/api/openai/requestBody.ts +++ b/src/services/api/openai/requestBody.ts @@ -5,7 +5,6 @@ */ import type { ChatCompletionCreateParamsStreaming } from 'openai/resources/chat/completions/completions.mjs' import { isEnvTruthy, isEnvDefinedFalsy } from '../../../utils/envUtils.js' -import { getOpenAIPromptCacheKey } from './openaiShared.js' /** * Detect whether thinking mode should be enabled for this model. @@ -76,7 +75,7 @@ export function buildOpenAIRequestBody(params: { enableThinking: boolean maxTokens: number temperatureOverride?: number - /** Override for tests; production uses the current CCB session id. */ + /** Session-scoped routing key for official OpenAI requests. */ promptCacheKey?: string }): ChatCompletionCreateParamsStreaming & { thinking?: { type: string } @@ -99,15 +98,13 @@ export function buildOpenAIRequestBody(params: { model, messages, max_tokens: maxTokens, + ...(promptCacheKey && { prompt_cache_key: promptCacheKey }), ...(tools.length > 0 && { tools, ...(toolChoice && { tool_choice: toolChoice }), }), stream: true, stream_options: { include_usage: true }, - // Sticky cache routing for multi-turn OpenAI/compatible endpoints that - // honor prompt_cache_key. Process-stable by default (not message-derived). - prompt_cache_key: promptCacheKey ?? getOpenAIPromptCacheKey(), // Enable chain-of-thought output for DeepSeek and MiMo models. // When active, temperature/top_p/presence_penalty/frequency_penalty are ignored. ...(enableThinking && { diff --git a/src/services/api/openai/responsesAdapter.ts b/src/services/api/openai/responsesAdapter.ts index 31f9c4c2d8..74b3c739a7 100644 --- a/src/services/api/openai/responsesAdapter.ts +++ b/src/services/api/openai/responsesAdapter.ts @@ -1,7 +1,7 @@ import { randomUUID } from 'crypto' import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' +import { normalizeOpenAIUsage, type AnthropicUsage } from '@ant/model-provider' import { getValidChatGPTAuth } from './chatgptAuth.js' -import { getOpenAIPromptCacheKey } from './openaiShared.js' type ResponsesInputItem = Record type ResponsesTool = Record @@ -26,13 +26,6 @@ type ResponsesRequest = { prompt_cache_key: string } -type AnthropicUsage = { - input_tokens: number - output_tokens: number - cache_creation_input_tokens: number - cache_read_input_tokens: number -} - function textFromContent(content: unknown): string { if (typeof content === 'string') return content if (!Array.isArray(content)) return '' @@ -176,8 +169,8 @@ export function buildResponsesRequest(params: { tools: unknown[] toolChoice: unknown reasoningEffort?: ResponsesReasoningEffort - /** Override for tests; production uses the current CCB session id. */ - promptCacheKey?: string + /** Session-scoped key supplied only by the ChatGPT OAuth route. */ + promptCacheKey: string }): ResponsesRequest { const { input, instructions } = convertMessagesToResponsesInput( params.messages, @@ -197,9 +190,9 @@ export function buildResponsesRequest(params: { ? { reasoning: { effort: params.reasoningEffort } } : {}), parallel_tool_calls: true, - // Same process/session → same key so OpenAI can sticky-route to a cache node. + // Same OAuth session → same key so OpenAI can sticky-route to a cache node. // Must not hash the full message list (would change every turn). - prompt_cache_key: params.promptCacheKey ?? getOpenAIPromptCacheKey(), + prompt_cache_key: params.promptCacheKey, } } @@ -266,19 +259,12 @@ export function extractUsage( ? inputDetails.cache_write_tokens : 0 - // Cap segments so they stay non-negative and do not exceed total input. - // Prefer preserving cache_read for hit-rate accuracy, then cache_creation. - const cacheRead = Math.min(Math.max(0, cachedRaw), Math.max(0, totalInput)) - const remainingAfterRead = Math.max(0, totalInput - cacheRead) - const cacheCreation = Math.min(Math.max(0, writeRaw), remainingAfterRead) - const inputTokens = Math.max(0, remainingAfterRead - cacheCreation) - - return { - input_tokens: inputTokens, - output_tokens: outputTokens, - cache_creation_input_tokens: cacheCreation, - cache_read_input_tokens: cacheRead, - } + return normalizeOpenAIUsage({ + totalInputTokens: totalInput, + outputTokens, + cacheReadTokens: cachedRaw, + cacheWriteTokens: writeRaw, + }) } function mapStopReason(response: Record | undefined): string { diff --git a/src/utils/__tests__/sideQuery.chatgptAuth.test.ts b/src/utils/__tests__/sideQuery.chatgptAuth.test.ts index ee6da27760..c39c9bb4f7 100644 --- a/src/utils/__tests__/sideQuery.chatgptAuth.test.ts +++ b/src/utils/__tests__/sideQuery.chatgptAuth.test.ts @@ -29,6 +29,7 @@ mock.module('src/services/analytics/index.js', () => ({ let getOpenAIClientCallCount = 0 let chatCompletionsCreateCount = 0 let lastChatCompletionsArgs: Record | null = null +let chatCompletionsUsage: Record = {} mock.module('src/services/api/openai/client.js', () => ({ getOpenAIClient: () => { @@ -59,7 +60,7 @@ mock.module('src/services/api/openai/client.js', () => ({ }, }, ], - usage: { prompt_tokens: 3, completion_tokens: 2 }, + usage: chatCompletionsUsage, } }, }, @@ -103,6 +104,7 @@ const ENV_KEYS = [ 'CLAUDE_CODE_USE_FOUNDRY', 'OPENAI_AUTH_MODE', 'OPENAI_API_KEY', + 'OPENAI_BASE_URL', ] as const const savedEnv: Record = {} @@ -158,6 +160,7 @@ beforeEach(() => { getOpenAIClientCallCount = 0 chatCompletionsCreateCount = 0 lastChatCompletionsArgs = null + chatCompletionsUsage = { prompt_tokens: 3, completion_tokens: 2 } capturedFetch = null enableOpenAIProvider() originalFetch = globalThis.fetch @@ -266,9 +269,18 @@ describe('sideQuery OpenAI ChatGPT OAuth path', () => { expect(result.usage.output_tokens).toBe(7) }) - test('API key mode still uses Chat Completions client', async () => { + test('official API key mode uses a session cache key and normalized usage', async () => { delete process.env.OPENAI_AUTH_MODE + delete process.env.OPENAI_BASE_URL process.env.OPENAI_API_KEY = 'sk-test-not-real' + chatCompletionsUsage = { + prompt_tokens: 1000, + completion_tokens: 50, + prompt_tokens_details: { + cached_tokens: 600, + cache_write_tokens: 250, + }, + } const { sideQuery } = await import('../sideQuery.js') const result = await sideQuery({ @@ -283,12 +295,43 @@ describe('sideQuery OpenAI ChatGPT OAuth path', () => { expect(chatCompletionsCreateCount).toBe(1) expect(capturedFetch).toBeNull() expect(lastChatCompletionsArgs?.model).toBe('gpt-4o') + expect(lastChatCompletionsArgs?.prompt_cache_key).toMatch(/^ccb:/) const toolUse = result.content.find(b => b.type === 'tool_use') as | { type: 'tool_use'; name: string; input: unknown } | undefined expect(toolUse?.name).toBe('classify_result') expect(toolUse?.input).toEqual({ shouldBlock: false }) + expect(result.usage.input_tokens).toBe(150) + expect(result.usage.cache_read_input_tokens).toBe(600) + expect(result.usage.cache_creation_input_tokens).toBe(250) + }) + + test('compatible API key mode omits official cache fields', async () => { + delete process.env.OPENAI_AUTH_MODE + process.env.OPENAI_BASE_URL = 'https://api.deepseek.com/v1' + process.env.OPENAI_API_KEY = 'sk-test-not-real' + chatCompletionsUsage = { + prompt_tokens: 1000, + completion_tokens: 50, + prompt_tokens_details: { + cached_tokens: 600, + cache_write_tokens: 250, + }, + } + const { sideQuery } = await import('../sideQuery.js') + + const result = await sideQuery({ + querySource: 'auto_mode', + model: 'deepseek-chat', + messages: [{ role: 'user', content: 'hi' }], + }) + + expect(lastChatCompletionsArgs).not.toBeNull() + expect('prompt_cache_key' in lastChatCompletionsArgs!).toBe(false) + expect(result.usage.input_tokens).toBe(400) + expect(result.usage.cache_read_input_tokens).toBe(600) + expect(result.usage.cache_creation_input_tokens).toBe(0) }) test('ChatGPT OAuth request failure propagates for fail-closed classifiers', async () => { diff --git a/src/utils/model/__tests__/chatgptModelRouting.test.ts b/src/utils/model/__tests__/chatgptModelRouting.test.ts new file mode 100644 index 0000000000..04437bde3b --- /dev/null +++ b/src/utils/model/__tests__/chatgptModelRouting.test.ts @@ -0,0 +1,76 @@ +import { describe, expect, test } from 'bun:test' +import { + CHATGPT_CODEX_MODELS_BY_TIER, + resolveChatGPTCodexModelForTier, +} from '../chatgptModels.js' + +describe('resolveChatGPTCodexModelForTier', () => { + test('maps CCB capability tiers to the matching GPT-5.6 models', () => { + expect( + resolveChatGPTCodexModelForTier({ + tier: 'opus', + isChatGPTAuth: true, + }), + ).toBe('gpt-5.6-sol') + expect( + resolveChatGPTCodexModelForTier({ + tier: 'sonnet', + isChatGPTAuth: true, + }), + ).toBe('gpt-5.6-terra') + expect( + resolveChatGPTCodexModelForTier({ + tier: 'haiku', + isChatGPTAuth: true, + }), + ).toBe('gpt-5.6-luna') + }) + + test('keeps the tier map as the single source of default assignments', () => { + expect(CHATGPT_CODEX_MODELS_BY_TIER).toEqual({ + opus: 'gpt-5.6-sol', + sonnet: 'gpt-5.6-terra', + haiku: 'gpt-5.6-luna', + }) + }) + + test('prefers family overrides over OAuth defaults', () => { + expect( + resolveChatGPTCodexModelForTier({ + tier: 'haiku', + isChatGPTAuth: true, + tierOverride: 'custom-haiku', + }), + ).toBe('custom-haiku') + }) + + test('prefers a task-specific override over the family override', () => { + expect( + resolveChatGPTCodexModelForTier({ + tier: 'haiku', + isChatGPTAuth: true, + tierOverride: 'custom-haiku', + taskOverride: 'custom-small-fast', + }), + ).toBe('custom-small-fast') + }) + + test('does not apply GPT defaults outside ChatGPT OAuth mode', () => { + expect( + resolveChatGPTCodexModelForTier({ + tier: 'opus', + isChatGPTAuth: false, + }), + ).toBeUndefined() + }) + + test('preserves explicit compatible-provider tier configuration', () => { + expect( + resolveChatGPTCodexModelForTier({ + tier: 'sonnet', + isChatGPTAuth: false, + tierOverride: 'compatible-provider-model', + }), + ).toBe('compatible-provider-model') + }) +}) diff --git a/src/utils/model/chatgptModels.ts b/src/utils/model/chatgptModels.ts index 89de8ff1cf..d22ef850c7 100644 --- a/src/utils/model/chatgptModels.ts +++ b/src/utils/model/chatgptModels.ts @@ -6,9 +6,34 @@ export type ChatGPTCodexModelOption = { /** Default ChatGPT/Codex model (newest frontier). */ export const CHATGPT_CODEX_DEFAULT_MODEL = 'gpt-5.6-sol' +export const CHATGPT_CODEX_BALANCED_MODEL = 'gpt-5.6-terra' /** Fast/small default for lighter tasks. */ export const CHATGPT_CODEX_FAST_MODEL = 'gpt-5.6-luna' +export const CHATGPT_CODEX_MODELS_BY_TIER = { + opus: CHATGPT_CODEX_DEFAULT_MODEL, + sonnet: CHATGPT_CODEX_BALANCED_MODEL, + haiku: CHATGPT_CODEX_FAST_MODEL, +} as const + +export type ChatGPTCodexModelTier = keyof typeof CHATGPT_CODEX_MODELS_BY_TIER + +/** Resolve one CCB capability tier without coupling the policy to settings. */ +export function resolveChatGPTCodexModelForTier(params: { + tier: ChatGPTCodexModelTier + isChatGPTAuth: boolean + tierOverride?: string + taskOverride?: string +}): string | undefined { + return ( + params.taskOverride ?? + params.tierOverride ?? + (params.isChatGPTAuth + ? CHATGPT_CODEX_MODELS_BY_TIER[params.tier] + : undefined) + ) +} + /** * ChatGPT OAuth / Codex subscription practical context window. * Codex with ChatGPT login is product-limited to ~272k (not the full API 1.05M). diff --git a/src/utils/model/model.ts b/src/utils/model/model.ts index 19c0e15619..c8fd1866c8 100644 --- a/src/utils/model/model.ts +++ b/src/utils/model/model.ts @@ -30,19 +30,45 @@ import { isModelAllowed } from './modelAllowlist.js' import { type ModelAlias, isModelAlias } from './aliases.js' import { capitalize } from '../stringUtils.js' import { - CHATGPT_CODEX_DEFAULT_MODEL, - CHATGPT_CODEX_FAST_MODEL, + type ChatGPTCodexModelTier, isChatGPTAuthMode, + resolveChatGPTCodexModelForTier, } from './chatgptModels.js' export type ModelShortName = string export type ModelName = string export type ModelSetting = ModelName | ModelAlias | null +const OPENAI_DEFAULT_MODEL_ENV_BY_TIER: Record = + { + opus: 'OPENAI_DEFAULT_OPUS_MODEL', + sonnet: 'OPENAI_DEFAULT_SONNET_MODEL', + haiku: 'OPENAI_DEFAULT_HAIKU_MODEL', + } + +function getOpenAIModelForTier( + provider: ReturnType, + tier: ChatGPTCodexModelTier, +): ModelName | undefined { + if (provider !== 'openai') return undefined + + return resolveChatGPTCodexModelForTier({ + tier, + isChatGPTAuth: isChatGPTAuthMode(), + tierOverride: process.env[OPENAI_DEFAULT_MODEL_ENV_BY_TIER[tier]], + }) +} + export function getSmallFastModel(): ModelName { const provider = getAPIProvider() if (provider === 'openai' && isChatGPTAuthMode()) { - return process.env.OPENAI_SMALL_FAST_MODEL ?? CHATGPT_CODEX_FAST_MODEL + const chatGPTModel = resolveChatGPTCodexModelForTier({ + tier: 'haiku', + isChatGPTAuth: true, + tierOverride: process.env.OPENAI_DEFAULT_HAIKU_MODEL, + taskOverride: process.env.OPENAI_SMALL_FAST_MODEL, + }) + if (chatGPTModel) return chatGPTModel } // Provider-specific small fast model if (provider === 'openai' && process.env.OPENAI_SMALL_FAST_MODEL) { @@ -136,13 +162,8 @@ function getProviderPrimaryModel(): ModelName | undefined { // @[MODEL LAUNCH]: Update the default Opus model (3P providers may lag so keep defaults unchanged). export function getDefaultOpusModel(): ModelName { const provider = getAPIProvider() - if (provider === 'openai' && isChatGPTAuthMode()) { - return CHATGPT_CODEX_DEFAULT_MODEL - } - // For OpenAI provider, check OPENAI_DEFAULT_OPUS_MODEL first - if (provider === 'openai' && process.env.OPENAI_DEFAULT_OPUS_MODEL) { - return process.env.OPENAI_DEFAULT_OPUS_MODEL - } + const openAIModel = getOpenAIModelForTier(provider, 'opus') + if (openAIModel) return openAIModel // For Gemini provider, check GEMINI_DEFAULT_OPUS_MODEL if (provider === 'gemini' && process.env.GEMINI_DEFAULT_OPUS_MODEL) { return process.env.GEMINI_DEFAULT_OPUS_MODEL @@ -166,13 +187,8 @@ export function getDefaultOpusModel(): ModelName { // @[MODEL LAUNCH]: Update the default Sonnet model (3P providers may lag so keep defaults unchanged). export function getDefaultSonnetModel(): ModelName { const provider = getAPIProvider() - if (provider === 'openai' && isChatGPTAuthMode()) { - return CHATGPT_CODEX_DEFAULT_MODEL - } - // For OpenAI provider, check OPENAI_DEFAULT_SONNET_MODEL first - if (provider === 'openai' && process.env.OPENAI_DEFAULT_SONNET_MODEL) { - return process.env.OPENAI_DEFAULT_SONNET_MODEL - } + const openAIModel = getOpenAIModelForTier(provider, 'sonnet') + if (openAIModel) return openAIModel // For Gemini provider, check GEMINI_DEFAULT_SONNET_MODEL if (provider === 'gemini' && process.env.GEMINI_DEFAULT_SONNET_MODEL) { return process.env.GEMINI_DEFAULT_SONNET_MODEL @@ -195,13 +211,8 @@ export function getDefaultSonnetModel(): ModelName { // @[MODEL LAUNCH]: Update the default Haiku model (3P providers may lag so keep defaults unchanged). export function getDefaultHaikuModel(): ModelName { const provider = getAPIProvider() - if (provider === 'openai' && isChatGPTAuthMode()) { - return CHATGPT_CODEX_FAST_MODEL - } - // For OpenAI provider, check OPENAI_DEFAULT_HAIKU_MODEL first - if (provider === 'openai' && process.env.OPENAI_DEFAULT_HAIKU_MODEL) { - return process.env.OPENAI_DEFAULT_HAIKU_MODEL - } + const openAIModel = getOpenAIModelForTier(provider, 'haiku') + if (openAIModel) return openAIModel // For Gemini provider, check GEMINI_DEFAULT_HAIKU_MODEL if (provider === 'gemini' && process.env.GEMINI_DEFAULT_HAIKU_MODEL) { return process.env.GEMINI_DEFAULT_HAIKU_MODEL diff --git a/src/utils/sideQuery.ts b/src/utils/sideQuery.ts index 33e012eb58..7b85c2c4b4 100644 --- a/src/utils/sideQuery.ts +++ b/src/utils/sideQuery.ts @@ -40,6 +40,10 @@ import { buildResponsesRequest, createChatGPTResponsesStream, } from '../services/api/openai/responsesAdapter.js' +import { + formatOpenAIPromptCacheKey, + getOfficialOpenAIPromptCacheKey, +} from '../services/api/openai/openaiShared.js' import { anthropicMessagesToOpenAI, resolveOpenAIModel, @@ -49,6 +53,7 @@ import { resolveGeminiModel, anthropicToolsToGemini, anthropicToolChoiceToGemini, + normalizeOpenAIUsage, } from '@ant/model-provider' import type { SystemPrompt } from './systemPromptType.js' import type { BetaRawMessageStreamEvent } from '@anthropic-ai/sdk/resources/beta/messages/messages.mjs' @@ -574,6 +579,7 @@ async function sideQueryViaChatGPTResponses( messages: openaiMessages, tools: openaiTools ?? [], toolChoice: openaiToolChoice, + promptCacheKey: formatOpenAIPromptCacheKey(getSessionId()), }) const rawStream = await createChatGPTResponsesStream({ @@ -693,6 +699,14 @@ async function sideQueryViaOpenAICompatible( messages: openaiMessages, max_tokens, } + const promptCacheKey = + provider === 'openai' + ? getOfficialOpenAIPromptCacheKey( + process.env.OPENAI_BASE_URL, + getSessionId(), + ) + : undefined + if (promptCacheKey) requestParams.prompt_cache_key = promptCacheKey if (temperature !== undefined) requestParams.temperature = temperature if (openaiTools && openaiTools.length > 0) { requestParams.tools = openaiTools @@ -733,6 +747,26 @@ async function sideQueryViaOpenAICompatible( } } + const responseUsage = response.usage + const usageRecord = responseUsage as unknown as + | Record + | undefined + const detailsValue = usageRecord?.prompt_tokens_details + const details = + detailsValue && typeof detailsValue === 'object' + ? (detailsValue as Record) + : undefined + const usage = normalizeOpenAIUsage({ + totalInputTokens: responseUsage?.prompt_tokens ?? 0, + outputTokens: responseUsage?.completion_tokens ?? 0, + cacheReadTokens: + typeof details?.cached_tokens === 'number' ? details.cached_tokens : 0, + cacheWriteTokens: + promptCacheKey && typeof details?.cache_write_tokens === 'number' + ? details.cache_write_tokens + : 0, + }) + const now = Date.now() const requestId = response.id const lastCompletion = getLastApiCompletionTimestamp() @@ -743,10 +777,10 @@ async function sideQueryViaOpenAICompatible( opts.querySource as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, model: openaiModel as AnalyticsMetadata_I_VERIFIED_THIS_IS_NOT_CODE_OR_FILEPATHS, - inputTokens: response.usage?.prompt_tokens ?? 0, - outputTokens: response.usage?.completion_tokens ?? 0, - cachedInputTokens: 0, - uncachedInputTokens: response.usage?.prompt_tokens ?? 0, + inputTokens: usage.input_tokens, + outputTokens: usage.output_tokens, + cachedInputTokens: usage.cache_read_input_tokens, + uncachedInputTokens: usage.cache_creation_input_tokens, durationMsIncludingRetries: now - start, timeSinceLastApiCallMs: lastCompletion !== null ? now - lastCompletion : undefined, @@ -768,10 +802,7 @@ async function sideQueryViaOpenAICompatible( model: openaiModel, stop_reason: stopReason as BetaMessage['stop_reason'], stop_sequence: null, - usage: { - input_tokens: response.usage?.prompt_tokens ?? 0, - output_tokens: response.usage?.completion_tokens ?? 0, - }, + usage, } as BetaMessage }