From 988d130b705e81c4720beb3b454114d46a905b65 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 16 Jul 2026 04:18:38 +0200 Subject: [PATCH 1/8] fix(llmobs): format AI SDK rich tool results AI SDK tool outputs can contain multipart content and error variants. Preserve text, replace file and image payloads with safe summaries, and fall back without throwing on malformed values. Fixes: https://github.com/DataDog/dd-trace-js/issues/9395 --- .../dd-trace/src/llmobs/plugins/ai/util.js | 93 +++++++--- .../test/llmobs/plugins/ai/util.spec.js | 173 ++++++++++++++++++ 2 files changed, 244 insertions(+), 22 deletions(-) create mode 100644 packages/dd-trace/test/llmobs/plugins/ai/util.spec.js diff --git a/packages/dd-trace/src/llmobs/plugins/ai/util.js b/packages/dd-trace/src/llmobs/plugins/ai/util.js index 0909431e9a..4bba6925e3 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/util.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/util.js @@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([ const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.' const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.' const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.' +const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' +const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' /** * @typedef {import('../../../opentracing/span')} Span @@ -27,7 +29,8 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.' /** * @typedef {{ * type: string, - * value?: unknown + * value?: unknown, + * reason?: unknown * }} ToolCallOutput * * @typedef {{ output?: ToolCallOutput, result?: unknown } & Record} ToolCallResultContent @@ -304,32 +307,78 @@ function getToolNameFromTags (tags) { } /** - * Get the content of a tool call result. - * Version 5 of the ai sdk sets this tag as `content.output`, with a ` - * @param {{ output?: { type: string, value: unknown }, result?: unknown }} content + * @param {unknown} value * @returns {string} */ -function getToolCallResultContent (content) { - const { output, result } = content - if (output) { - if (output.type === 'text') { - return output.value - } else if (output.type === 'json') { - return JSON.stringify(output.value) - } - return '[Unparsable Tool Result]' - } else if (result) { - if (typeof result === 'string') { - return result +function stringifyToolCallResult (value) { + return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT +} + +/** + * @param {unknown} value + * @returns {string} + */ +function formatToolCallContent (value) { + if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT + + let result = '' + for (const part of value) { + if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT + + const { type } = part + if (type === 'text') { + if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT + result += part.text + } else if (type === 'media') { + const { mediaType } = part + if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT + result += mediaType.startsWith('image/') ? '[Image]' : '[File]' + } else if (type === 'file-data' || type === 'file-url' || type === 'file-id') { + result += '[File]' + } else if (type === 'image-data' || type === 'image-url' || type === 'image-file-id') { + result += '[Image]' + } else if (type === 'custom') { + result += '[Custom Content]' + } else { + return UNPARSABLE_TOOL_RESULT } + } + + return result +} - try { - return JSON.stringify(result) - } catch { - return '[Unparsable Tool Result]' +/** + * @param {ToolCallResultContent | null | undefined} content + * @returns {string} + */ +function getToolCallResultContent (content) { + try { + if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT + + const { output, result } = content + if (output !== undefined) { + if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT + + const { type, value } = output + if (type === 'text' || type === 'error-text') { + return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT + } else if (type === 'json' || type === 'error-json') { + return stringifyToolCallResult(value) + } else if (type === 'content') { + return formatToolCallContent(value) + } else if (type === 'execution-denied') { + const { reason } = output + if (reason === undefined) return '[Tool Execution Denied]' + return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT + } + return UNPARSABLE_TOOL_RESULT + } else if (result !== undefined) { + return typeof result === 'string' ? result : stringifyToolCallResult(result) } - } else { - return '[Unsupported Tool Result]' + + return UNSUPPORTED_TOOL_RESULT + } catch { + return UNPARSABLE_TOOL_RESULT } } diff --git a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js new file mode 100644 index 0000000000..8db2970388 --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js @@ -0,0 +1,173 @@ +'use strict' + +const assert = require('node:assert/strict') + +const { getToolCallResultContent } = require('../../../../src/llmobs/plugins/ai/util') + +const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' +const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' + +describe('AI SDK LLMObs utilities', () => { + describe('getToolCallResultContent', () => { + it('formats text output variants', () => { + assert.strictEqual(getToolCallResultContent({ output: { type: 'text', value: 'result' } }), 'result') + assert.strictEqual(getToolCallResultContent({ output: { type: 'error-text', value: 'failure' } }), 'failure') + }) + + it('formats JSON output variants', () => { + const value = { answer: 42 } + + assert.strictEqual( + getToolCallResultContent({ output: { type: 'json', value } }), + JSON.stringify(value) + ) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'error-json', value } }), + JSON.stringify(value) + ) + }) + + it('formats all legacy result values', () => { + assert.strictEqual(getToolCallResultContent({ result: '' }), '') + assert.strictEqual(getToolCallResultContent({ result: 'result' }), 'result') + assert.strictEqual(getToolCallResultContent({ result: 0 }), '0') + assert.strictEqual(getToolCallResultContent({ result: false }), 'false') + assert.strictEqual(getToolCallResultContent({ result: null }), 'null') + assert.strictEqual(getToolCallResultContent({ result: { answer: 42 } }), '{"answer":42}') + }) + + it('distinguishes absent legacy results', () => { + assert.strictEqual(getToolCallResultContent({}), UNSUPPORTED_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({ result: undefined }), UNSUPPORTED_TOOL_RESULT) + }) + + it('concatenates text and summarizes rich content without serializing payloads', () => { + const content = [ + { type: 'text', text: 'before' }, + { type: 'media', mediaType: 'image/png', data: 'legacy-image-data' }, + { type: 'media', mediaType: 'application/pdf', data: 'legacy-file-data' }, + { type: 'file-data', mediaType: 'application/pdf', data: 'file-data' }, + { type: 'file-url', url: 'https://example.com/private-file' }, + { type: 'file-id', fileId: 'private-file-id' }, + { type: 'image-data', mediaType: 'image/png', data: 'image-data' }, + { type: 'image-url', url: 'https://example.com/private-image' }, + { type: 'image-file-id', fileId: 'private-image-id' }, + { type: 'custom', providerOptions: { secret: 'provider-data' } }, + { type: 'text', text: 'after' }, + ] + + assert.strictEqual( + getToolCallResultContent({ output: { type: 'content', value: content } }), + 'before[Image][File][File][File][File][Image][Image][Image][Custom Content]after' + ) + }) + + it('formats empty multipart content', () => { + assert.strictEqual(getToolCallResultContent({ output: { type: 'content', value: [] } }), '') + }) + + it('formats denied executions', () => { + assert.strictEqual( + getToolCallResultContent({ output: { type: 'execution-denied', reason: 'Approval rejected' } }), + 'Approval rejected' + ) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'execution-denied' } }), + '[Tool Execution Denied]' + ) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'execution-denied', reason: '' } }), + '' + ) + }) + + it('rejects malformed output variants', () => { + const malformedOutputs = [ + null, + 'text', + { type: 'text' }, + { type: 'text', value: 42 }, + { type: 'error-text', value: {} }, + { type: 'json', value: undefined }, + { type: 'json', value: Symbol('result') }, + { type: 'error-json', value: () => {} }, + { type: 'content', value: {} }, + { type: 'execution-denied', reason: {} }, + { type: 'unknown', value: 'result' }, + ] + + for (const output of malformedOutputs) { + assert.strictEqual(getToolCallResultContent({ output }), UNPARSABLE_TOOL_RESULT) + } + }) + + it('rejects malformed multipart content', () => { + const sparseContent = new Array(1) + const malformedValues = [ + [null], + ['text'], + [{ type: 'text' }], + [{ type: 'text', text: 42 }], + [{ type: 'media' }], + [{ type: 'unknown' }], + [{ type: Symbol('text'), text: 'result' }], + [{ type: 'text', text: 'before' }, { type: 'unknown' }], + sparseContent, + ] + + for (const value of malformedValues) { + assert.strictEqual( + getToolCallResultContent({ output: { type: 'content', value } }), + UNPARSABLE_TOOL_RESULT + ) + } + }) + + it('does not throw for circular values', () => { + const circular = {} + circular.self = circular + + assert.strictEqual( + getToolCallResultContent({ output: { type: 'json', value: circular } }), + UNPARSABLE_TOOL_RESULT + ) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'error-json', value: circular } }), + UNPARSABLE_TOOL_RESULT + ) + assert.strictEqual(getToolCallResultContent({ result: circular }), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({ result: 1n }), UNPARSABLE_TOOL_RESULT) + }) + + it('does not throw for throwing external objects', () => { + const throwingContent = new Proxy({}, { + get () { + throw new Error('unexpected content access') + }, + }) + const throwingOutput = new Proxy({}, { + get () { + throw new Error('unexpected output access') + }, + }) + const throwingParts = new Proxy([], { + get () { + throw new Error('unexpected content part access') + }, + }) + + assert.strictEqual(getToolCallResultContent(throwingContent), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({ output: throwingOutput }), UNPARSABLE_TOOL_RESULT) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'content', value: throwingParts } }), + UNPARSABLE_TOOL_RESULT + ) + }) + + it('rejects malformed content containers', () => { + assert.strictEqual(getToolCallResultContent(), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent(null), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent('result'), UNPARSABLE_TOOL_RESULT) + }) + }) +}) From 34f4ea3dec05dce39396fd613793f248dcd42ad2 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Mon, 20 Jul 2026 20:12:13 +0200 Subject: [PATCH 2/8] fix(llmobs): support AI SDK file result parts AI SDK 7 emits generic file and provider-reference parts that were reported as unparsable. Model the supported output shapes locally and exercise content outputs through both telemetry paths so SDK shape changes fail on the user path. --- .../dd-trace/src/llmobs/plugins/ai/util.js | 48 +++++++++++++++---- .../test/llmobs/plugins/ai/index.spec.js | 4 ++ .../test/llmobs/plugins/ai/index.v7.spec.js | 4 ++ .../test/llmobs/plugins/ai/util.spec.js | 10 +++- 4 files changed, 57 insertions(+), 9 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/util.js b/packages/dd-trace/src/llmobs/plugins/ai/util.js index 4bba6925e3..a70959512b 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/util.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/util.js @@ -28,9 +28,31 @@ const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' /** * @typedef {{ - * type: string, - * value?: unknown, - * reason?: unknown + * type: 'text', + * text: string + * } | { + * type: 'media' | 'file', + * mediaType: string + * } | { + * type: 'file-data' | 'file-url' | 'file-id' | 'file-reference' + * } | { + * type: 'image-data' | 'image-url' | 'image-file-id' | 'image-file-reference' + * } | { + * type: 'custom' + * }} ToolCallContentPart + * + * @typedef {{ + * type: 'text' | 'error-text', + * value: string + * } | { + * type: 'json' | 'error-json', + * value: unknown + * } | { + * type: 'content', + * value: ToolCallContentPart[] + * } | { + * type: 'execution-denied', + * reason?: string * }} ToolCallOutput * * @typedef {{ output?: ToolCallOutput, result?: unknown } & Record} ToolCallResultContent @@ -315,7 +337,7 @@ function stringifyToolCallResult (value) { } /** - * @param {unknown} value + * @param {ToolCallContentPart[]} value * @returns {string} */ function formatToolCallContent (value) { @@ -329,13 +351,23 @@ function formatToolCallContent (value) { if (type === 'text') { if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT result += part.text - } else if (type === 'media') { + } else if (type === 'media' || type === 'file') { const { mediaType } = part if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT - result += mediaType.startsWith('image/') ? '[Image]' : '[File]' - } else if (type === 'file-data' || type === 'file-url' || type === 'file-id') { + result += mediaType === 'image' || mediaType.startsWith('image/') ? '[Image]' : '[File]' + } else if ( + type === 'file-data' || + type === 'file-url' || + type === 'file-id' || + type === 'file-reference' + ) { result += '[File]' - } else if (type === 'image-data' || type === 'image-url' || type === 'image-file-id') { + } else if ( + type === 'image-data' || + type === 'image-url' || + type === 'image-file-id' || + type === 'image-file-reference' + ) { result += '[Image]' } else if (type === 'custom') { result += '[Custom Content]' diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 2810f30aa8..9ff14cacc1 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -481,6 +481,10 @@ describe('Plugin', () => { location, temperature: 72, }), + toModelOutput: () => ({ + type: 'content', + value: [{ type: 'text', text: '{"location":"Tokyo","temperature":72}' }], + }), }), } diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index 7ff867c297..fdfd8751a1 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -261,6 +261,10 @@ describe('Plugin', () => { location, temperature: 72, }), + toModelOutput: () => ({ + type: 'content', + value: [{ type: 'text', text: '{"location":"Tokyo","temperature":72}' }], + }), }), }, stopWhen: ai.stepCountIs(5), diff --git a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js index 8db2970388..40a1ae9305 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js @@ -46,19 +46,25 @@ describe('AI SDK LLMObs utilities', () => { { type: 'text', text: 'before' }, { type: 'media', mediaType: 'image/png', data: 'legacy-image-data' }, { type: 'media', mediaType: 'application/pdf', data: 'legacy-file-data' }, + { type: 'file', mediaType: 'image', data: { type: 'data', data: 'image-data' } }, + { type: 'file', mediaType: 'image/png', data: { type: 'url', url: 'https://example.com/image' } }, + { type: 'file', mediaType: 'application/pdf', data: { type: 'text', text: 'file-data' } }, { type: 'file-data', mediaType: 'application/pdf', data: 'file-data' }, { type: 'file-url', url: 'https://example.com/private-file' }, { type: 'file-id', fileId: 'private-file-id' }, + { type: 'file-reference', providerReference: { openai: 'private-file-id' } }, { type: 'image-data', mediaType: 'image/png', data: 'image-data' }, { type: 'image-url', url: 'https://example.com/private-image' }, { type: 'image-file-id', fileId: 'private-image-id' }, + { type: 'image-file-reference', providerReference: { openai: 'private-image-id' } }, { type: 'custom', providerOptions: { secret: 'provider-data' } }, { type: 'text', text: 'after' }, ] assert.strictEqual( getToolCallResultContent({ output: { type: 'content', value: content } }), - 'before[Image][File][File][File][File][Image][Image][Image][Custom Content]after' + 'before[Image][File][Image][Image][File][File][File][File][File]' + + '[Image][Image][Image][Image][Custom Content]after' ) }) @@ -109,6 +115,8 @@ describe('AI SDK LLMObs utilities', () => { [{ type: 'text' }], [{ type: 'text', text: 42 }], [{ type: 'media' }], + [{ type: 'file' }], + [{ type: 'file', mediaType: 42 }], [{ type: 'unknown' }], [{ type: Symbol('text'), text: 'result' }], [{ type: 'text', text: 'before' }, { type: 'unknown' }], From d91d6d858c05f4681fe91d5d0bc792239c4024fb Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 21 Jul 2026 03:16:53 +0200 Subject: [PATCH 3/8] test(llmobs): preserve recorded AI tool requests Adding toModelOutput changed the second provider request without a matching cassette, so every AI SDK v5-v7 matrix forwarded to OpenAI with the fake key. Keep the recorded workflow payload stable while formatter coverage pins the rich result shapes and schema precedence. --- .../test/llmobs/plugins/ai/index.spec.js | 4 ---- .../test/llmobs/plugins/ai/index.v7.spec.js | 4 ---- .../test/llmobs/plugins/ai/util.spec.js | 24 +++++++++++++++++++ 3 files changed, 24 insertions(+), 8 deletions(-) diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 9ff14cacc1..2810f30aa8 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -481,10 +481,6 @@ describe('Plugin', () => { location, temperature: 72, }), - toModelOutput: () => ({ - type: 'content', - value: [{ type: 'text', text: '{"location":"Tokyo","temperature":72}' }], - }), }), } diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index fdfd8751a1..7ff867c297 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -261,10 +261,6 @@ describe('Plugin', () => { location, temperature: 72, }), - toModelOutput: () => ({ - type: 'content', - value: [{ type: 'text', text: '{"location":"Tokyo","temperature":72}' }], - }), }), }, stopWhen: ai.stepCountIs(5), diff --git a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js index 40a1ae9305..1cac908c0f 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js @@ -41,6 +41,30 @@ describe('AI SDK LLMObs utilities', () => { assert.strictEqual(getToolCallResultContent({ result: undefined }), UNSUPPORTED_TOOL_RESULT) }) + it('prefers defined output over legacy results', () => { + assert.strictEqual( + getToolCallResultContent({ + output: { type: 'text', value: '' }, + result: 'legacy result', + }), + '' + ) + assert.strictEqual( + getToolCallResultContent({ + output: null, + result: 'legacy result', + }), + UNPARSABLE_TOOL_RESULT + ) + assert.strictEqual( + getToolCallResultContent({ + output: undefined, + result: 'legacy result', + }), + 'legacy result' + ) + }) + it('concatenates text and summarizes rich content without serializing payloads', () => { const content = [ { type: 'text', text: 'before' }, From a8c371adb05a573e8f9ef0e2f6eeb8bcb7111e3c Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Tue, 21 Jul 2026 13:00:52 +0200 Subject: [PATCH 4/8] test(llmobs): exercise tool results through AI SDK Direct utility tests pinned hand-built shapes and missed the SDK transformations that feed LLMObs. Drive supported results through generateText across all model protocol versions, while keeping direct safety assertions only for values the SDK rejects before model invocation. --- .../test/llmobs/plugins/ai/index.spec.js | 280 ++++++++++++++++++ .../test/llmobs/plugins/ai/index.v7.spec.js | 115 +++++++ .../llmobs/plugins/ai/tool-result-model.js | 141 +++++++++ .../test/llmobs/plugins/ai/util.spec.js | 205 ------------- 4 files changed, 536 insertions(+), 205 deletions(-) create mode 100644 packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js delete mode 100644 packages/dd-trace/test/llmobs/plugins/ai/util.spec.js diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 2810f30aa8..52209bfefb 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -6,6 +6,7 @@ const semifies = require('semifies') const { useEnv } = require('../../../../../../integration-tests/helpers') const { withVersions } = require('../../../setup/mocha') const iastFilter = require('../../../../src/appsec/iast/taint-tracking/filter') +const { getToolCallResultContent } = require('../../../../src/llmobs/plugins/ai/util') const { NODE_MAJOR } = require('../../../../../../version') @@ -18,6 +19,14 @@ const { MOCK_NUMBER, MOCK_OBJECT, } = require('../../util') +const { + createToolResultModelV1, + createToolResultModelV2, + createToolResultModelV3, +} = require('./tool-result-model') + +const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' +const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' // ai<4.0.2 is not supported in CommonJS with Node.js < 22 const range = NODE_MAJOR < 22 ? '>=4.0.2 <7.0.0' : '>=4.0.0 <7.0.0' @@ -79,6 +88,107 @@ const MOCK_TELEMETRY_METADATA = { conversationId: 'convAbc123', } +const TOOL_RESULT_SCENARIOS = [ + { + name: 'multipart content', + execute: () => 'result', + createModelOutput: () => ({ + type: 'content', + value: [ + { type: 'text', text: 'before' }, + { type: 'media', mediaType: 'image/png', data: 'private-image-data' }, + { type: 'media', mediaType: 'application/pdf', data: 'private-file-data' }, + { type: 'file-data', mediaType: 'application/pdf', data: 'private-file-data' }, + { type: 'file-id', fileId: 'private-file-id' }, + { type: 'image-data', mediaType: 'image/png', data: 'private-image-data' }, + { type: 'image-file-id', fileId: 'private-image-id' }, + { type: 'custom', providerOptions: { secret: 'provider-data' } }, + { type: 'text', text: 'after' }, + ], + }), + expectedContent: 'before[Image][File][File][File][Image][Image][Custom Content]after', + }, + { + name: 'text', + execute: () => 'result', + expectedContent: 'result', + }, + { + name: 'JSON', + execute: () => ({ answer: 42 }), + expectedContent: '{"answer":42}', + }, + { + name: 'error text', + execute: () => { + throw new Error('tool failure') + }, + expectedContent: 'tool failure', + }, + { + name: 'error JSON', + execute: () => 'result', + createModelOutput: () => ({ + type: 'error-json', + value: { message: 'tool failure' }, + }), + expectedContent: '{"message":"tool failure"}', + }, + { + name: 'empty multipart content', + execute: () => 'result', + createModelOutput: () => ({ type: 'content', value: [] }), + expectedContent: '', + }, + { + name: 'denied execution', + execute: () => 'result', + createModelOutput: () => ({ + type: 'execution-denied', + reason: 'Approval rejected', + }), + expectedContent: 'Approval rejected', + }, + { + name: 'denied execution without a reason', + execute: () => 'result', + createModelOutput: () => ({ type: 'execution-denied' }), + expectedContent: '[Tool Execution Denied]', + }, + { + name: 'malformed multipart content', + execute: () => 'result', + createModelOutput: () => ({ + type: 'content', + value: [{ type: 'unknown' }], + }), + expectedContent: UNPARSABLE_TOOL_RESULT, + }, +] + +const LEGACY_TOOL_RESULT_SCENARIOS = [ + { + name: 'empty legacy result', + execute: () => '', + expectedContent: '', + }, + { + name: 'zero legacy result', + execute: () => 0, + expectedContent: '0', + }, + { + name: 'false legacy result', + execute: () => false, + expectedContent: 'false', + }, + { + name: 'null legacy result', + execute: () => null, + expectedContent: 'null', + }, +] + describe('Plugin', () => { useEnv({ OPENAI_API_KEY: '', @@ -1076,6 +1186,176 @@ describe('Plugin', () => { }) }) + describe('tool result formatting', () => { + withVersions('ai', 'ai', range, (version, _, realVersion) => { + let ai + + beforeEach(() => { + ai = require(`../../../../../../versions/ai@${version}`).get() + }) + + /** + * @param {{ + * execute: () => unknown, + * createModelOutput?: () => unknown, + * expectedContent: string + * }} scenario + */ + async function assertToolResult (scenario) { + const isLegacy = semifies(realVersion, '<5.0.0') + const schema = ai.jsonSchema({ + type: 'object', + properties: {}, + }) + let model + let tool + if (isLegacy) { + model = createToolResultModelV1() + tool = ai.tool({ + id: 'testTool', + parameters: schema, + execute: scenario.execute, + }) + } else { + model = semifies(realVersion, '>=6.0.0') + ? createToolResultModelV3() + : createToolResultModelV2() + tool = ai.tool({ + inputSchema: schema, + execute: scenario.execute, + toModelOutput: scenario.createModelOutput, + }) + } + + const options = { + model, + prompt: 'Run the test tool', + tools: isLegacy ? [tool] : { testTool: tool }, + } + if (isLegacy) { + options.maxSteps = 2 + } else { + options.stopWhen = ai.stepCountIs(2) + } + + await ai.generateText(options) + + const { apmSpans, llmobsSpans } = await getEvents(4) + let finalModelSpan + let finalModelApmSpan + let workflowSpan + for (const span of llmobsSpans) { + if (span.name === 'doGenerate') { + finalModelSpan = span + } else if (span.name === 'generateText') { + workflowSpan = span + } + } + for (const span of apmSpans) { + if (span.name === 'ai.generateText.doGenerate') finalModelApmSpan = span + } + + assertLlmObsSpanEvent(finalModelSpan, { + span: finalModelApmSpan, + parentId: workflowSpan.span_id, + spanKind: 'llm', + modelName: 'test', + modelProvider: 'test', + name: 'doGenerate', + inputMessages: [ + { content: 'Run the test tool', role: 'user' }, + { + content: '', + role: 'assistant', + tool_calls: [{ + tool_id: 'call-1', + name: 'testTool', + arguments: {}, + type: 'function', + }], + }, + { + content: scenario.expectedContent, + role: 'tool', + tool_id: 'call-1', + }, + ], + outputMessages: [{ content: 'done', role: 'assistant' }], + metrics: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + } + + const scenarios = semifies(realVersion, '<5.0.0') + ? LEGACY_TOOL_RESULT_SCENARIOS + : TOOL_RESULT_SCENARIOS + for (const scenario of scenarios) { + it(`formats ${scenario.name} produced by the AI SDK`, async () => { + await assertToolResult(scenario) + }) + } + }) + + it('falls back for values rejected before model invocation', () => { + const circular = {} + circular.self = circular + + const malformedOutputs = [ + null, + 'text', + { type: 'text' }, + { type: 'text', value: 42 }, + { type: 'json', value: undefined }, + { type: 'json', value: Symbol('result') }, + { type: 'json', value: 1n }, + { type: 'json', value: circular }, + { type: 'content', value: {} }, + { type: 'content', value: [null] }, + { type: 'content', value: [{ type: 'text' }] }, + { type: 'content', value: [{ type: 'media' }] }, + { type: 'execution-denied', reason: {} }, + { type: 'unknown', value: 'result' }, + ] + + for (const output of malformedOutputs) { + assert.strictEqual( + getToolCallResultContent({ output }), + UNPARSABLE_TOOL_RESULT + ) + } + + assert.strictEqual(getToolCallResultContent(), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent(null), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent('result'), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({}), UNSUPPORTED_TOOL_RESULT) + }) + + it('does not throw for external objects rejected before model invocation', () => { + const throwingContent = new Proxy({}, { + get () { + throw new Error('unexpected content access') + }, + }) + const throwingOutput = new Proxy({}, { + get () { + throw new Error('unexpected output access') + }, + }) + const throwingParts = new Proxy([], { + get () { + throw new Error('unexpected content part access') + }, + }) + + assert.strictEqual(getToolCallResultContent(throwingContent), UNPARSABLE_TOOL_RESULT) + assert.strictEqual(getToolCallResultContent({ output: throwingOutput }), UNPARSABLE_TOOL_RESULT) + assert.strictEqual( + getToolCallResultContent({ output: { type: 'content', value: throwingParts } }), + UNPARSABLE_TOOL_RESULT + ) + }) + }) + describe('prompt cache token capture', () => { // Both @ai-sdk/amazon-bedrock and @ai-sdk/anthropic use globalThis.fetch // (not node:http), so nock cannot intercept them. Each provider's diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index 7ff867c297..71980ea690 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -11,6 +11,7 @@ const { useLlmObs, MOCK_NUMBER, } = require('../../util') +const { createToolResultModelV3 } = require('./tool-result-model') /** * @param {(version: string, openaiVersion: string) => void} callback @@ -934,6 +935,120 @@ describe('Plugin', () => { }) }) + describe('tool result formatting', () => { + withVersions('ai', 'ai', '>=7.0.0', version => { + let ai + + beforeEach(() => { + ai = require(`../../../../../../versions/ai@${version}`).get() + }) + + it('formats file result parts produced by the AI SDK', async () => { + await ai.generateText({ + model: createToolResultModelV3(), + prompt: 'Run the test tool', + tools: { + testTool: ai.tool({ + inputSchema: ai.jsonSchema({ + type: 'object', + properties: {}, + }), + execute: () => 'result', + toModelOutput: () => ({ + type: 'content', + value: [ + { type: 'text', text: 'before' }, + { + type: 'file', + mediaType: 'image/png', + data: { type: 'data', data: 'cHJpdmF0ZS1pbWFnZS1kYXRh' }, + }, + { + type: 'file', + mediaType: 'application/pdf', + data: { type: 'data', data: 'cHJpdmF0ZS1maWxlLWRhdGE=' }, + }, + { + type: 'file', + mediaType: 'application/pdf', + data: { type: 'reference', reference: { test: 'private-file-id' } }, + }, + { + type: 'file', + mediaType: 'image/png', + data: { type: 'reference', reference: { test: 'private-image-id' } }, + }, + { type: 'custom', providerOptions: { secret: 'provider-data' } }, + { type: 'text', text: 'after' }, + ], + }), + }), + }, + stopWhen: ai.stepCountIs(2), + }) + + const { apmSpans, llmobsSpans } = await getEvents(6) + let finalModelSpan + let finalModelApmSpan + let finalStepSpan + for (const span of llmobsSpans) { + if (span.name === 'languageModelCall') { + finalModelSpan = span + } else if (span.name === 'step') { + finalStepSpan = span + } + } + for (const span of apmSpans) { + if (span.name === 'languageModelCall') finalModelApmSpan = span + } + + assertLlmObsSpanEvent(finalModelSpan, { + span: finalModelApmSpan, + parentId: finalStepSpan.span_id, + spanKind: 'llm', + modelName: 'test', + modelProvider: 'test', + name: 'languageModelCall', + inputMessages: [ + { content: 'Run the test tool', role: 'user' }, + { + role: 'assistant', + tool_calls: [{ + tool_id: 'call-1', + name: 'testTool', + arguments: {}, + type: 'function', + }], + }, + { + content: 'before[Image][File][File][Image][Custom Content]after', + role: 'tool', + tool_id: 'call-1', + }, + ], + outputMessages: [{ content: 'done', role: 'assistant' }], + toolDefinitions: [{ + name: 'testTool', + schema: { + type: 'object', + properties: {}, + required: [], + }, + }], + metadata: {}, + metrics: { + input_tokens: 1, + cache_write_input_tokens: 0, + cache_read_input_tokens: 0, + output_tokens: 1, + reasoning_output_tokens: 0, + }, + tags: { ml_app: 'test', integration: 'ai' }, + }) + }) + }) + }) + describe('prompt cache token capture', () => { function makeMockFetch (scenario) { const fixture = require(`../../../../../datadog-plugin-ai/test/resources/${scenario}.json`) diff --git a/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js b/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js new file mode 100644 index 0000000000..35cfa0dea9 --- /dev/null +++ b/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js @@ -0,0 +1,141 @@ +'use strict' + +/** + * @typedef {{ + * specificationVersion: 'v1' | 'v2' | 'v3', + * provider: string, + * modelId: string, + * defaultObjectGenerationMode: undefined, + * supportedUrls: Record, + * doGenerate: () => Promise> + * }} ToolResultModel + */ + +/** + * @param {'v1' | 'v2' | 'v3'} specificationVersion + * @param {Array>} responses + * @returns {ToolResultModel} + */ +function createModel (specificationVersion, responses) { + let responseIndex = 0 + + return { + specificationVersion, + provider: 'test', + modelId: 'test', + defaultObjectGenerationMode: undefined, + supportedUrls: {}, + async doGenerate () { + return responses[responseIndex++] + }, + } +} + +function createToolResultModelV1 () { + const usage = { + promptTokens: 1, + completionTokens: 1, + } + const rawCall = { + rawPrompt: undefined, + rawSettings: {}, + } + + return createModel('v1', [ + { + toolCalls: [{ + toolCallType: 'function', + toolCallId: 'call-1', + toolName: '0', + args: '{}', + }], + finishReason: 'tool-calls', + usage, + rawCall, + warnings: [], + }, + { + text: 'done', + finishReason: 'stop', + usage, + rawCall, + warnings: [], + }, + ]) +} + +function createToolResultModelV2 () { + const usage = { + inputTokens: 1, + outputTokens: 1, + totalTokens: 2, + } + + return createModel('v2', [ + { + content: [{ + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'testTool', + input: '{}', + }], + finishReason: 'tool-calls', + usage, + warnings: [], + }, + { + content: [{ type: 'text', text: 'done' }], + finishReason: 'stop', + usage, + warnings: [], + }, + ]) +} + +function createToolResultModelV3 () { + const usage = { + inputTokens: { + total: 1, + noCache: 1, + cacheRead: 0, + cacheWrite: 0, + }, + outputTokens: { + total: 1, + text: 1, + reasoning: 0, + }, + } + + return createModel('v3', [ + { + content: [{ + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'testTool', + input: '{}', + }], + finishReason: { + unified: 'tool-calls', + raw: 'tool_calls', + }, + usage, + warnings: [], + }, + { + content: [{ type: 'text', text: 'done' }], + finishReason: { + unified: 'stop', + raw: 'stop', + }, + usage, + warnings: [], + }, + ]) +} + +module.exports = { + createToolResultModelV1, + createToolResultModelV2, + createToolResultModelV3, +} diff --git a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js deleted file mode 100644 index 1cac908c0f..0000000000 --- a/packages/dd-trace/test/llmobs/plugins/ai/util.spec.js +++ /dev/null @@ -1,205 +0,0 @@ -'use strict' - -const assert = require('node:assert/strict') - -const { getToolCallResultContent } = require('../../../../src/llmobs/plugins/ai/util') - -const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' -const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' - -describe('AI SDK LLMObs utilities', () => { - describe('getToolCallResultContent', () => { - it('formats text output variants', () => { - assert.strictEqual(getToolCallResultContent({ output: { type: 'text', value: 'result' } }), 'result') - assert.strictEqual(getToolCallResultContent({ output: { type: 'error-text', value: 'failure' } }), 'failure') - }) - - it('formats JSON output variants', () => { - const value = { answer: 42 } - - assert.strictEqual( - getToolCallResultContent({ output: { type: 'json', value } }), - JSON.stringify(value) - ) - assert.strictEqual( - getToolCallResultContent({ output: { type: 'error-json', value } }), - JSON.stringify(value) - ) - }) - - it('formats all legacy result values', () => { - assert.strictEqual(getToolCallResultContent({ result: '' }), '') - assert.strictEqual(getToolCallResultContent({ result: 'result' }), 'result') - assert.strictEqual(getToolCallResultContent({ result: 0 }), '0') - assert.strictEqual(getToolCallResultContent({ result: false }), 'false') - assert.strictEqual(getToolCallResultContent({ result: null }), 'null') - assert.strictEqual(getToolCallResultContent({ result: { answer: 42 } }), '{"answer":42}') - }) - - it('distinguishes absent legacy results', () => { - assert.strictEqual(getToolCallResultContent({}), UNSUPPORTED_TOOL_RESULT) - assert.strictEqual(getToolCallResultContent({ result: undefined }), UNSUPPORTED_TOOL_RESULT) - }) - - it('prefers defined output over legacy results', () => { - assert.strictEqual( - getToolCallResultContent({ - output: { type: 'text', value: '' }, - result: 'legacy result', - }), - '' - ) - assert.strictEqual( - getToolCallResultContent({ - output: null, - result: 'legacy result', - }), - UNPARSABLE_TOOL_RESULT - ) - assert.strictEqual( - getToolCallResultContent({ - output: undefined, - result: 'legacy result', - }), - 'legacy result' - ) - }) - - it('concatenates text and summarizes rich content without serializing payloads', () => { - const content = [ - { type: 'text', text: 'before' }, - { type: 'media', mediaType: 'image/png', data: 'legacy-image-data' }, - { type: 'media', mediaType: 'application/pdf', data: 'legacy-file-data' }, - { type: 'file', mediaType: 'image', data: { type: 'data', data: 'image-data' } }, - { type: 'file', mediaType: 'image/png', data: { type: 'url', url: 'https://example.com/image' } }, - { type: 'file', mediaType: 'application/pdf', data: { type: 'text', text: 'file-data' } }, - { type: 'file-data', mediaType: 'application/pdf', data: 'file-data' }, - { type: 'file-url', url: 'https://example.com/private-file' }, - { type: 'file-id', fileId: 'private-file-id' }, - { type: 'file-reference', providerReference: { openai: 'private-file-id' } }, - { type: 'image-data', mediaType: 'image/png', data: 'image-data' }, - { type: 'image-url', url: 'https://example.com/private-image' }, - { type: 'image-file-id', fileId: 'private-image-id' }, - { type: 'image-file-reference', providerReference: { openai: 'private-image-id' } }, - { type: 'custom', providerOptions: { secret: 'provider-data' } }, - { type: 'text', text: 'after' }, - ] - - assert.strictEqual( - getToolCallResultContent({ output: { type: 'content', value: content } }), - 'before[Image][File][Image][Image][File][File][File][File][File]' + - '[Image][Image][Image][Image][Custom Content]after' - ) - }) - - it('formats empty multipart content', () => { - assert.strictEqual(getToolCallResultContent({ output: { type: 'content', value: [] } }), '') - }) - - it('formats denied executions', () => { - assert.strictEqual( - getToolCallResultContent({ output: { type: 'execution-denied', reason: 'Approval rejected' } }), - 'Approval rejected' - ) - assert.strictEqual( - getToolCallResultContent({ output: { type: 'execution-denied' } }), - '[Tool Execution Denied]' - ) - assert.strictEqual( - getToolCallResultContent({ output: { type: 'execution-denied', reason: '' } }), - '' - ) - }) - - it('rejects malformed output variants', () => { - const malformedOutputs = [ - null, - 'text', - { type: 'text' }, - { type: 'text', value: 42 }, - { type: 'error-text', value: {} }, - { type: 'json', value: undefined }, - { type: 'json', value: Symbol('result') }, - { type: 'error-json', value: () => {} }, - { type: 'content', value: {} }, - { type: 'execution-denied', reason: {} }, - { type: 'unknown', value: 'result' }, - ] - - for (const output of malformedOutputs) { - assert.strictEqual(getToolCallResultContent({ output }), UNPARSABLE_TOOL_RESULT) - } - }) - - it('rejects malformed multipart content', () => { - const sparseContent = new Array(1) - const malformedValues = [ - [null], - ['text'], - [{ type: 'text' }], - [{ type: 'text', text: 42 }], - [{ type: 'media' }], - [{ type: 'file' }], - [{ type: 'file', mediaType: 42 }], - [{ type: 'unknown' }], - [{ type: Symbol('text'), text: 'result' }], - [{ type: 'text', text: 'before' }, { type: 'unknown' }], - sparseContent, - ] - - for (const value of malformedValues) { - assert.strictEqual( - getToolCallResultContent({ output: { type: 'content', value } }), - UNPARSABLE_TOOL_RESULT - ) - } - }) - - it('does not throw for circular values', () => { - const circular = {} - circular.self = circular - - assert.strictEqual( - getToolCallResultContent({ output: { type: 'json', value: circular } }), - UNPARSABLE_TOOL_RESULT - ) - assert.strictEqual( - getToolCallResultContent({ output: { type: 'error-json', value: circular } }), - UNPARSABLE_TOOL_RESULT - ) - assert.strictEqual(getToolCallResultContent({ result: circular }), UNPARSABLE_TOOL_RESULT) - assert.strictEqual(getToolCallResultContent({ result: 1n }), UNPARSABLE_TOOL_RESULT) - }) - - it('does not throw for throwing external objects', () => { - const throwingContent = new Proxy({}, { - get () { - throw new Error('unexpected content access') - }, - }) - const throwingOutput = new Proxy({}, { - get () { - throw new Error('unexpected output access') - }, - }) - const throwingParts = new Proxy([], { - get () { - throw new Error('unexpected content part access') - }, - }) - - assert.strictEqual(getToolCallResultContent(throwingContent), UNPARSABLE_TOOL_RESULT) - assert.strictEqual(getToolCallResultContent({ output: throwingOutput }), UNPARSABLE_TOOL_RESULT) - assert.strictEqual( - getToolCallResultContent({ output: { type: 'content', value: throwingParts } }), - UNPARSABLE_TOOL_RESULT - ) - }) - - it('rejects malformed content containers', () => { - assert.strictEqual(getToolCallResultContent(), UNPARSABLE_TOOL_RESULT) - assert.strictEqual(getToolCallResultContent(null), UNPARSABLE_TOOL_RESULT) - assert.strictEqual(getToolCallResultContent('result'), UNPARSABLE_TOOL_RESULT) - }) - }) -}) From f22981a51a820ce5447cc23d63ddc28ece98b148 Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Wed, 22 Jul 2026 16:20:02 -0400 Subject: [PATCH 5/8] update tests with cassettes --- ...openai_chat_completions_post_0361f19b.json | 50 +++++++ ...openai_chat_completions_post_0529368f.json | 50 +++++++ ...openai_chat_completions_post_09caa127.json | 50 +++++++ ...openai_chat_completions_post_0b6f9297.json | 50 +++++++ ...openai_chat_completions_post_1a817464.json | 50 +++++++ ...openai_chat_completions_post_1b92db1c.json | 50 +++++++ ...openai_chat_completions_post_20b36358.json | 50 +++++++ ...openai_chat_completions_post_20e896cf.json | 50 +++++++ ...openai_chat_completions_post_22e8e00d.json | 50 +++++++ ...openai_chat_completions_post_23de8c14.json | 50 +++++++ ...openai_chat_completions_post_24e03d01.json | 50 +++++++ ...openai_chat_completions_post_2f30d5b0.json | 50 +++++++ ...openai_chat_completions_post_3c8e8f64.json | 50 +++++++ ...openai_chat_completions_post_3de7cf9f.json | 50 +++++++ ...openai_chat_completions_post_40f5975e.json | 50 +++++++ ...openai_chat_completions_post_444a857f.json | 50 +++++++ ...openai_chat_completions_post_4ced261c.json | 50 +++++++ ...openai_chat_completions_post_57593709.json | 50 +++++++ ...openai_chat_completions_post_5bbf5257.json | 50 +++++++ ...openai_chat_completions_post_613472dd.json | 50 +++++++ ...openai_chat_completions_post_6163bd35.json | 50 +++++++ ...openai_chat_completions_post_62af4eb0.json | 50 +++++++ ...openai_chat_completions_post_6c05f6ae.json | 50 +++++++ ...openai_chat_completions_post_6c485410.json | 50 +++++++ ...openai_chat_completions_post_9cf39adf.json | 50 +++++++ ...openai_chat_completions_post_a17b7d89.json | 50 +++++++ ...openai_chat_completions_post_a233b9ea.json | 50 +++++++ ...openai_chat_completions_post_a509302d.json | 50 +++++++ ...openai_chat_completions_post_a51ba16a.json | 50 +++++++ ...openai_chat_completions_post_ad68426d.json | 50 +++++++ ...openai_chat_completions_post_adfc828f.json | 50 +++++++ ...openai_chat_completions_post_b9510f9a.json | 50 +++++++ ...openai_chat_completions_post_c0ec5af0.json | 50 +++++++ ...openai_chat_completions_post_d65abfba.json | 50 +++++++ ...openai_chat_completions_post_e6f7f8ad.json | 50 +++++++ ...openai_chat_completions_post_ea7e2114.json | 50 +++++++ ...openai_chat_completions_post_f0546f43.json | 50 +++++++ ...openai_chat_completions_post_f1893fa3.json | 50 +++++++ ...openai_chat_completions_post_fe44e2dd.json | 50 +++++++ ...openai_chat_completions_post_ff5c5e3d.json | 50 +++++++ ...openai_chat_completions_post_ffbf4f37.json | 50 +++++++ .../test/llmobs/plugins/ai/index.spec.js | 66 +++++--- .../test/llmobs/plugins/ai/index.v7.spec.js | 39 +++-- .../llmobs/plugins/ai/tool-result-model.js | 141 ------------------ 44 files changed, 2120 insertions(+), 176 deletions(-) create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json create mode 100644 packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json delete mode 100644 packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json new file mode 100644 index 0000000000..d34da23252 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0361f19b.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "558" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:31 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b39dc8af82f-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "614", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6e1a2998fd994324bdc2bd3ae10e3b19", + "set-cookie": "__cf_bm=YKu653J0zCeRmStBop1XaLs6fJWkrbz81oF6AMvk3Uw-1784751210.5337121-1.0.1.1-CC0wup.gI4qlYGFdD0aSjt7HqBGyfpOZU62zc.bTIq.SkmUw2hyT.n2jL4HcHnhFgzZkt_GuAacYFbzdji6fn5oXH3pLCYraOvYzfg.eU7eV9mkLC8Ngk2XuJmj889Bn; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:31 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbmnhhTOv8VuwXC9YKsgbUKoh53\",\n \"object\": \"chat.completion\",\n \"created\": 1784751210,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems there was an error when trying to run the test tool. The tool returned a failure message. Let me know if there's anything else you'd like to do or explore!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 36,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json new file mode 100644 index 0000000000..aa06581a43 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0529368f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/7.0.31 ai-sdk/provider-utils/5.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1163" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1pbWFnZS1kYXRh\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"data\\\",\\\"data\\\":\\\"cHJpdmF0ZS1maWxlLWRhdGE=\\\"}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-file-id\\\"}}},{\\\"type\\\":\\\"file\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":{\\\"type\\\":\\\"reference\\\",\\\"reference\\\":{\\\"test\\\":\\\"private-image-id\\\"}}},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:14:35 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51cc4eed4558f-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1456", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999855", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_ad352507330141ffb9fa2c32e9beed74", + "set-cookie": "__cf_bm=cGrX9T7WZTNYAiDLRBsji0Wm88HYpSxoSeEs52rseXc-1784751273.7446923-1.0.1.1-e6qf06EukNFlCF4bAl0AV.W7a2KMVtVg.nTGe.LgdDn7ewb7cNi0sdsC54KAAJnIy56ttxZwjxM0FxvFfezWKPlGaNjdAmhfHonkKb7h0JO3RcyhR73i8jkpIOKVPoOI; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:44:35 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xcovu6Kem6n8VYf3IRKOSorwOzI\",\n \"object\": \"chat.completion\",\n \"created\": 1784751274,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed successfully, and here are the results:\\n\\n1. **Text Output**: \\\"before\\\"\\n2. **File Outputs**:\\n - **Image (PNG)**: Data of the image (base64 encoded).\\n - **PDF**: Data of the PDF (base64 encoded).\\n - **PDF (reference)**: Reference to a private file ID.\\n - **Image (reference)**: Reference to a private image ID.\\n3. **Custom Output**: Contains provider options with a secret (\\\"provider-data\\\").\\n4. **Text Output**: \\\"after\\\"\\n\\nIf you need any specific detail or further processing for any of these outputs, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 217,\n \"completion_tokens\": 141,\n \"total_tokens\": 358,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json new file mode 100644 index 0000000000..3ae952aacc --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_09caa127.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "536" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:25 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1539540f69-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "491", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_10b5047ada3644d5b49cf674f4178817", + "set-cookie": "__cf_bm=pCM4br.NJaCS_q3lJ7RgSCtgULi9q_CZARrA7qujqpc-1784751204.681613-1.0.1.1-VSYB8tzdJcDE3E_cIGT6QHL1tA562__HsypCMqKrb5oP4JcmoaVzv3SdrFA0IRXCCieehEuUPvkQBSYb86mBDuvBpY.DhrdtG8aTAXcuQI80Uo4VhhfFsObJ0OI8maKN; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:25 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbgbuVSTdLedI8u3Qyiff75AQvo\",\n \"object\": \"chat.completion\",\n \"created\": 1784751204,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool ran and returned a result of type \\\"unknown.\\\" If you need further assistance or specific information, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 28,\n \"total_tokens\": 97,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json new file mode 100644 index 0000000000..1bd2406986 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_0b6f9297.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1139" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:28 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1cbdc5fef2-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "2094", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999862", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d45d4c4aa6154079acb9ba3e5d1fd06d", + "set-cookie": "__cf_bm=mpYZZ398HYHntWHmVHBRj2nndk7WrFG6l6J.2wlnq5U-1784751205.878469-1.0.1.1-yChyq3Mi4NV7U0iWTlk6tuCCRNLe.JT_WNOh7msyOjs32yBJIXSCVCaDM8hKkuF3WwsIHx0f2F3nGsQscd6FdYlyJqJ6TZu0GO6Cs6QsH5vaSbELNSmy0ZpKdCCiuas9; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:28 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbi4NNoi2wHb7RGCRHZ6lyDUfPn\",\n \"object\": \"chat.completion\",\n \"created\": 1784751206,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed, and here are the results:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Media (Image)**: PNG data (private)\\n3. **Media (PDF)**: PDF data (private)\\n4. **File Data (PDF)**: PDF file data (private)\\n5. **File ID**: private-file-id\\n6. **Image Data**: PNG image data (private)\\n7. **Image File ID**: private-image-id\\n8. **Custom**: Provider options with secret (private data)\\n9. **Text**: \\\"after\\\"\\n\\nIf you need any specific information or further actions based on these results, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 185,\n \"completion_tokens\": 141,\n \"total_tokens\": 326,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json new file mode 100644 index 0000000000..a5abc273a0 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1a817464.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "518" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:21 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51afe0b7af834-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "554", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_297c199134694dc6b763c81286de676d", + "set-cookie": "__cf_bm=EYO0B1S_ra4XD6097FmJ9hnxcw2kOpyVoUtVyYWdRtE-1784751200.9636052-1.0.1.1-PDvZhHjL7z4yX4t_hJzFBy6_bmfSQJHCd5XgnbWs28HC8aQO3VmKUbgug.8lvjJZoiGvcKEDOpdtTTR8HWYqu6MeIle.3rJHDSID6BF7Bt2jVlOm0dN2o2npmz6sKmeu; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:21 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbdvwo8PsNteG6VniMS5JCkLUcl\",\n \"object\": \"chat.completion\",\n \"created\": 1784751201,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has successfully run and returned the result. If you need more specific information or further assistance, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 27,\n \"total_tokens\": 90,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json new file mode 100644 index 0000000000..fb67c9926d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_1b92db1c.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "519" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:43 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b89ed1cb637-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "492", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d96e1a04d0214f86b2294bab0859eb59", + "set-cookie": "__cf_bm=xtSVSvtJlPb5QbLDYGLYell_RpKExImPcS3MQ6tR0nA-1784751223.3429987-1.0.1.1-MXuBKd3ASlYJSiVi2rqPcAiwjStik13EWl2oX7a3sgvKbUyhy.GhEkn_gD5QN9pVTui56EQ.EG3U65jpfK1Vw3REOLQAi2UwmYMhbIyJ2sQqrQDPcNVHlhaq9r9oWvpR; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:43 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbzdYCDs2V5nzdUAbS4WQHKokjW\",\n \"object\": \"chat.completion\",\n \"created\": 1784751223,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has successfully run and returned the result. Would you like to know more about it or need assistance with anything else?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 27,\n \"total_tokens\": 90,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json new file mode 100644 index 0000000000..42cf30bbf4 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20b36358.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "523" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:37 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b61cd81fbfb-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "599", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_62204cf23b1340339458766ff2b175be", + "set-cookie": "__cf_bm=XbJVsCo9UlomqUW0e38lKhKkqFkWv82L9KxI7cLFQ5M-1784751216.9298835-1.0.1.1-qEWhQpQSb7AAftdgJKmrXIui80Aa999HaE8iRL4pne53AwGyKFpCPoB4sCQR92bWjpgDBCyL4ZIel656Pr1c9EOSQLUmXfYwTbtCWBsqEO2nAiK_XXlYKSqqyZITzCIS; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:37 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbthUpMY96YvaKt2oKEvStCch6t\",\n \"object\": \"chat.completion\",\n \"created\": 1784751217,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when trying to run the test tool. If you need assistance with something specific, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 29,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json new file mode 100644 index 0000000000..f4b4052c91 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_20e896cf.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "276" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:15 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ad79af143d6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "482", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_c6e1d1a71b1448c6a01a4789b68b96fd", + "set-cookie": "__cf_bm=Ku3aEuPaXwCuI._sg76.n2alVtPLJYj4dlULIw2dMIA-1784751194.8122158-1.0.1.1-GYu5iJGI26HhHwDQat1mTAET6_PHCXDmeTC6q7wh7uiy3224ETgLFTbSh0.tp4eyEYmmn5lQ9OvJrZ7QVxMwyV2mfqDLZDHhm.aYn6pmDo71ijhjD5rSEIkfG15Gk18d; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:15 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbXfTLgre8oEblfCmgvGvJmrwSK\",\n \"object\": \"chat.completion\",\n \"created\": 1784751195,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_q604yF7U7NC9ME4wbnaOfqHD\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"0\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json new file mode 100644 index 0000000000..a9a277d531 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_22e8e00d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1131" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"data\\\":\\\"private-image-data\\\",\\\"mediaType\\\":\\\"image/png\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"data\\\":\\\"private-file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:35 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b4efda87611-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1528", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999857", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6bfb162699a848d9866dd8025748bcb2", + "set-cookie": "__cf_bm=xPGEY4GN3FLXmexCdEmwnF8lLQG_9063crRRgsXMKQg-1784751213.9207683-1.0.1.1-J_pMwqB_w_z_zChtxbRVc0cR1cb7u7BNQx4LDW74aaWvM1EuSs6W45DW.h0GjbSmseJeIVHlr5S_v.aBH8yrqzxtGyPy6mfMD2eKnLwhCefkMVxAqZOnI55_309C_p2t; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:35 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbqSj8WZ9GUxOt1AzXeAaDnD7Jk\",\n \"object\": \"chat.completion\",\n \"created\": 1784751214,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run and returned the following data:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Image Data**: (private image data in PNG format)\\n3. **File Data**: (private file data in PDF format)\\n4. **File Data**: (another private file data in PDF format)\\n5. **File ID**: (private file ID)\\n6. **Image Data**: (private image data in PNG format)\\n7. **Image File ID**: (private image ID)\\n8. **Custom Data**: (provider options with secret data)\\n9. **Text**: \\\"after\\\"\\n\\nPlease let me know if you need any further assistance or analysis on this data!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 187,\n \"completion_tokens\": 147,\n \"total_tokens\": 334,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json new file mode 100644 index 0000000000..869f550838 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_23de8c14.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "515" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:46 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b9c7cabda80-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "398", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_1b719827a70c46469531724e5c92d261", + "set-cookie": "__cf_bm=ByDsdIfHw1XGpPR_2g1BO26huZa0wQQZzN6xMHppM0w-1784751226.3170166-1.0.1.1-d0.lEGbk3Zy3VE6zBQYA0GHhgEJWip4ZJHcQypX960JqoVfbVjLpvshLjvSR8B.s.TJ1kgMYkmTnTa0VfMpdaDmM4ok7cXjx4d_obJvkJXqrxqTYdjGNhb4TRlyjlJIk; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:46 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc2u32wFzClxUrJkOjwYApahNLs\",\n \"object\": \"chat.completion\",\n \"created\": 1784751226,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and there are no issues to report.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 16,\n \"total_tokens\": 79,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json new file mode 100644 index 0000000000..ca2518f555 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_24e03d01.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "543" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:46 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b97c80860e6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "600", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_be1349635cb04a1a854ac7d2515cbf7f", + "set-cookie": "__cf_bm=7Koc3j5iC4BcjlIQhENAOsyBS2iNntf9XE1nyY_ZOtM-1784751225.5662088-1.0.1.1-qPt.UxSZ.o_Ks1_enlxV5CKVG2EtDMBDhLxUuXFLE3w7xsyHA6Q._rlao1AJ6Uyp4pwGZvnbyJxgj0GjqTKn10KTx2ZC2V5RXqQPG3HqBWAXf6Skvv3QWWz_D2zb5djD; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:46 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc1Id8CsCI59aHQEnY5aamMJFzo\",\n \"object\": \"chat.completion\",\n \"created\": 1784751225,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"There was a failure while attempting to run the test tool. If you need assistance with something specific, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 26,\n \"total_tokens\": 94,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json new file mode 100644 index 0000000000..f3fbf2df3a --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_2f30d5b0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "530" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Approval rejected\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:47 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b9ffcc2b12a-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "671", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_bb636c69889f421ebeb27db9c20067c4", + "set-cookie": "__cf_bm=ZMhjVUg40xaXkK7Cqo9uwGDTJnyo1LcFvmtnDVi5bMs-1784751226.8730886-1.0.1.1-x2.ptX1ozk34O9cFNEdm0YXBxxOQ47GrxlaNBvx8tf2xjueKr3zMXrYDVdjRqVn1PwM47L_FnWv_IgQfeDy2nVx4QYbz20H5ZHqXoAGYol9XeQ5zNRoMjJeFuJQd8A5Q; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:47 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc3RJysabFeBmjZucvAdNS6kdQE\",\n \"object\": \"chat.completion\",\n \"created\": 1784751227,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool execution was rejected. If you have any other requests or need assistance with something else, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 31,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json new file mode 100644 index 0000000000..5bfe90d8ad --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3c8e8f64.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "296" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:25 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b1978ddf2f9-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "378", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_0ccc5fed4b9f4e7e88c81c76f1b33ed1", + "set-cookie": "__cf_bm=P8FC.2zKl3RVBZGhTwmqssabz0cIdEBELaAxuYX3uRA-1784751205.3579347-1.0.1.1-ReLQDpggzHiZCntDPzvO289r8PWy0oaMEUSkcd13GZoM._XZeAOLoo.JmQCJI7o6Ias6YbXC0xxXLACnGfSzIhJquECbUDun_hE8MN7u_ofsW6evPTK0zCB48I0CVWFE; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:25 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbhydjr9JkTAeQCGNskVrKmoMxS\",\n \"object\": \"chat.completion\",\n \"created\": 1784751205,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_6CIsJhzx7yJcZ0igevQZacnB\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json new file mode 100644 index 0000000000..c67724b7f8 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_3de7cf9f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "513" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:39 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b6c4c187a02-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "581", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_356ce2f3a65149ca9e12813b43d03da9", + "set-cookie": "__cf_bm=MjMCAMrPAiw1pcdDz5brZgZUYKvB4Tv6.x6Uj38QCao-1784751218.6057847-1.0.1.1-it15KMe0DnApYaPT9LgOBWsTNvFVV0owZH093vw6Jf1Qg3O..SqNkcZQhVvV1lcK6RaeYC.BtuO41Znr9Ho4k9WuN3DXrvDKeUrfXwPgQkIpEO1qY17OzVaNxYI9VFw0; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:39 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbuoPTqtuuY0RuzbvtkqYb5q0K0\",\n \"object\": \"chat.completion\",\n \"created\": 1784751218,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and it returned an empty result.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 15,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json new file mode 100644 index 0000000000..8429611f39 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_40f5975e.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "500" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"0\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:16 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae1d8708095-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "461", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2470a02e38174c75b819395c57aaaf6a", + "set-cookie": "__cf_bm=n2qmzFo1ic2HmXH25168IMs56l1C_iwCTiYN6G8hrUA-1784751196.455854-1.0.1.1-6JHpCmlq3tkIalUWnP2vvcss2rledbWSoeIAsPQqjCxTmgkbMOH63srqISRSV4V612V711w1nr0s1jB1jV_XLlIHn80WD86zmBkEU6Y5qFsFr5lhy8Dm5tk0ARhWl2eP; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:16 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbY2oNI714Ubagbuih8ZiXneI9R\",\n \"object\": \"chat.completion\",\n \"created\": 1784751196,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been successfully run, and the result is 0.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 16,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json new file mode 100644 index 0000000000..9be787c2c2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_444a857f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "527" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:22 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0269040cfb-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "376", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_41280cee84014927be0ecd99e14a6a83", + "set-cookie": "__cf_bm=I3raLAOPtBvp526rJcA_3wRkO.kpmGZ6CL4Elnlqn2M-1784751201.6612327-1.0.1.1-VEQoEQ5iIg2P2lRX1kbAErbb5zkoQ3_kPq92Sg7kUrZBUT02uAo4PiBqRat4_RXRsfyxzGAHM25s0t.K8eH90C..2Qj6rQFj7EnGmYNYJCyMemiWM2tp7e9N2f.yRV_e; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:22 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbdXPkVqWQqtC7p6NSu1phnzj24\",\n \"object\": \"chat.completion\",\n \"created\": 1784751201,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json new file mode 100644 index 0000000000..b580f9920f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_4ced261c.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "524" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:23 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0588e732b1-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "908", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_d5083fe955fd4043a5b0534fb2069b40", + "set-cookie": "__cf_bm=87tOuW20ywX4MQkQNojhTe2gaiROiJ5a49_taI7lq7w-1784751202.1638446-1.0.1.1-0wxodCi7JjJfT8MEIfkm8tGu_xjhrZZ0M1wB_SGG0ojupeE2e71VSM10r.d1G9UkT3yJEw60ExzluQR9mxODb9J.tBJVCaveJI.KYxCr786KLL8BTs1.PKXf7RZP9SUy; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:23 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbeo6qekjtam5gjVrQpRswru9Y7\",\n \"object\": \"chat.completion\",\n \"created\": 1784751202,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any further instructions or if there's something specific you'd like to know or try, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 39,\n \"total_tokens\": 103,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json new file mode 100644 index 0000000000..8c885189d8 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_57593709.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1133" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"data\\\":\\\"private-image-data\\\",\\\"mediaType\\\":\\\"image/png\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"data\\\":\\\"private-file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:43 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b7f79d67ca5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1487", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999860", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_f1074054fab048ac88b4f01d5461905f", + "set-cookie": "__cf_bm=qBNnt.uECmOeSxrkl_3ZCpnjm1b5zpZPzmRGakuGWao-1784751221.6794298-1.0.1.1-g8L6H0EzVOE8NZlhs8UZtRHdwNST_K2e6nHv1TpoMPoQm6rQU0CX5yz1rh1yRbwqP8ytaGX.zn__P1SA46l2uofMycarwBN8xNxAd2w3E.bQb6CA01.USW14X553L0ik; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:43 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbxdCpE8KaPfAWKuODz61iYL525\",\n \"object\": \"chat.completion\",\n \"created\": 1784751221,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been executed successfully. Here are the results:\\n\\n1. **Text**: \\\"before\\\"\\n2. **Image Data**: (Image data in PNG format)\\n3. **File Data**: (PDF file data)\\n4. **File Data**: (PDF file data)\\n5. **File ID**: (ID for a private file)\\n6. **Image Data**: (Image data in PNG format)\\n7. **Image File ID**: (ID for a private image)\\n8. **Custom Data**: (Includes provider options with secret)\\n9. **Text**: \\\"after\\\"\\n\\nIf you need further assistance or specific details about these results, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 187,\n \"completion_tokens\": 143,\n \"total_tokens\": 330,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json new file mode 100644 index 0000000000..729d44bd6d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_5bbf5257.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "533" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Tool execution denied.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:40 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b75aab46e53-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "613", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_41efd80ca3cf450fa53d5c9373208b56", + "set-cookie": "__cf_bm=c06xgYCI9gejrgqYPFFdMtI0iugtMMJuEmj7aU5ZoEI-1784751220.104493-1.0.1.1-FNXn_3lin7ooTB9jLgjJa7E.3PD6pIzhEePrv2cSI7lxt2Fux8JO7aLRDwVrXbHYLfq6gPC44D1.Ym6iyeI14HeOcF51Dhz.q6IKtje.u7k3Dg0pa_wO4VT0HiegoaGP; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:40 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbwIoMppg1fNk9LYCJ5vzlAEI1U\",\n \"object\": \"chat.completion\",\n \"created\": 1784751220,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"I'm unable to execute the test tool at the moment due to restrictions. If there's something specific you'd like to know or another way I can assist you, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 66,\n \"completion_tokens\": 36,\n \"total_tokens\": 102,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json new file mode 100644 index 0000000000..90f81aeef2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_613472dd.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "540" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Tool call execution denied.\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:48 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ba4f91a7a9c-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "759", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999982", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2569847e1d604d0890b6347393dae874", + "set-cookie": "__cf_bm=LsvM5mbIaOt2YS1B8GrlhEHd3RqBW6vVyPMdfxdZtZ0-1784751227.674948-1.0.1.1-fUoG.BAcFseDn.lYorJI2CkEdcmIkh.w75BuOOQxL0LcTcG.UWkdC09r.1IUfvE_pCInHsb70c.10wg9edHJHAY3ln4TH_V0sEb6kQbyHbhZMv3edwYmvT7cp9e_yqtM; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:48 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc4CcfUre2asSIdEvmgXTMA1U6i\",\n \"object\": \"chat.completion\",\n \"created\": 1784751228,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that I'm unable to run the test tool due to restrictions on execution. If you have any specific questions or need assistance with something else, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 35,\n \"total_tokens\": 102,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json new file mode 100644 index 0000000000..1a84e837ff --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6163bd35.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "528" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"Approval rejected\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:40 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b715d929630-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "542", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a0d7b60d7d0a493aaf3a19034667aa94", + "set-cookie": "__cf_bm=_NYAnQPr3oW2fhOZVXzJKqvDsdEj0L1moRqkKeVCCQM-1784751219.4145305-1.0.1.1-LhLGdqC6yXufp5DV1JSmK9SSBI6RwFlD2yuHZmRXR4u9Hsv_aOBQT6j53T9eIp_2n7yobRa6QEsZR.J2hapgtb3UAQDTsGvU40vZqd5.yHJucjusOCBJ.REbkddDdwhe; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:40 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbvgjRmNStygKS6QZNDgTbc1a4F\",\n \"object\": \"chat.completion\",\n \"created\": 1784751219,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool's execution resulted in an \\\"Approval rejected\\\" response. If you have any specific tasks or questions, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 31,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json new file mode 100644 index 0000000000..89529873f0 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_62af4eb0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "540" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:30 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b31dfe60d00-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "855", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3c6be94d02a648babf9be23026a73253", + "set-cookie": "__cf_bm=RSNfG4jxmlpdX_2x2cJH04sZp4P4cKAeq0WVfgnXnJ4-1784751209.2534914-1.0.1.1-XNgapA16TOPv7Co7aQAGNf1EIINoV3ln81KcXjuewOfagBDkzh8Bnq4Cwx97hNx4ap7AI2UcqC6JPWWc5yF5yFWqsGMUTNjVFXFyUPlg_.AzhfdiQ6Q3BVWNbvblTScb; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:30 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xblgq0buhNTMYsMbpdKrTAXSiT1\",\n \"object\": \"chat.completion\",\n \"created\": 1784751209,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any specific details or requests you'd like to share, I can try to assist further!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 36,\n \"total_tokens\": 100,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json new file mode 100644 index 0000000000..658596e45a --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c05f6ae.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "525" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"tool failure\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:45 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b914beac4d1-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "867", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_5a7e00cd2f2f45cbaf9851aada34f451", + "set-cookie": "__cf_bm=Sj89kb3nTZkdSSOpgbhUJMytnotJKfvUO51BNDL5tB4-1784751224.5265749-1.0.1.1-7fsO2Monifz3jKTodw6WftFK7abHP_9g6FTuUj9.7WLAaXwz7fPoI2X.Gd6v1YxU_Q0ZohLllIvRWBhJ07OVvpCzR3OmoegYBLsgzNa9ZW0Ytyk.VPduE0WDkcHM4BNR; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:45 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc0c8asWv7MM02JyzyvedPJ7GnP\",\n \"object\": \"chat.completion\",\n \"created\": 1784751224,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems there was an issue running the test tool. Would you like me to assist you with something else or try running the tool again?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 64,\n \"completion_tokens\": 29,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json new file mode 100644 index 0000000000..d5dae7247f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_6c485410.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "281" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:33 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b4b1bc23448-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "479", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a07f4223ae4b42c98b1cd5cdc1664c2a", + "set-cookie": "__cf_bm=X07fg6iyVC2ZLgESW1t0V7csLl35TvtL2PWLZA.4U6c-1784751213.2972581-1.0.1.1-tT8FPQjXicdyMvB1lnnqyiurUc9Nl9zf8WSKFxQXsfLKmO7HYczcGYj7O2bjdiOy4sgDefDMHo.Xz0_MHPwgkm962RVsE6g.zf0e4QmDTHulFr3Y3zRaPrNX0ZmqUrK.; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:33 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbp9eavrDkG1EdR47AqltNCElO1\",\n \"object\": \"chat.completion\",\n \"created\": 1784751213,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_ivr6y32ed6IgA8Df696Hq5TV\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json new file mode 100644 index 0000000000..b917d0e515 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_9cf39adf.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "542" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:23 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0c7890da48-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "424", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_a7579857c8d24373ac4229d38b385d4f", + "set-cookie": "__cf_bm=Y4.HBtnweGax8e9yelwxTiy9rl0CHGHP3xu2RUh0oH0-1784751203.271045-1.0.1.1-AuxB.AXslvF2t0sLXxTeBPUAMi.NwNa.K5ZOi993x2cBascPVNFDyB6YorIpY1VMDGeEHd6LeT93O5hYq9zw13VZriZ0BC6gfrUlokgr5OdJ8GyClUrWbL2ud6_PxAjl; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:23 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbfibyy4npm8fAryRGiHyfgvv4R\",\n \"object\": \"chat.completion\",\n \"created\": 1784751203,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool encountered a failure. How can I assist you further?\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 15,\n \"total_tokens\": 83,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json new file mode 100644 index 0000000000..14bc786368 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a17b7d89.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "1123" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"before\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"media\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-data\\\",\\\"mediaType\\\":\\\"application/pdf\\\",\\\"data\\\":\\\"private-file-data\\\"},{\\\"type\\\":\\\"file-id\\\",\\\"fileId\\\":\\\"private-file-id\\\"},{\\\"type\\\":\\\"image-data\\\",\\\"mediaType\\\":\\\"image/png\\\",\\\"data\\\":\\\"private-image-data\\\"},{\\\"type\\\":\\\"image-file-id\\\",\\\"fileId\\\":\\\"private-image-id\\\"},{\\\"type\\\":\\\"custom\\\",\\\"providerOptions\\\":{\\\"secret\\\":\\\"provider-data\\\"}},{\\\"type\\\":\\\"text\\\",\\\"text\\\":\\\"after\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:20 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51af3bf244b05-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "1511", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999860", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_6249f88c64014cd1bf47344a3dfca024", + "set-cookie": "__cf_bm=qGUDyxGppIZmuRO79s_KGLMtRZPapFL_BNuUCio3b1Q-1784751199.314885-1.0.1.1-Nsmo7BiuF_SOSS3BX_3bpjsCRXCBmFQazwL4CeWZUZoVRb4F8doxj5atXcTpUCjVhpQ9WTsHqvJTpJMXQ7cYZCTA1Hnf7opb4F4yZdRqXCWICi.SoVdD1ddEPv2m1p19; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:20 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbbdhlckyPXklTDOMjygf5LIxVi\",\n \"object\": \"chat.completion\",\n \"created\": 1784751199,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully, and here are the results:\\n\\n- **Text Output**: \\\"before\\\"\\n- **Media Outputs**:\\n - Image (PNG): Data is available but private.\\n - PDF: Data is available but private (two representations).\\n- **File Information**:\\n - PDF: File ID is private.\\n - Image: File ID is private.\\n- **Custom Output**: Provider options contain secret data.\\n- **Text Output**: \\\"after\\\"\\n\\nIf you need further analysis or specific actions based on this output, let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 185,\n \"completion_tokens\": 116,\n \"total_tokens\": 301,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json new file mode 100644 index 0000000000..64a11f012f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a233b9ea.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "535" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:41 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b7a89efa6af-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "627", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_1c254192d8574605a36257e5109ae566", + "set-cookie": "__cf_bm=iuia7OhoWEZY7lSpmY6ZX10b55NGLhg_h3D.o3R654s-1784751220.8863275-1.0.1.1-sIr2iezgv6mAAkOrkYblVkpFQmz2OrwKl1AwoF5GtN9SplUBnaXevlbxwQfAfo0iXreFr.0essqyop6pNaoh65_.bI5Q5tIw_L0y6xcVqbbbxMT5go1MTsq8efZ9i10k; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:41 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xbxmnk84PLXoT7bplxy4JnLdkEr\",\n \"object\": \"chat.completion\",\n \"created\": 1784751221,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool returned an unexpected result with the type \\\"unknown.\\\" If you need further assistance or would like to run another test, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 35,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json new file mode 100644 index 0000000000..b0b5ed0a7f --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a509302d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "504" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"false\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:17 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae5e8df4693-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "396", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_cfa8bea4c1cd425992005d7ca09be2a1", + "set-cookie": "__cf_bm=LZ5xXOVoNKgcA90JYCkxSYWJJQqbM5wFOOszP.hru8I-1784751197.1020737-1.0.1.1-GX.74RyXaRUI4VBIzYuZTr_PZO_wCM7kuk4jN6aQjNtgYdfq7a_NoEawxdWBgYFISBdfGRM1zuli7NAcksgqFbz8gXzarSEpT8QjHiclSHo9Fdh78kcwTPW1VyKBFAux; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:17 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbZv01VAKoxJX8NxHdbefVtekvS\",\n \"object\": \"chat.completion\",\n \"created\": 1784751197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has returned a result of `false`.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 12,\n \"total_tokens\": 74,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json new file mode 100644 index 0000000000..9915e6de50 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_a51ba16a.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "534" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:28 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b2adf54c413-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "437", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9fb1a586cd984a3bba19f65c7443aa8b", + "set-cookie": "__cf_bm=EzEjU1HhKiY9hreJQIATKJOpxMBCsEWC9MIfLn0PAKs-1784751208.135568-1.0.1.1-FgY.uv1fBU_GPmVGsFatD1VksQKR4D5zSjU_Is2YkBsDULOYVEUumNVLbx85ad3E9ln0792xvYQKBjK8CJAbnM1ybwKKmeNN1UY21wlQlXEyRINCF0heI0RIMGaWnpMf; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:28 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbkTxii73qYvsuL9lFBmp6HdM76\",\n \"object\": \"chat.completion\",\n \"created\": 1784751208,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run successfully, and the result is \\\"result.\\\"\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 16,\n \"total_tokens\": 79,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json new file mode 100644 index 0000000000..7c43375f2c --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ad68426d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "541" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"message\\\":\\\"tool failure\\\"}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:38 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b670cf65594-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "697", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_2d93713c6e7e457993f81c8971c3712e", + "set-cookie": "__cf_bm=phxPS1gE0FdWZIT5AxLS3E.rOnWEG19zmZ87RDuz3rE-1784751217.7661781-1.0.1.1-LsTzBkm4m8szg_0iHXr6srtsMYfiLTm5VXaYKn75_y_26RMAMX0UgKODFSfSEVkrXkkawyiAawt_CKAMoM4hkMmKqR31BlbSHsm32KEgiKOKbGJxYPiVdpN1rx9bQ6Se; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:38 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbtiCYiG8SaNXcVlBmTC23WSqIJ\",\n \"object\": \"chat.completion\",\n \"created\": 1784751217,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that there was a failure when attempting to run the test tool. If you have any other requests or need assistance with something else, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 68,\n \"completion_tokens\": 36,\n \"total_tokens\": 104,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json new file mode 100644 index 0000000000..90b0511309 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_adfc828f.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "528" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:44 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b8dfbc627c6-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "398", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_4ea1b722c19340a1995a17f2d80cc3bb", + "set-cookie": "__cf_bm=cekRPRKQ_EWZLWb4CbVyU6RarEqf759OA2VOYApCXRU-1784751223.9933553-1.0.1.1-nAFttum.0veky8iCizQlrG33.6wP8upoTOwM8DDL6ZHQYjhtezjGGn3g9xpUKKnpuXn5D25ys.k0rnvNgu3MfzpEvRGBBK8vXVn8VcOEYmtmMKVQzh9oZ0kWkVowzdED; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:44 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc0DmliNZMDOO5rEVnjgCRTcHt3\",\n \"object\": \"chat.completion\",\n \"created\": 1784751224,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json new file mode 100644 index 0000000000..3404d04741 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_b9510f9a.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "517" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"result\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:36 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b597b952067-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "596", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_18679b6262724719a768fef0ebb89228", + "set-cookie": "__cf_bm=zu53laSLyfyXEspdvsbqK3ndkzvrALN0red4uvlVkN4-1784751215.6010828-1.0.1.1-WXI7vVSQN_M9HJ8tFkDVaGSByT7iTsUx.r9D2TnXtp8BstJ7_gQMejbOb4x1zi5WianpDu6u42JBK3yqGfVlMIe.pYaFDjkiRo6GUwpWKDADrPAKFq_NJmnpxd7kfHHJ; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:36 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbrMOv54SAbjYV5tKyZDKNUPXr3\",\n \"object\": \"chat.completion\",\n \"created\": 1784751215,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been successfully executed, and the result is \\\"result.\\\" If you need any further assistance or details, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 30,\n \"total_tokens\": 93,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json new file mode 100644 index 0000000000..f8b83b9012 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_c0ec5af0.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "503" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"\\\"\\\"\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:16 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51adcb93541f3-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "625", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_e0f1d5e68c3b4be38c6b3e94aec4ad3e", + "set-cookie": "__cf_bm=7Eq9Mj09AgZ_tgk3xJTfcSyR51ID6v9j3EXp_axKwE8-1784751195.634347-1.0.1.1-daQssUPtYhnf7GM8z7lRp0.YLX6fVXuRkDtWoM.1nZVDzNbPgdxKfiS62yxEiJ3BdXA88PFuDFvnFwcG9NEqzJs8pq951iEjVPOIR8BZthy.G2npliXatCQ3Zrs9pb7X; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:16 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbXwGkPpgqyjtl4sni9SgIATWa0\",\n \"object\": \"chat.completion\",\n \"created\": 1784751195,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run, and it returned an empty result. If you need further assistance or have specific tasks in mind, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 34,\n \"total_tokens\": 96,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json new file mode 100644 index 0000000000..96adef2604 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_d65abfba.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "514" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_tMh2IPARQABNSUlF7F8j20b1\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:24 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b0fd81afe89-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "723", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3ead4596161c4bd8bc82183a19ea2866", + "set-cookie": "__cf_bm=NRlAnE9YEbDhcqR0gXinhw8h_lT.8ex6MKXU7gNUm_8-1784751203.8204072-1.0.1.1-yjzQwdoKBYJV7Q3nBCG2tTtJ2I0vO7I3iamAUbGLY9WIAgab_k4vGYwt1btJUvNwmhfuTE.CpNRLRo_WPFGUU3VP.akQsU.HOSEJa2mVJ5WhQ4G_WVeswRgUU5iLsPTw; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:24 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbgqTe0w7dJJyedm7XiWemCAIl6\",\n \"object\": \"chat.completion\",\n \"created\": 1784751204,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has run successfully and returned an empty result. If you need further assistance or have any specific tasks in mind, feel free to let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 33,\n \"total_tokens\": 96,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json new file mode 100644 index 0000000000..656c01e39d --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_e6f7f8ad.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.40 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "537" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:49 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51baacd1c7ca5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "689", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_9a21210a1e65464da99121b5d8d972b6", + "set-cookie": "__cf_bm=YBa3kvvV1p5XfTA6nl6dsnxIABIa8R.GUpOLhne2ofE-1784751228.6086779-1.0.1.1-ybCNUCGoCLVntZI67Lsz811g3Qfln87yaR_VdxUvFB.All4ppikVUq70KVu3slQ1EdngxeXYVESMhojJsnhnJtMEFjn.5xddg3QGsyYRFdbkYPtCZuQ7jOBbmchW8eL2; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:49 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4Xc4mWZCtMXeQzHn8eoTMONG9Dx7\",\n \"object\": \"chat.completion\",\n \"created\": 1784751228,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned an unexpected result with a type categorized as \\\"unknown.\\\" If you have any specific tests or checks in mind that I should perform, please let me know!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 36,\n \"total_tokens\": 105,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json new file mode 100644 index 0000000000..9fa0bf1595 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ea7e2114.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "User-Agent": "node", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "503" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"temperature\":0,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"type\":\"function\",\"function\":{\"name\":\"0\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_q604yF7U7NC9ME4wbnaOfqHD\",\"content\":\"null\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"0\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:18 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51ae9cad5114c-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "657", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999990", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_967755fe6eee4fe9bfab28750c6c09f1", + "set-cookie": "__cf_bm=I4UqtCrvps0v2ty3gz1xSFh26ZIuAX8Xiqicdshuh9U-1784751197.724774-1.0.1.1-4rAMdrQLz405QzLdUkF60cbJn3vbykmnHH2yQ9BpTTu56lHqeG8augxU92RGnVDo_IovNjgouF4xoPAX2Z7vV1ZHv4UpW95yGKdBQ6RJXu5qJ4O1m8ZHj_EI9wCSwxoy; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:18 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbZLyCelQ3nvFiq0xDDyY5Hi6pA\",\n \"object\": \"chat.completion\",\n \"created\": 1784751197,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool has been run, and it returned a null result. If you have any specific tasks or questions, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 62,\n \"completion_tokens\": 29,\n \"total_tokens\": 91,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_176e862412\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json new file mode 100644 index 0000000000..352507b599 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f0546f43.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "543" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:29 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b2ea8e54ba5-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "377", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_3da9f65def934135b9911152252e0b1b", + "set-cookie": "__cf_bm=nJBTYPDabmHNdR5pg59YNxg5nn6XjReam076HODmcFE-1784751208.743452-1.0.1.1-Ax7siAdtZeXIKEYNyCMsupm5A77y34C8uQJhqvEZhlComB_QQ8CUy2wkbvOXh4xRkE1ES7Qv9kX7noEGxPn99aQgXIC1O8M0XQFDx2nrrsxIGzDjDVo_NeBPC8TpXwNb; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:29 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbkThLDPnMm8iH15p8PmVQrvYcU\",\n \"object\": \"chat.completion\",\n \"created\": 1784751208,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json new file mode 100644 index 0000000000..25df7907f5 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_f1893fa3.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "530" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:32 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b3f2d6c10f3-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "617", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999992", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_4530a5faf6444944b5e4778c8b6f7b09", + "set-cookie": "__cf_bm=6_5U_Z3fUByPXyzymtL4UkNsg49jMyqQzHmnvTW4Pgg-1784751211.3817-1.0.1.1-PGuYa1WNodIwMWGVmB7uD_pMRmtu2hYKWNaoBJE7UsMfG5C0BZSWrGbTVy1riU6UcqqGMRlwNpnAk0q33b.UnPq2eVPu8fRVXBHlHw1fFAcHLzAvj5_rXjyHaviW5j8u; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:32 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbnMCi8Zy5Ez28pKEqhIZqRbe29\",\n \"object\": \"chat.completion\",\n \"created\": 1784751211,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool ran successfully, and there were no issues or errors reported. If you need further assistance or have any other requests, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 63,\n \"completion_tokens\": 32,\n \"total_tokens\": 95,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json new file mode 100644 index 0000000000..10566974a2 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_fe44e2dd.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "282" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:19 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51af01ce025d8-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "440", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999995", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_0e884b0f8b904434a4f06e24815083f3", + "set-cookie": "__cf_bm=PzlTWDz15ANmZ1wb9wHasUnOrz0eXJylzeuGWq2wfBQ-1784751198.7381263-1.0.1.1-I0FeBDkEOq1outWGxw0fDhoUwmLQ8I2MJD8uG_pcrCtdEBw01fkLJRJMQRjDSMy_No3NtuaUNlaPACyr5PTxxshRI7xQRokq4QVaR9D0ec1gHpSKxODsIucmzexzm38q; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:19 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbaJe2aGSrM21YQfqgww3Ro1fNn\",\n \"object\": \"chat.completion\",\n \"created\": 1784751198,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": null,\n \"tool_calls\": [\n {\n \"id\": \"call_tMh2IPARQABNSUlF7F8j20b1\",\n \"type\": \"function\",\n \"function\": {\n \"name\": \"testTool\",\n \"arguments\": \"{}\"\n }\n }\n ],\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"tool_calls\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 44,\n \"completion_tokens\": 10,\n \"total_tokens\": 54,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json new file mode 100644 index 0000000000..eb48f5c669 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ff5c5e3d.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/6.0.233 ai-sdk/provider-utils/4.0.0 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "526" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":\"\",\"tool_calls\":[{\"id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_ivr6y32ed6IgA8Df696Hq5TV\",\"content\":\"{\\\"answer\\\":42}\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}}}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:36 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b5e0fddde6d-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "372", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999987", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_7edc4616bfe840228c24cd1c74c5f10b", + "set-cookie": "__cf_bm=D3eloq1tHW95.9H0vqeworhMTqoy3kdMPxgyTOHZKxw-1784751216.3240464-1.0.1.1-yErURngg3hVKX_Xs6nbmfHEbytAbMJDVlsQlKH1PAYyPINtwZadmeiAh6G3NTdcZ6lFVLjVaM_kGBcqbN9ILsJWnjV4B1yWLr_mjtyKiCGM9Z7lQ0VTCptilduTsx4up; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:36 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbsY3n3vv3N5zbigam1ZOdubvrk\",\n \"object\": \"chat.completion\",\n \"created\": 1784751216,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"The test tool returned the answer: 42.\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 67,\n \"completion_tokens\": 11,\n \"total_tokens\": 78,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json new file mode 100644 index 0000000000..bf6a900748 --- /dev/null +++ b/packages/dd-trace/test/llmobs/cassettes/openai/openai_chat_completions_post_ffbf4f37.json @@ -0,0 +1,50 @@ +{ + "request": { + "method": "POST", + "url": "https://api.openai.com/v1/chat/completions", + "headers": { + "Connection": "keep-alive", + "Content-Type": "application/json", + "User-Agent": "ai/5.0.218 ai-sdk/provider-utils/3.0.30 runtime/node.js/24", + "Accept": "*/*", + "Accept-Language": "*", + "sec-fetch-mode": "cors", + "Accept-Encoding": "gzip, deflate", + "Content-Length": "552" + }, + "body": "{\"model\":\"gpt-4o-mini\",\"store\":false,\"messages\":[{\"role\":\"user\",\"content\":\"Run the test tool\"},{\"role\":\"assistant\",\"content\":null,\"tool_calls\":[{\"id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"arguments\":\"{}\"}}]},{\"role\":\"tool\",\"tool_call_id\":\"call_6CIsJhzx7yJcZ0igevQZacnB\",\"content\":\"[{\\\"type\\\":\\\"unknown\\\"}]\"}],\"tools\":[{\"type\":\"function\",\"function\":{\"name\":\"testTool\",\"description\":\"Run the test tool and return its result\",\"parameters\":{\"type\":\"object\",\"properties\":{}},\"strict\":false}}],\"tool_choice\":\"auto\"}" + }, + "response": { + "status": { + "code": 200, + "message": "OK" + }, + "headers": { + "Date": "Wed, 22 Jul 2026 20:13:33 GMT", + "Content-Type": "application/json", + "Transfer-Encoding": "chunked", + "Connection": "keep-alive", + "CF-Ray": "a1f51b443caab547-EWR", + "CF-Cache-Status": "DYNAMIC", + "Server": "cloudflare", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains; preload", + "X-Content-Type-Options": "nosniff", + "access-control-expose-headers": "X-Request-ID, CF-Ray, CF-Ray", + "openai-processing-ms": "739", + "openai-project": "proj_gt6TQZPRbZfoY2J9AQlEJMpd", + "openai-version": "2020-10-01", + "x-openai-proxy-wasm": "v0.1", + "x-ratelimit-limit-requests": "30000", + "x-ratelimit-limit-tokens": "150000000", + "x-ratelimit-remaining-requests": "29999", + "x-ratelimit-remaining-tokens": "149999985", + "x-ratelimit-reset-requests": "2ms", + "x-ratelimit-reset-tokens": "0s", + "x-request-id": "req_300d60eb6b40492b87fb0e3eb0b99185", + "set-cookie": "__cf_bm=_ICJux.AGa1KeFSScSnFEOLM8tO9.tXDz0oymwkPC3g-1784751212.1932561-1.0.1.1-q_4cQNvCdSokThOa3dkaLUbTH9DAkAuubeyyHqY37h2LZEuawROmbjZn8KN8W3u2qYvSwm1mecjNNS3k5BFNICibwIPYx3Pa9JlbrJqisMaJVrhmGzgZ0YOw3gR_ozl4; HttpOnly; SameSite=None; Secure; Path=/; Domain=api.openai.com; Expires=Wed, 22 Jul 2026 20:43:33 GMT", + "Content-Encoding": "gzip", + "alt-svc": "h3=\":443\"; ma=86400" + }, + "body": "{\n \"id\": \"chatcmpl-E4XbowYSzLdthND0NmxbSOsvMxoPx\",\n \"object\": \"chat.completion\",\n \"created\": 1784751212,\n \"model\": \"gpt-4o-mini-2024-07-18\",\n \"choices\": [\n {\n \"index\": 0,\n \"message\": {\n \"role\": \"assistant\",\n \"content\": \"It seems that the test tool has returned an unknown result. If you have specific questions or need assistance with something else, feel free to ask!\",\n \"refusal\": null,\n \"annotations\": []\n },\n \"logprobs\": null,\n \"finish_reason\": \"stop\"\n }\n ],\n \"usage\": {\n \"prompt_tokens\": 69,\n \"completion_tokens\": 30,\n \"total_tokens\": 99,\n \"prompt_tokens_details\": {\n \"cached_tokens\": 0,\n \"audio_tokens\": 0\n },\n \"completion_tokens_details\": {\n \"reasoning_tokens\": 0,\n \"audio_tokens\": 0,\n \"accepted_prediction_tokens\": 0,\n \"rejected_prediction_tokens\": 0\n }\n },\n \"service_tier\": \"default\",\n \"system_fingerprint\": \"fp_67faaa3b02\"\n}\n" + } +} \ No newline at end of file diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js index 52209bfefb..61b4400194 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.spec.js @@ -19,11 +19,6 @@ const { MOCK_NUMBER, MOCK_OBJECT, } = require('../../util') -const { - createToolResultModelV1, - createToolResultModelV2, - createToolResultModelV3, -} = require('./tool-result-model') const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]' const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]' @@ -72,12 +67,17 @@ function getAiSdkAnthropicOrGoogleRange (vercelAiVersion) { /** * @param {string} versionRange - * @param {(version: string, realVersion: string, openaiVersion: string) => void} callback + * @param {( + * version: string, + * realVersion: string, + * openaiVersionKey: string, + * openaiVersion: string + * ) => void} callback */ function withAiSdkOpenAiVersions (versionRange, callback) { withVersions('ai', 'ai', versionRange, (version, _, realVersion) => { - withVersions('ai', '@ai-sdk/openai', getAiSdkOpenAiRange(realVersion), openaiVersion => { - callback(version, realVersion, openaiVersion) + withVersions('ai', '@ai-sdk/openai', getAiSdkOpenAiRange(realVersion), (openaiVersionKey, _, openaiVersion) => { + callback(version, realVersion, openaiVersionKey, openaiVersion) }) }) } @@ -142,6 +142,7 @@ const TOOL_RESULT_SCENARIOS = [ }, { name: 'denied execution', + openaiRange: '>=3.0.0', execute: () => 'result', createModelOutput: () => ({ type: 'execution-denied', @@ -151,6 +152,7 @@ const TOOL_RESULT_SCENARIOS = [ }, { name: 'denied execution without a reason', + openaiRange: '>=3.0.0', execute: () => 'result', createModelOutput: () => ({ type: 'execution-denied' }), expectedContent: '[Tool Execution Denied]', @@ -1187,18 +1189,27 @@ describe('Plugin', () => { }) describe('tool result formatting', () => { - withVersions('ai', 'ai', range, (version, _, realVersion) => { + withAiSdkOpenAiVersions(range, (version, realVersion, openaiVersionKey, openaiVersion) => { let ai + let openai beforeEach(() => { ai = require(`../../../../../../versions/ai@${version}`).get() + + const OpenAIModule = require(`../../../../../../versions/@ai-sdk/openai@${openaiVersionKey}`) + const OpenAI = OpenAIModule.get() + openai = OpenAI.createOpenAI({ + baseURL: 'http://127.0.0.1:9126/vcr/openai', + compatibility: 'strict', + }) }) /** * @param {{ * execute: () => unknown, * createModelOutput?: () => unknown, - * expectedContent: string + * expectedContent: string, + * openaiRange?: string * }} scenario */ async function assertToolResult (scenario) { @@ -1207,20 +1218,17 @@ describe('Plugin', () => { type: 'object', properties: {}, }) - let model let tool if (isLegacy) { - model = createToolResultModelV1() tool = ai.tool({ id: 'testTool', + description: 'Run the test tool and return its result', parameters: schema, execute: scenario.execute, }) } else { - model = semifies(realVersion, '>=6.0.0') - ? createToolResultModelV3() - : createToolResultModelV2() tool = ai.tool({ + description: 'Run the test tool and return its result', inputSchema: schema, execute: scenario.execute, toModelOutput: scenario.createModelOutput, @@ -1228,7 +1236,9 @@ describe('Plugin', () => { } const options = { - model, + // Chat Completions serializes every tool result as text. The Responses API rejects some valid AI SDK + // result variants before the instrumented request can capture their formatted value. + model: openai.chat('gpt-4o-mini'), prompt: 'Run the test tool', tools: isLegacy ? [tool] : { testTool: tool }, } @@ -1237,8 +1247,16 @@ describe('Plugin', () => { } else { options.stopWhen = ai.stepCountIs(2) } + if (semifies(openaiVersion, '>=2.0.50')) { + options.providerOptions = { + openai: { + store: false, + }, + } + } - await ai.generateText(options) + const result = await ai.generateText(options) + const toolCallId = result.steps[0].toolCalls[0].toolCallId const { apmSpans, llmobsSpans } = await getEvents(4) let finalModelSpan @@ -1259,8 +1277,8 @@ describe('Plugin', () => { span: finalModelApmSpan, parentId: workflowSpan.span_id, spanKind: 'llm', - modelName: 'test', - modelProvider: 'test', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', name: 'doGenerate', inputMessages: [ { content: 'Run the test tool', role: 'user' }, @@ -1268,7 +1286,7 @@ describe('Plugin', () => { content: '', role: 'assistant', tool_calls: [{ - tool_id: 'call-1', + tool_id: toolCallId, name: 'testTool', arguments: {}, type: 'function', @@ -1277,18 +1295,18 @@ describe('Plugin', () => { { content: scenario.expectedContent, role: 'tool', - tool_id: 'call-1', + tool_id: toolCallId, }, ], - outputMessages: [{ content: 'done', role: 'assistant' }], - metrics: { input_tokens: 1, output_tokens: 1, total_tokens: 2 }, + outputMessages: [MOCK_OBJECT], + metrics: { input_tokens: MOCK_NUMBER, output_tokens: MOCK_NUMBER, total_tokens: MOCK_NUMBER }, tags: { ml_app: 'test', integration: 'ai' }, }) } const scenarios = semifies(realVersion, '<5.0.0') ? LEGACY_TOOL_RESULT_SCENARIOS - : TOOL_RESULT_SCENARIOS + : TOOL_RESULT_SCENARIOS.filter(({ openaiRange }) => !openaiRange || semifies(openaiVersion, openaiRange)) for (const scenario of scenarios) { it(`formats ${scenario.name} produced by the AI SDK`, async () => { await assertToolResult(scenario) diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index 71980ea690..0478572385 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -10,8 +10,8 @@ const { MOCK_STRING, useLlmObs, MOCK_NUMBER, + MOCK_OBJECT, } = require('../../util') -const { createToolResultModelV3 } = require('./tool-result-model') /** * @param {(version: string, openaiVersion: string) => void} callback @@ -936,19 +936,29 @@ describe('Plugin', () => { }) describe('tool result formatting', () => { - withVersions('ai', 'ai', '>=7.0.0', version => { + withAiSdkOpenAiVersions((version, openaiVersion) => { let ai + let openai beforeEach(() => { ai = require(`../../../../../../versions/ai@${version}`).get() + + const OpenAI = require(`../../../../../../versions/@ai-sdk/openai@${openaiVersion}`).get() + openai = OpenAI.createOpenAI({ + baseURL: 'http://127.0.0.1:9126/vcr/openai', + compatibility: 'strict', + }) }) it('formats file result parts produced by the AI SDK', async () => { - await ai.generateText({ - model: createToolResultModelV3(), + const result = await ai.generateText({ + // Chat Completions serializes every tool result as text. The Responses API validates file contents before + // the instrumented request can capture their formatted value. + model: openai.chat('gpt-4o-mini'), prompt: 'Run the test tool', tools: { testTool: ai.tool({ + description: 'Run the test tool and return its result', inputSchema: ai.jsonSchema({ type: 'object', properties: {}, @@ -985,7 +995,13 @@ describe('Plugin', () => { }), }, stopWhen: ai.stepCountIs(2), + providerOptions: { + openai: { + store: false, + }, + }, }) + const toolCallId = result.steps[0].toolCalls[0].toolCallId const { apmSpans, llmobsSpans } = await getEvents(6) let finalModelSpan @@ -1006,15 +1022,15 @@ describe('Plugin', () => { span: finalModelApmSpan, parentId: finalStepSpan.span_id, spanKind: 'llm', - modelName: 'test', - modelProvider: 'test', + modelName: 'gpt-4o-mini', + modelProvider: 'openai', name: 'languageModelCall', inputMessages: [ { content: 'Run the test tool', role: 'user' }, { role: 'assistant', tool_calls: [{ - tool_id: 'call-1', + tool_id: toolCallId, name: 'testTool', arguments: {}, type: 'function', @@ -1023,12 +1039,13 @@ describe('Plugin', () => { { content: 'before[Image][File][File][Image][Custom Content]after', role: 'tool', - tool_id: 'call-1', + tool_id: toolCallId, }, ], - outputMessages: [{ content: 'done', role: 'assistant' }], + outputMessages: [MOCK_OBJECT], toolDefinitions: [{ name: 'testTool', + description: 'Run the test tool and return its result', schema: { type: 'object', properties: {}, @@ -1037,10 +1054,10 @@ describe('Plugin', () => { }], metadata: {}, metrics: { - input_tokens: 1, + input_tokens: MOCK_NUMBER, cache_write_input_tokens: 0, cache_read_input_tokens: 0, - output_tokens: 1, + output_tokens: MOCK_NUMBER, reasoning_output_tokens: 0, }, tags: { ml_app: 'test', integration: 'ai' }, diff --git a/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js b/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js deleted file mode 100644 index 35cfa0dea9..0000000000 --- a/packages/dd-trace/test/llmobs/plugins/ai/tool-result-model.js +++ /dev/null @@ -1,141 +0,0 @@ -'use strict' - -/** - * @typedef {{ - * specificationVersion: 'v1' | 'v2' | 'v3', - * provider: string, - * modelId: string, - * defaultObjectGenerationMode: undefined, - * supportedUrls: Record, - * doGenerate: () => Promise> - * }} ToolResultModel - */ - -/** - * @param {'v1' | 'v2' | 'v3'} specificationVersion - * @param {Array>} responses - * @returns {ToolResultModel} - */ -function createModel (specificationVersion, responses) { - let responseIndex = 0 - - return { - specificationVersion, - provider: 'test', - modelId: 'test', - defaultObjectGenerationMode: undefined, - supportedUrls: {}, - async doGenerate () { - return responses[responseIndex++] - }, - } -} - -function createToolResultModelV1 () { - const usage = { - promptTokens: 1, - completionTokens: 1, - } - const rawCall = { - rawPrompt: undefined, - rawSettings: {}, - } - - return createModel('v1', [ - { - toolCalls: [{ - toolCallType: 'function', - toolCallId: 'call-1', - toolName: '0', - args: '{}', - }], - finishReason: 'tool-calls', - usage, - rawCall, - warnings: [], - }, - { - text: 'done', - finishReason: 'stop', - usage, - rawCall, - warnings: [], - }, - ]) -} - -function createToolResultModelV2 () { - const usage = { - inputTokens: 1, - outputTokens: 1, - totalTokens: 2, - } - - return createModel('v2', [ - { - content: [{ - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'testTool', - input: '{}', - }], - finishReason: 'tool-calls', - usage, - warnings: [], - }, - { - content: [{ type: 'text', text: 'done' }], - finishReason: 'stop', - usage, - warnings: [], - }, - ]) -} - -function createToolResultModelV3 () { - const usage = { - inputTokens: { - total: 1, - noCache: 1, - cacheRead: 0, - cacheWrite: 0, - }, - outputTokens: { - total: 1, - text: 1, - reasoning: 0, - }, - } - - return createModel('v3', [ - { - content: [{ - type: 'tool-call', - toolCallId: 'call-1', - toolName: 'testTool', - input: '{}', - }], - finishReason: { - unified: 'tool-calls', - raw: 'tool_calls', - }, - usage, - warnings: [], - }, - { - content: [{ type: 'text', text: 'done' }], - finishReason: { - unified: 'stop', - raw: 'stop', - }, - usage, - warnings: [], - }, - ]) -} - -module.exports = { - createToolResultModelV1, - createToolResultModelV2, - createToolResultModelV3, -} From 5a593daa69639e9cc46475a3f373ad3bfc1d5eca Mon Sep 17 00:00:00 2001 From: Sam Brenner Date: Wed, 22 Jul 2026 17:02:23 -0400 Subject: [PATCH 6/8] update from bad rebase --- packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js index 08ae295376..dd3b400ca2 100644 --- a/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js +++ b/packages/dd-trace/test/llmobs/plugins/ai/index.v7.spec.js @@ -948,7 +948,7 @@ describe('Plugin', () => { }) describe('tool result formatting', () => { - withAiSdkOpenAiVersions((version, openaiVersion) => { + withAiSdkOpenAiVersions((version, _, openaiVersion) => { let ai let openai From fa10797d8e1113d1ab153a8f7a392282858a71c4 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 23 Jul 2026 16:53:00 +0200 Subject: [PATCH 7/8] Apply suggestion from @BridgeAR --- packages/dd-trace/src/llmobs/plugins/ai/util.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/util.js b/packages/dd-trace/src/llmobs/plugins/ai/util.js index a70959512b..df7c29b544 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/util.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/util.js @@ -404,9 +404,10 @@ function getToolCallResultContent (content) { return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT } return UNPARSABLE_TOOL_RESULT - } else if (result !== undefined) { - return typeof result === 'string' ? result : stringifyToolCallResult(result) } + if (result !== undefined) { + return typeof result === 'string' ? result : stringifyToolCallResult(result) +} return UNSUPPORTED_TOOL_RESULT } catch { From 8c60bfbb6029a996b260398000612f5398ee6008 Mon Sep 17 00:00:00 2001 From: Ruben Bridgewater Date: Thu, 23 Jul 2026 17:17:41 +0200 Subject: [PATCH 8/8] Apply suggestion from @BridgeAR --- packages/dd-trace/src/llmobs/plugins/ai/util.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/dd-trace/src/llmobs/plugins/ai/util.js b/packages/dd-trace/src/llmobs/plugins/ai/util.js index df7c29b544..8b629dab50 100644 --- a/packages/dd-trace/src/llmobs/plugins/ai/util.js +++ b/packages/dd-trace/src/llmobs/plugins/ai/util.js @@ -407,7 +407,7 @@ function getToolCallResultContent (content) { } if (result !== undefined) { return typeof result === 'string' ? result : stringifyToolCallResult(result) -} + } return UNSUPPORTED_TOOL_RESULT } catch {