Skip to content

Commit aaa217b

Browse files
BridgeARBridgeAR
committed
fix(llmobs): format AI SDK rich tool results
AI SDK tool outputs can contain multipart content and error variants. Preserve text, replace file and image payloads with safe summaries, and fall back without throwing on malformed values. Fixes: #9395
1 parent 9018e7c commit aaa217b

2 files changed

Lines changed: 244 additions & 22 deletions

File tree

‎packages/dd-trace/src/llmobs/plugins/ai/util.js‎

Lines changed: 71 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ const MODEL_METADATA_KEYS = new Set([
1313
const VERCEL_AI_TELEMETRY_METADATA_PREFIX = 'ai.telemetry.metadata.'
1414
const VERCEL_AI_MODEL_METADATA_PREFIX = 'gen_ai.request.'
1515
const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
16+
const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]'
17+
const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]'
1618

1719
/**
1820
* @typedef {import('../../../opentracing/span')} Span
@@ -27,7 +29,8 @@ const VERCEL_AI_GENERATION_METADATA_PREFIX = 'ai.settings.'
2729
/**
2830
* @typedef {{
2931
* type: string,
30-
* value?: unknown
32+
* value?: unknown,
33+
* reason?: unknown
3134
* }} ToolCallOutput
3235
*
3336
* @typedef {{ output?: ToolCallOutput, result?: unknown } & Record<string, unknown>} ToolCallResultContent
@@ -304,32 +307,78 @@ function getToolNameFromTags (tags) {
304307
}
305308

306309
/**
307-
* Get the content of a tool call result.
308-
* Version 5 of the ai sdk sets this tag as `content.output`, with a `
309-
* @param {{ output?: { type: string, value: unknown }, result?: unknown }} content
310+
* @param {unknown} value
310311
* @returns {string}
311312
*/
312-
function getToolCallResultContent (content) {
313-
const { output, result } = content
314-
if (output) {
315-
if (output.type === 'text') {
316-
return output.value
317-
} else if (output.type === 'json') {
318-
return JSON.stringify(output.value)
319-
}
320-
return '[Unparsable Tool Result]'
321-
} else if (result) {
322-
if (typeof result === 'string') {
323-
return result
313+
function stringifyToolCallResult (value) {
314+
return JSON.stringify(value) ?? UNPARSABLE_TOOL_RESULT
315+
}
316+
317+
/**
318+
* @param {unknown} value
319+
* @returns {string}
320+
*/
321+
function formatToolCallContent (value) {
322+
if (!Array.isArray(value)) return UNPARSABLE_TOOL_RESULT
323+
324+
let result = ''
325+
for (const part of value) {
326+
if (typeof part !== 'object' || part === null) return UNPARSABLE_TOOL_RESULT
327+
328+
const { type } = part
329+
if (type === 'text') {
330+
if (typeof part.text !== 'string') return UNPARSABLE_TOOL_RESULT
331+
result += part.text
332+
} else if (type === 'media') {
333+
const { mediaType } = part
334+
if (typeof mediaType !== 'string') return UNPARSABLE_TOOL_RESULT
335+
result += mediaType.startsWith('image/') ? '[Image]' : '[File]'
336+
} else if (type === 'file-data' || type === 'file-url' || type === 'file-id') {
337+
result += '[File]'
338+
} else if (type === 'image-data' || type === 'image-url' || type === 'image-file-id') {
339+
result += '[Image]'
340+
} else if (type === 'custom') {
341+
result += '[Custom Content]'
342+
} else {
343+
return UNPARSABLE_TOOL_RESULT
324344
}
345+
}
346+
347+
return result
348+
}
325349

326-
try {
327-
return JSON.stringify(result)
328-
} catch {
329-
return '[Unparsable Tool Result]'
350+
/**
351+
* @param {ToolCallResultContent | null | undefined} content
352+
* @returns {string}
353+
*/
354+
function getToolCallResultContent (content) {
355+
try {
356+
if (typeof content !== 'object' || content === null) return UNPARSABLE_TOOL_RESULT
357+
358+
const { output, result } = content
359+
if (output !== undefined) {
360+
if (typeof output !== 'object' || output === null) return UNPARSABLE_TOOL_RESULT
361+
362+
const { type, value } = output
363+
if (type === 'text' || type === 'error-text') {
364+
return typeof value === 'string' ? value : UNPARSABLE_TOOL_RESULT
365+
} else if (type === 'json' || type === 'error-json') {
366+
return stringifyToolCallResult(value)
367+
} else if (type === 'content') {
368+
return formatToolCallContent(value)
369+
} else if (type === 'execution-denied') {
370+
const { reason } = output
371+
if (reason === undefined) return '[Tool Execution Denied]'
372+
return typeof reason === 'string' ? reason : UNPARSABLE_TOOL_RESULT
373+
}
374+
return UNPARSABLE_TOOL_RESULT
375+
} else if (result !== undefined) {
376+
return typeof result === 'string' ? result : stringifyToolCallResult(result)
330377
}
331-
} else {
332-
return '[Unsupported Tool Result]'
378+
379+
return UNSUPPORTED_TOOL_RESULT
380+
} catch {
381+
return UNPARSABLE_TOOL_RESULT
333382
}
334383
}
335384

‎packages/dd-trace/test/llmobs/plugins/ai/util.spec.js‎

Lines changed: 173 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,173 @@
1+
'use strict'
2+
3+
const assert = require('node:assert/strict')
4+
5+
const { getToolCallResultContent } = require('../../../../src/llmobs/plugins/ai/util')
6+
7+
const UNPARSABLE_TOOL_RESULT = '[Unparsable Tool Result]'
8+
const UNSUPPORTED_TOOL_RESULT = '[Unsupported Tool Result]'
9+
10+
describe('AI SDK LLMObs utilities', () => {
11+
describe('getToolCallResultContent', () => {
12+
it('formats text output variants', () => {
13+
assert.strictEqual(getToolCallResultContent({ output: { type: 'text', value: 'result' } }), 'result')
14+
assert.strictEqual(getToolCallResultContent({ output: { type: 'error-text', value: 'failure' } }), 'failure')
15+
})
16+
17+
it('formats JSON output variants', () => {
18+
const value = { answer: 42 }
19+
20+
assert.strictEqual(
21+
getToolCallResultContent({ output: { type: 'json', value } }),
22+
JSON.stringify(value)
23+
)
24+
assert.strictEqual(
25+
getToolCallResultContent({ output: { type: 'error-json', value } }),
26+
JSON.stringify(value)
27+
)
28+
})
29+
30+
it('formats all legacy result values', () => {
31+
assert.strictEqual(getToolCallResultContent({ result: '' }), '')
32+
assert.strictEqual(getToolCallResultContent({ result: 'result' }), 'result')
33+
assert.strictEqual(getToolCallResultContent({ result: 0 }), '0')
34+
assert.strictEqual(getToolCallResultContent({ result: false }), 'false')
35+
assert.strictEqual(getToolCallResultContent({ result: null }), 'null')
36+
assert.strictEqual(getToolCallResultContent({ result: { answer: 42 } }), '{"answer":42}')
37+
})
38+
39+
it('distinguishes absent legacy results', () => {
40+
assert.strictEqual(getToolCallResultContent({}), UNSUPPORTED_TOOL_RESULT)
41+
assert.strictEqual(getToolCallResultContent({ result: undefined }), UNSUPPORTED_TOOL_RESULT)
42+
})
43+
44+
it('concatenates text and summarizes rich content without serializing payloads', () => {
45+
const content = [
46+
{ type: 'text', text: 'before' },
47+
{ type: 'media', mediaType: 'image/png', data: 'legacy-image-data' },
48+
{ type: 'media', mediaType: 'application/pdf', data: 'legacy-file-data' },
49+
{ type: 'file-data', mediaType: 'application/pdf', data: 'file-data' },
50+
{ type: 'file-url', url: 'https://example.com/private-file' },
51+
{ type: 'file-id', fileId: 'private-file-id' },
52+
{ type: 'image-data', mediaType: 'image/png', data: 'image-data' },
53+
{ type: 'image-url', url: 'https://example.com/private-image' },
54+
{ type: 'image-file-id', fileId: 'private-image-id' },
55+
{ type: 'custom', providerOptions: { secret: 'provider-data' } },
56+
{ type: 'text', text: 'after' },
57+
]
58+
59+
assert.strictEqual(
60+
getToolCallResultContent({ output: { type: 'content', value: content } }),
61+
'before[Image][File][File][File][File][Image][Image][Image][Custom Content]after'
62+
)
63+
})
64+
65+
it('formats empty multipart content', () => {
66+
assert.strictEqual(getToolCallResultContent({ output: { type: 'content', value: [] } }), '')
67+
})
68+
69+
it('formats denied executions', () => {
70+
assert.strictEqual(
71+
getToolCallResultContent({ output: { type: 'execution-denied', reason: 'Approval rejected' } }),
72+
'Approval rejected'
73+
)
74+
assert.strictEqual(
75+
getToolCallResultContent({ output: { type: 'execution-denied' } }),
76+
'[Tool Execution Denied]'
77+
)
78+
assert.strictEqual(
79+
getToolCallResultContent({ output: { type: 'execution-denied', reason: '' } }),
80+
''
81+
)
82+
})
83+
84+
it('rejects malformed output variants', () => {
85+
const malformedOutputs = [
86+
null,
87+
'text',
88+
{ type: 'text' },
89+
{ type: 'text', value: 42 },
90+
{ type: 'error-text', value: {} },
91+
{ type: 'json', value: undefined },
92+
{ type: 'json', value: Symbol('result') },
93+
{ type: 'error-json', value: () => {} },
94+
{ type: 'content', value: {} },
95+
{ type: 'execution-denied', reason: {} },
96+
{ type: 'unknown', value: 'result' },
97+
]
98+
99+
for (const output of malformedOutputs) {
100+
assert.strictEqual(getToolCallResultContent({ output }), UNPARSABLE_TOOL_RESULT)
101+
}
102+
})
103+
104+
it('rejects malformed multipart content', () => {
105+
const sparseContent = new Array(1)
106+
const malformedValues = [
107+
[null],
108+
['text'],
109+
[{ type: 'text' }],
110+
[{ type: 'text', text: 42 }],
111+
[{ type: 'media' }],
112+
[{ type: 'unknown' }],
113+
[{ type: Symbol('text'), text: 'result' }],
114+
[{ type: 'text', text: 'before' }, { type: 'unknown' }],
115+
sparseContent,
116+
]
117+
118+
for (const value of malformedValues) {
119+
assert.strictEqual(
120+
getToolCallResultContent({ output: { type: 'content', value } }),
121+
UNPARSABLE_TOOL_RESULT
122+
)
123+
}
124+
})
125+
126+
it('does not throw for circular values', () => {
127+
const circular = {}
128+
circular.self = circular
129+
130+
assert.strictEqual(
131+
getToolCallResultContent({ output: { type: 'json', value: circular } }),
132+
UNPARSABLE_TOOL_RESULT
133+
)
134+
assert.strictEqual(
135+
getToolCallResultContent({ output: { type: 'error-json', value: circular } }),
136+
UNPARSABLE_TOOL_RESULT
137+
)
138+
assert.strictEqual(getToolCallResultContent({ result: circular }), UNPARSABLE_TOOL_RESULT)
139+
assert.strictEqual(getToolCallResultContent({ result: 1n }), UNPARSABLE_TOOL_RESULT)
140+
})
141+
142+
it('does not throw for throwing external objects', () => {
143+
const throwingContent = new Proxy({}, {
144+
get () {
145+
throw new Error('unexpected content access')
146+
},
147+
})
148+
const throwingOutput = new Proxy({}, {
149+
get () {
150+
throw new Error('unexpected output access')
151+
},
152+
})
153+
const throwingParts = new Proxy([], {
154+
get () {
155+
throw new Error('unexpected content part access')
156+
},
157+
})
158+
159+
assert.strictEqual(getToolCallResultContent(throwingContent), UNPARSABLE_TOOL_RESULT)
160+
assert.strictEqual(getToolCallResultContent({ output: throwingOutput }), UNPARSABLE_TOOL_RESULT)
161+
assert.strictEqual(
162+
getToolCallResultContent({ output: { type: 'content', value: throwingParts } }),
163+
UNPARSABLE_TOOL_RESULT
164+
)
165+
})
166+
167+
it('rejects malformed content containers', () => {
168+
assert.strictEqual(getToolCallResultContent(), UNPARSABLE_TOOL_RESULT)
169+
assert.strictEqual(getToolCallResultContent(null), UNPARSABLE_TOOL_RESULT)
170+
assert.strictEqual(getToolCallResultContent('result'), UNPARSABLE_TOOL_RESULT)
171+
})
172+
})
173+
})

0 commit comments

Comments
 (0)