Skip to content

Commit ab883e7

Browse files
RulaKhaledRulaKhaled
authored
fix(core): Improve message truncation for multimodal content and normalize streaming span names (#19500)
This PR introduces some fixes across AI integrations. 1. Normalize streaming span description Removes the stream-response suffix from span descriptions across OpenAI, Anthropic, and Google GenAI integrations. Streaming and non-streaming spans now use the same naming format (e.g., chat gpt-4 instead of chat gpt-4 stream-response), making grouping and filtering more consistent. 2. Expand multimodal media stripping Adds detection and redaction of three additional inline media formats in truncateGenAiMessages: - OpenAI vision format with nested image_url objects ({ image_url: { url: "data:..." } }) - OpenAI input_audio parts ({ type: "input_audio", input_audio: { data: "..." } }) - OpenAI file parts with inline data ({ type: "file", file: { file_data: "..." } }) 4. Rename redaction placeholder Changes the placeholder for stripped binary data from [Filtered] to [Blob substitute] to better communicate what happened. 5. Fix off-by-one in truncation byte accounting Subtracts the 2-byte JSON array wrapper ([ and ]) from the effective max bytes budget, and fixes a boundary condition in truncateTextByBytes to avoid strings that land exactly on the byte limit. Closes #19496 #19479
1 parent d245097 commit ab883e7

File tree

13 files changed

+640
-125
lines changed

13 files changed

+640
-125
lines changed

‎dev-packages/node-integration-tests/suites/tracing/anthropic/test.ts‎

Lines changed: 14 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -266,7 +266,7 @@ describe('Anthropic integration', () => {
266266
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.chat',
267267
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.anthropic',
268268
}),
269-
description: 'chat claude-3-haiku-20240307 stream-response',
269+
description: 'chat claude-3-haiku-20240307',
270270
op: 'gen_ai.chat',
271271
origin: 'auto.ai.anthropic',
272272
status: 'ok',
@@ -296,7 +296,7 @@ describe('Anthropic integration', () => {
296296
[GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'claude-3-haiku-20240307',
297297
[GEN_AI_REQUEST_STREAM_ATTRIBUTE]: true,
298298
}),
299-
description: 'chat claude-3-haiku-20240307 stream-response',
299+
description: 'chat claude-3-haiku-20240307',
300300
op: 'gen_ai.chat',
301301
origin: 'auto.ai.anthropic',
302302
status: 'ok',
@@ -401,7 +401,7 @@ describe('Anthropic integration', () => {
401401
spans: expect.arrayContaining([
402402
// messages.create with stream: true
403403
expect.objectContaining({
404-
description: 'chat claude-3-haiku-20240307 stream-response',
404+
description: 'chat claude-3-haiku-20240307',
405405
op: 'gen_ai.chat',
406406
data: expect.objectContaining({
407407
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
@@ -419,7 +419,7 @@ describe('Anthropic integration', () => {
419419
}),
420420
// messages.stream
421421
expect.objectContaining({
422-
description: 'chat claude-3-haiku-20240307 stream-response',
422+
description: 'chat claude-3-haiku-20240307',
423423
op: 'gen_ai.chat',
424424
data: expect.objectContaining({
425425
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
@@ -435,7 +435,7 @@ describe('Anthropic integration', () => {
435435
}),
436436
// messages.stream with redundant stream: true param
437437
expect.objectContaining({
438-
description: 'chat claude-3-haiku-20240307 stream-response',
438+
description: 'chat claude-3-haiku-20240307',
439439
op: 'gen_ai.chat',
440440
data: expect.objectContaining({
441441
[GEN_AI_SYSTEM_ATTRIBUTE]: 'anthropic',
@@ -457,7 +457,7 @@ describe('Anthropic integration', () => {
457457
transaction: 'main',
458458
spans: expect.arrayContaining([
459459
expect.objectContaining({
460-
description: 'chat claude-3-haiku-20240307 stream-response',
460+
description: 'chat claude-3-haiku-20240307',
461461
op: 'gen_ai.chat',
462462
data: expect.objectContaining({
463463
[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true,
@@ -466,15 +466,15 @@ describe('Anthropic integration', () => {
466466
}),
467467
}),
468468
expect.objectContaining({
469-
description: 'chat claude-3-haiku-20240307 stream-response',
469+
description: 'chat claude-3-haiku-20240307',
470470
op: 'gen_ai.chat',
471471
data: expect.objectContaining({
472472
[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true,
473473
[GEN_AI_RESPONSE_TEXT_ATTRIBUTE]: 'Hello from stream!',
474474
}),
475475
}),
476476
expect.objectContaining({
477-
description: 'chat claude-3-haiku-20240307 stream-response',
477+
description: 'chat claude-3-haiku-20240307',
478478
op: 'gen_ai.chat',
479479
data: expect.objectContaining({
480480
[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true,
@@ -536,7 +536,7 @@ describe('Anthropic integration', () => {
536536
transaction: {
537537
spans: expect.arrayContaining([
538538
expect.objectContaining({
539-
description: expect.stringContaining('stream-response'),
539+
description: 'chat claude-3-haiku-20240307',
540540
op: 'gen_ai.chat',
541541
data: expect.objectContaining({
542542
[GEN_AI_REQUEST_AVAILABLE_TOOLS_ATTRIBUTE]: EXPECTED_TOOLS_JSON,
@@ -557,7 +557,7 @@ describe('Anthropic integration', () => {
557557
spans: expect.arrayContaining([
558558
// Error with messages.create on stream initialization
559559
expect.objectContaining({
560-
description: 'chat error-stream-init stream-response',
560+
description: 'chat error-stream-init',
561561
op: 'gen_ai.chat',
562562
status: 'internal_error', // Actual status coming from the instrumentation
563563
data: expect.objectContaining({
@@ -567,7 +567,7 @@ describe('Anthropic integration', () => {
567567
}),
568568
// Error with messages.stream on stream initialization
569569
expect.objectContaining({
570-
description: 'chat error-stream-init stream-response',
570+
description: 'chat error-stream-init',
571571
op: 'gen_ai.chat',
572572
status: 'internal_error', // Actual status coming from the instrumentation
573573
data: expect.objectContaining({
@@ -577,7 +577,7 @@ describe('Anthropic integration', () => {
577577
// Error midway with messages.create on streaming - note: The stream is started successfully
578578
// so we get a successful span with the content that was streamed before the error
579579
expect.objectContaining({
580-
description: 'chat error-stream-midway stream-response',
580+
description: 'chat error-stream-midway',
581581
op: 'gen_ai.chat',
582582
status: 'ok',
583583
data: expect.objectContaining({
@@ -589,7 +589,7 @@ describe('Anthropic integration', () => {
589589
}),
590590
// Error midway with messages.stream - same behavior, we get a span with the streamed data
591591
expect.objectContaining({
592-
description: 'chat error-stream-midway stream-response',
592+
description: 'chat error-stream-midway',
593593
op: 'gen_ai.chat',
594594
status: 'ok',
595595
data: expect.objectContaining({
@@ -731,7 +731,7 @@ describe('Anthropic integration', () => {
731731
source: {
732732
type: 'base64',
733733
media_type: 'image/png',
734-
data: '[Filtered]',
734+
data: '[Blob substitute]',
735735
},
736736
},
737737
],

‎dev-packages/node-integration-tests/suites/tracing/google-genai/test.ts‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -272,7 +272,7 @@ describe('Google GenAI integration', () => {
272272
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 10,
273273
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22,
274274
}),
275-
description: 'generate_content gemini-2.0-flash-001 stream-response',
275+
description: 'generate_content gemini-2.0-flash-001',
276276
op: 'gen_ai.generate_content',
277277
origin: 'auto.ai.google_genai',
278278
status: 'ok',
@@ -327,7 +327,7 @@ describe('Google GenAI integration', () => {
327327
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12,
328328
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22,
329329
}),
330-
description: 'generate_content gemini-1.5-flash stream-response',
330+
description: 'generate_content gemini-1.5-flash',
331331
op: 'gen_ai.generate_content',
332332
origin: 'auto.ai.google_genai',
333333
status: 'ok',
@@ -361,7 +361,7 @@ describe('Google GenAI integration', () => {
361361
[GEN_AI_RESPONSE_ID_ATTRIBUTE]: 'mock-response-streaming-id',
362362
[GEN_AI_RESPONSE_MODEL_ATTRIBUTE]: 'gemini-1.5-pro',
363363
}),
364-
description: 'chat gemini-1.5-pro stream-response',
364+
description: 'chat gemini-1.5-pro',
365365
op: 'gen_ai.chat',
366366
origin: 'auto.ai.google_genai',
367367
status: 'ok',
@@ -373,7 +373,7 @@ describe('Google GenAI integration', () => {
373373
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content',
374374
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai',
375375
}),
376-
description: 'generate_content blocked-model stream-response',
376+
description: 'generate_content blocked-model',
377377
op: 'gen_ai.generate_content',
378378
origin: 'auto.ai.google_genai',
379379
status: 'internal_error',
@@ -385,7 +385,7 @@ describe('Google GenAI integration', () => {
385385
[SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content',
386386
[SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.ai.google_genai',
387387
}),
388-
description: 'generate_content error-model stream-response',
388+
description: 'generate_content error-model',
389389
op: 'gen_ai.generate_content',
390390
origin: 'auto.ai.google_genai',
391391
status: 'internal_error',
@@ -416,7 +416,7 @@ describe('Google GenAI integration', () => {
416416
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12,
417417
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22,
418418
}),
419-
description: 'generate_content gemini-1.5-flash stream-response',
419+
description: 'generate_content gemini-1.5-flash',
420420
op: 'gen_ai.generate_content',
421421
origin: 'auto.ai.google_genai',
422422
status: 'ok',
@@ -455,7 +455,7 @@ describe('Google GenAI integration', () => {
455455
[GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 12,
456456
[GEN_AI_USAGE_TOTAL_TOKENS_ATTRIBUTE]: 22,
457457
}),
458-
description: 'chat gemini-1.5-pro stream-response',
458+
description: 'chat gemini-1.5-pro',
459459
op: 'gen_ai.chat',
460460
origin: 'auto.ai.google_genai',
461461
status: 'ok',
@@ -472,7 +472,7 @@ describe('Google GenAI integration', () => {
472472
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true
473473
[GEN_AI_RESPONSE_STREAMING_ATTRIBUTE]: true,
474474
}),
475-
description: 'generate_content blocked-model stream-response',
475+
description: 'generate_content blocked-model',
476476
op: 'gen_ai.generate_content',
477477
origin: 'auto.ai.google_genai',
478478
status: 'internal_error',
@@ -488,7 +488,7 @@ describe('Google GenAI integration', () => {
488488
[GEN_AI_REQUEST_TEMPERATURE_ATTRIBUTE]: 0.7,
489489
[GEN_AI_INPUT_MESSAGES_ATTRIBUTE]: expect.any(String), // Should include contents when recordInputs: true
490490
}),
491-
description: 'generate_content error-model stream-response',
491+
description: 'generate_content error-model',
492492
op: 'gen_ai.generate_content',
493493
origin: 'auto.ai.google_genai',
494494
status: 'internal_error',

‎dev-packages/node-integration-tests/suites/tracing/openai/openai-tool-calls/test.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -133,7 +133,7 @@ describe('OpenAI Tool Calls integration', () => {
133133
[OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE]: 25,
134134
[OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE]: 15,
135135
},
136-
description: 'chat gpt-4 stream-response',
136+
description: 'chat gpt-4',
137137
op: 'gen_ai.chat',
138138
origin: 'auto.ai.openai',
139139
status: 'ok',
@@ -187,7 +187,7 @@ describe('OpenAI Tool Calls integration', () => {
187187
[OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE]: 12,
188188
[OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE]: 8,
189189
},
190-
description: 'chat gpt-4 stream-response',
190+
description: 'chat gpt-4',
191191
op: 'gen_ai.chat',
192192
origin: 'auto.ai.openai',
193193
status: 'ok',
@@ -254,7 +254,7 @@ describe('OpenAI Tool Calls integration', () => {
254254
[OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE]: 25,
255255
[OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE]: 15,
256256
},
257-
description: 'chat gpt-4 stream-response',
257+
description: 'chat gpt-4',
258258
op: 'gen_ai.chat',
259259
origin: 'auto.ai.openai',
260260
status: 'ok',
@@ -314,7 +314,7 @@ describe('OpenAI Tool Calls integration', () => {
314314
[OPENAI_USAGE_COMPLETION_TOKENS_ATTRIBUTE]: 12,
315315
[OPENAI_USAGE_PROMPT_TOKENS_ATTRIBUTE]: 8,
316316
},
317-
description: 'chat gpt-4 stream-response',
317+
description: 'chat gpt-4',
318318
op: 'gen_ai.chat',
319319
origin: 'auto.ai.openai',
320320
status: 'ok',

‎dev-packages/node-integration-tests/suites/tracing/openai/scenario-vision.mjs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
import * as Sentry from '@sentry/node';
2+
import express from 'express';
3+
import OpenAI from 'openai';
4+
5+
function startMockServer() {
6+
const app = express();
7+
app.use(express.json({ limit: '10mb' }));
8+
9+
app.post('/openai/chat/completions', (req, res) => {
10+
res.send({
11+
id: 'chatcmpl-vision-123',
12+
object: 'chat.completion',
13+
created: 1677652288,
14+
model: req.body.model,
15+
choices: [
16+
{
17+
index: 0,
18+
message: {
19+
role: 'assistant',
20+
content: 'I see a red square in the image.',
21+
},
22+
finish_reason: 'stop',
23+
},
24+
],
25+
usage: {
26+
prompt_tokens: 50,
27+
completion_tokens: 10,
28+
total_tokens: 60,
29+
},
30+
});
31+
});
32+
33+
return new Promise(resolve => {
34+
const server = app.listen(0, () => {
35+
resolve(server);
36+
});
37+
});
38+
}
39+
40+
// Small 10x10 red PNG image encoded as base64
41+
const RED_PNG_BASE64 =
42+
'iVBORw0KGgoAAAANSUhEUgAAAAoAAAAKCAYAAACNMs+9AAAAFUlEQVR42mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg==';
43+
44+
async function run() {
45+
const server = await startMockServer();
46+
47+
await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
48+
const client = new OpenAI({
49+
baseURL: `http://localhost:${server.address().port}/openai`,
50+
apiKey: 'mock-api-key',
51+
});
52+
53+
// Vision request with inline base64 image
54+
await client.chat.completions.create({
55+
model: 'gpt-4o',
56+
messages: [
57+
{
58+
role: 'user',
59+
content: [
60+
{ type: 'text', text: 'What is in this image?' },
61+
{
62+
type: 'image_url',
63+
image_url: {
64+
url: `data:image/png;base64,${RED_PNG_BASE64}`,
65+
},
66+
},
67+
],
68+
},
69+
],
70+
});
71+
72+
// Vision request with multiple images (one inline, one URL)
73+
await client.chat.completions.create({
74+
model: 'gpt-4o',
75+
messages: [
76+
{
77+
role: 'user',
78+
content: [
79+
{ type: 'text', text: 'Compare these images' },
80+
{
81+
type: 'image_url',
82+
image_url: {
83+
url: `data:image/png;base64,${RED_PNG_BASE64}`,
84+
},
85+
},
86+
{
87+
type: 'image_url',
88+
image_url: {
89+
url: 'https://example.com/image.png',
90+
},
91+
},
92+
],
93+
},
94+
],
95+
});
96+
});
97+
98+
server.close();
99+
}
100+
101+
run();

0 commit comments

Comments
 (0)