feat(core): Migrate Vercel AI event processor to span streaming (#20608)

Migrates the Vercel AI event processor so it also works in the span
streaming path via the `processSpan` hook. The event processor currently
serves three purposes:

**Attribute renaming.** The Vercel AI SDK emits attributes under `ai.*`
names that need to be renamed to OpenTelemetry semantic conventions
(`gen_ai.*`) and the `vercel.ai.*` namespace. This logic is
straightforward to port. It's extracted into a shared
`processVercelAiSpanAttributes` helper that is now called from both the
legacy event processor path (for transactions) and the new `processSpan`
hook (for streamed spans).

**Token accumulation on parent spans.** The event processor aggregates
token usage from child spans onto their parent `invoke_agent` spans.
This is a cross-span operation that fundamentally doesn't work in the
streaming model where spans are processed individually. The [span
streaming implementation
guide](https://develop.sentry.dev/sdk/telemetry/spans/implementation/)
explicitly lists this as a case that cannot be replaced. Since the plan
is for parent-level token accumulation to go away regardless, we simply
drop it for the streaming path.

**Tool descriptions on execute_tool spans.** The event processor
iterates over all spans in a transaction to find
`gen_ai.request.available_tools` on `doGenerate` spans and applies the
matching description to sibling `execute_tool` spans. This cross-span
iteration doesn't work when spans are processed individually. Instead,
we parse and store tool descriptions in a map at `spanStart` time of the
`doGenerate` span. Since `execute_tool` spans are siblings of
`doGenerate` (both children of `invoke_agent`), we key the map by the
parent span ID so `execute_tool` spans can look up descriptions by their
own `parent_span_id`. This assumes a flat sibling hierarchy, which holds
for our test scenarios. If we encounter more complex cases down the
road, I think it's fine to address in a follow-up.

Closes #20377

Author: nicohrubec

dev-packages/node-integration-tests/suites/tracing/vercelai/scenario-error-in-tool.mjs

@@ -35,6 +35,8 @@ async function run() {
       prompt: 'What is the weather in San Francisco?',
     });
   });
+
+  await Sentry.flush(2000);
 }
 
 run();

dev-packages/node-integration-tests/suites/tracing/vercelai/scenario.mjs

@@ -75,6 +75,8 @@ async function run() {
       prompt: 'Where is the third span?',
     });
   });
+
+  await Sentry.flush(2000);
 }
 
 run();

…racing/vercelai/instrument-streaming.mjs …pan-streaming-v4/instrument-with-pii.mjsdev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs renamed to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming.mjs renamed to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-pii.mjs

@@ -8,4 +8,5 @@ Sentry.init({
   sendDefaultPii: true,
   transport: loggingTransport,
   traceLifecycle: 'stream',
+  integrations: [Sentry.vercelAIIntegration()],
 });

…instrument-streaming-with-truncation.mjs …eaming-v4/instrument-with-truncation.mjsdev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs renamed to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs dev-packages/node-integration-tests/suites/tracing/vercelai/instrument-streaming-with-truncation.mjs renamed to dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument-with-truncation.mjs

@@ -0,0 +1,11 @@
+import * as Sentry from '@sentry/node';
+import { loggingTransport } from '@sentry-internal/node-integration-tests';
+
+Sentry.init({
+  dsn: 'https://public@dsn.ingest.sentry.io/1337',
+  release: '1.0',
+  tracesSampleRate: 1.0,
+  transport: loggingTransport,
+  traceLifecycle: 'stream',
+  integrations: [Sentry.vercelAIIntegration()],
+});

dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/instrument.mjs

@@ -0,0 +1,46 @@
+import * as Sentry from '@sentry/node';
+import { generateText } from 'ai';
+import { MockLanguageModelV1 } from 'ai/test';
+import { z } from 'zod';
+
+async function run() {
+  Sentry.setTag('test-tag', 'test-value');
+
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    try {
+      await generateText({
+        model: new MockLanguageModelV1({
+          doGenerate: async () => ({
+            rawCall: { rawPrompt: null, rawSettings: {} },
+            finishReason: 'tool-calls',
+            usage: { promptTokens: 15, completionTokens: 25 },
+            text: 'Tool call completed!',
+            toolCalls: [
+              {
+                toolCallType: 'function',
+                toolCallId: 'call-1',
+                toolName: 'getWeather',
+                args: '{ "location": "San Francisco" }',
+              },
+            ],
+          }),
+        }),
+        tools: {
+          getWeather: {
+            parameters: z.object({ location: z.string() }),
+            execute: async () => {
+              throw new Error('Error in tool');
+            },
+          },
+        },
+        prompt: 'What is the weather in San Francisco?',
+      });
+    } catch {
+      // Expected error - we want the spans to still be flushed
+    }
+  });
+
+  await Sentry.flush(2000);
+}
+
+run();

dev-packages/node-integration-tests/suites/tracing/vercelai/span-streaming-v4/scenario-error-in-tool.mjs

@@ -0,0 +1,82 @@
+import * as Sentry from '@sentry/node';
+import { generateText } from 'ai';
+import { MockLanguageModelV1 } from 'ai/test';
+import { z } from 'zod';
+
+async function run() {
+  await Sentry.startSpan({ op: 'function', name: 'main' }, async () => {
+    await generateText({
+      model: new MockLanguageModelV1({
+        doGenerate: async () => ({
+          rawCall: { rawPrompt: null, rawSettings: {} },
+          finishReason: 'stop',
+          usage: { promptTokens: 10, completionTokens: 20 },
+          text: 'First span here!',
+        }),
+      }),
+      prompt: 'Where is the first span?',
+    });
+
+    // This span should have input and output prompts attached because telemetry is explicitly enabled.
+    await generateText({
+      experimental_telemetry: { isEnabled: true },
+      model: new MockLanguageModelV1({
+        doGenerate: async () => ({
+          rawCall: { rawPrompt: null, rawSettings: {} },
+          finishReason: 'stop',
+          usage: { promptTokens: 10, completionTokens: 20 },
+          text: 'Second span here!',
+        }),
+      }),
+      prompt: 'Where is the second span?',
+    });
+
+    // This span should include tool calls and tool results
+    await generateText({
+      model: new MockLanguageModelV1({
+        doGenerate: async () => ({
+          rawCall: { rawPrompt: null, rawSettings: {} },
+          finishReason: 'tool-calls',
+          usage: { promptTokens: 15, completionTokens: 25 },
+          text: 'Tool call completed!',
+          toolCalls: [
+            {
+              toolCallType: 'function',
+              toolCallId: 'call-1',
+              toolName: 'getWeather',
+              args: '{ "location": "San Francisco" }',
+            },
+          ],
+        }),
+      }),
+      tools: {
+        getWeather: {
+          description: 'Get the current weather for a location',
+          parameters: z.object({ location: z.string() }),
+          execute: async args => {
+            return `Weather in ${args.location}: Sunny, 72°F`;
+          },
+        },
+      },
+      prompt: 'What is the weather in San Francisco?',
+    });
+
+    // This span should not be captured because we've disabled telemetry
+    await generateText({
+      experimental_telemetry: { isEnabled: false },
+      model: new MockLanguageModelV1({
+        doGenerate: async () => ({
+          rawCall: { rawPrompt: null, rawSettings: {} },
+          finishReason: 'stop',
+          usage: { promptTokens: 10, completionTokens: 20 },
+          text: 'Third span here!',
+        }),
+      }),
+      prompt: 'Where is the third span?',
+    });
+  });
+
+  await Sentry.flush(2000);
+}
+
+run();