feat: add --stream-gen-ai-spans flag and v2 envelope span parsing (#150)

* feat: add option for JS runners to stream gen_ai spans

* feat: support envelpe v2 format so that we can capture streamed spans as well

* feat: enhance error status handling in checkErrorCaptured for v1 and v2 protocols

* fix(cli): consistent --not-stream-gen-ai-spans naming and conflict check

Addresses Cursor Bugbot review on #150:
- Rename --not-stream-genai-spans → --not-stream-gen-ai-spans for hyphenation
  parity with --stream-gen-ai-spans.
- Reject passing both flags together instead of silently dropping the enable
  flag value.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

Author: constantinius

src/cli.ts

@@ -43,6 +43,8 @@ Options:
   --sentry-javascript <path> Use local Sentry JavaScript SDK (link)
   --sentry-php <path>        Use local Sentry PHP SDK (core sentry/sentry-php)
   --sentry-laravel <path>    Use local Sentry Laravel SDK (composer path repository)
+  --stream-gen-ai-spans      Enable streamGenAiSpans in JS Sentry.init() (default: on)
+  --not-stream-gen-ai-spans  Disable streamGenAiSpans in JS Sentry.init()
   --help, -h                 Show this help message
 
 Examples:
@@ -63,6 +65,16 @@ Examples:
   npm run test setup -- --framework openai --sync --streaming
 `;
 
+function resolveStreamGenAiSpans(enable: boolean, disable: boolean): boolean {
+  if (enable && disable) {
+    console.error(
+      "Error: --stream-gen-ai-spans and --not-stream-gen-ai-spans are mutually exclusive",
+    );
+    process.exit(1);
+  }
+  return !disable;
+}
+
 function parseCliArgs() {
   const { values, positionals } = parseArgs({
     args: process.argv.slice(2),
@@ -84,6 +96,8 @@ function parseCliArgs() {
       "sentry-javascript": { type: "string" },
       "sentry-php": { type: "string" },
       "sentry-laravel": { type: "string" },
+      "stream-gen-ai-spans": { type: "boolean", default: false },
+      "not-stream-gen-ai-spans": { type: "boolean", default: false },
       help: { type: "boolean", short: "h", default: false },
     },
     allowPositionals: true,
@@ -181,6 +195,12 @@ function parseCliArgs() {
     sentryJavaScriptPath: values["sentry-javascript"],
     sentryPhpPath: values["sentry-php"],
     sentryLaravelPath: values["sentry-laravel"],
+    // Default ON. --not-stream-gen-ai-spans disables it; --stream-gen-ai-spans
+    // is the explicit form of the default. Passing both is a conflict.
+    streamGenAiSpans: resolveStreamGenAiSpans(
+      values["stream-gen-ai-spans"] === true,
+      values["not-stream-gen-ai-spans"] === true,
+    ),
     help: values.help,
   };
 }
@@ -214,6 +234,7 @@ async function main() {
     parallel: options.parallel,
     openReport: options.open,
     optionFilters: options.optionFilters,
+    streamGenAiSpans: options.streamGenAiSpans,
   });
 
   try {

src/orchestrator.ts

@@ -49,6 +49,7 @@ export class Orchestrator {
   private streamingFilter?: boolean;
   private blockingFilter?: boolean;
   private optionFilters?: Record<string, string>;
+  private streamGenAiSpans: boolean = true;
 
   constructor(
     options: {
@@ -61,6 +62,7 @@ export class Orchestrator {
       parallel?: number;
       openReport?: boolean;
       optionFilters?: Record<string, string>;
+      streamGenAiSpans?: boolean;
     } = {},
   ) {
     this.spanCollector = new SpanCollector();
@@ -79,6 +81,7 @@ export class Orchestrator {
     this.blockingFilter = options.blocking;
     this.optionFilters = options.optionFilters;
     this.openReport = options.openReport === true;
+    this.streamGenAiSpans = options.streamGenAiSpans !== false;
 
     // Set verbose on validator
     this.validator.setVerbose(this.verbose);
@@ -235,6 +238,7 @@ export class Orchestrator {
           resolvedOptions: firstRun.framework.resolvedOptions,
           timeoutMs: firstRun.testDefinition.timeoutMs ?? 60000,
           verbose: this.verbose,
+          streamGenAiSpans: this.streamGenAiSpans,
         });
       } catch (setupError) {
         // Mark all tests for this framework as errors
@@ -280,6 +284,7 @@ export class Orchestrator {
             resolvedOptions: testRun.framework.resolvedOptions,
             timeoutMs: testRun.testDefinition.timeoutMs ?? 60000,
             verbose: false, // Suppress template rendering logs, we're logging above
+            streamGenAiSpans: this.streamGenAiSpans,
           });
 
           renderedTests.set(testRun.id, testPath);
@@ -495,6 +500,7 @@ export class Orchestrator {
         resolvedOptions: testRun.framework.resolvedOptions,
         timeoutMs: testRun.testDefinition.timeoutMs ?? 60000,
         verbose: this.verbose,
+        streamGenAiSpans: this.streamGenAiSpans,
       });
 
       console.log(`  ✓ Setup complete`);
@@ -995,6 +1001,7 @@ export class Orchestrator {
         resolvedOptions: testRun.framework.resolvedOptions,
         timeoutMs,
         verbose: this.verbose && !this.useLiveStatus, // Only verbose when flag is set and not live status
+        streamGenAiSpans: this.streamGenAiSpans,
       });
 
       // Wait for spans to be collected

src/runner/runner.ts

@@ -271,6 +271,7 @@ export class Runner {
       ...(testDefinition.agent && { agent: testDefinition.agent }),
       ...(testDefinition.mcpServer && { mcpServer: testDefinition.mcpServer }),
       inputs: processedInputs,
+      streamGenAiSpans: context.streamGenAiSpans !== false, // JS Sentry.init() flag, default true
     };
 
     // PHP/Laravel: add computed class names and command signature for templates

src/runner/templates/base.browser.njk

@@ -44,6 +44,7 @@
       sendDefaultPii: true,
       // Capture 100% of traces for testing
       tracesSampleRate: 1.0,
+      streamGenAiSpans: {{ streamGenAiSpans }},
     });
 
     log('Sentry initialized');

src/runner/templates/base.cloudflare.njk

@@ -18,6 +18,7 @@ export default Sentry.withSentry(
     dsn: env.SENTRY_DSN,
     sendDefaultPii: true,
     tracesSampleRate: 1.0,
+    streamGenAiSpans: {{ streamGenAiSpans }},
     integrations: [
 {% block sentry_integrations %}{% endblock %}
     ],

src/runner/templates/base.nextjs.njk

@@ -18,6 +18,7 @@ Sentry.init({
   sendDefaultPii: true,
   // Capture 100% of traces for testing
   tracesSampleRate: 1.0,
+  streamGenAiSpans: {{ streamGenAiSpans }},
 });
 
 // Import framework AFTER Sentry.init() to ensure instrumentation hooks are in place

src/runner/templates/base.node.njk

@@ -13,6 +13,7 @@ Sentry.init({
   sendDefaultPii: true,
   // Capture 100% of traces for testing
   tracesSampleRate: 1.0,
+  streamGenAiSpans: {{ streamGenAiSpans }},
 });
 
 // Import framework AFTER Sentry.init() to ensure instrumentation hooks are in place

src/span-collector/server.ts

@@ -12,6 +12,38 @@ import { promisify } from 'util';
 
 const gunzip = promisify(zlib.gunzip);
 
+/**
+ * Convert a v2 envelope span into the legacy CapturedSpan shape used by the
+ * rest of the test framework. The typed attribute map `{ key: { type, value } }`
+ * is flattened to a plain `data: { key: value }` dict, matching what v1
+ * transaction-embedded spans already expose, so existing checks work unchanged.
+ *
+ * Span protocol reference:
+ *   https://develop.sentry.dev/sdk/telemetry/spans/span-protocol
+ *   https://develop.sentry.dev/sdk/telemetry/spans/implementation/#how-to-approach-span-first-in-sdks
+ */
+function v2SpanToCapturedSpan(v2: any): CapturedSpan {
+  const data: Record<string, any> = {};
+  if (v2.attributes && typeof v2.attributes === 'object') {
+    for (const [key, attr] of Object.entries<any>(v2.attributes)) {
+      data[key] = attr && typeof attr === 'object' && 'value' in attr ? attr.value : attr;
+    }
+  }
+  const op = typeof data['sentry.op'] === 'string' ? data['sentry.op'] : undefined;
+  return {
+    span_id: v2.span_id,
+    trace_id: v2.trace_id,
+    parent_span_id: v2.parent_span_id,
+    op: op as string,
+    description: v2.name,
+    start_timestamp: v2.start_timestamp,
+    timestamp: v2.end_timestamp,
+    data,
+    status: v2.status,
+    is_segment: v2.is_segment,
+  };
+}
+
 export class SpanCollector {
   private app: Hono;
   private server: ReturnType<typeof serve> | null = null;
@@ -118,7 +150,21 @@ export class SpanCollector {
         const itemHeader = JSON.parse(lines[i]);
         const itemBody = JSON.parse(lines[i + 1]);
 
-        if (itemHeader.type === 'transaction' || itemHeader.type === 'span') {
+        // v2 span envelope: application/vnd.sentry.items.span.v2+json
+        // Body shape: { version: 2, items: [<v2span>, ...] }
+        // Used by Sentry JS SDK 10.53.0+ when streamGenAiSpans is enabled —
+        // gen_ai spans are stripped from the transaction and shipped here.
+        // Spec: https://develop.sentry.dev/sdk/telemetry/spans/span-protocol
+        if (
+          itemHeader.type === 'span' &&
+          typeof itemHeader.content_type === 'string' &&
+          itemHeader.content_type.includes('span.v2') &&
+          Array.isArray(itemBody?.items)
+        ) {
+          for (const v2 of itemBody.items) {
+            spans.push(v2SpanToCapturedSpan(v2));
+          }
+        } else if (itemHeader.type === 'transaction' || itemHeader.type === 'span') {
           // Transaction contains child spans
           if (itemBody.spans) {
             spans.push(...itemBody.spans);

src/test-cases/llm/basic-error.ts

@@ -21,9 +21,13 @@ const checkErrorCaptured: Check = {
       throw new CheckError("Should have at least one AI span but found none");
     }
 
-    // Any non-ok span status from Sentry's SpanStatusType is valid
+    // Accept both v1 SpanStatusType granular codes and the v2 protocol's
+    // normalized "error" value. v2 (span-protocol) constrains status to
+    // {"ok","error"}; v1 transaction-embedded spans use the legacy enum.
     // See: sentry-javascript/packages/core/src/types-hoist/spanStatus.ts
+    //      https://develop.sentry.dev/sdk/telemetry/spans/span-protocol
     const errorStatuses = new Set([
+      "error",
       "deadline_exceeded",
       "unauthenticated",
       "permission_denied",

src/types.ts

@@ -311,4 +311,6 @@ export interface RunnerContext {
   timeoutMs: number;
   // Controls whether to print verbose console output (default: true)
   verbose?: boolean;
+  // JS only: value to pass for `streamGenAiSpans` in Sentry.init() (default: true)
+  streamGenAiSpans?: boolean;
 }