From a47dff5bc28f73a1981fb7f131e274c0f7774c12 Mon Sep 17 00:00:00 2001 From: Junya Yamaguchi Date: Thu, 2 Apr 2026 14:40:30 +0000 Subject: [PATCH 1/2] fix(otel): gate content attributes behind captureContent in toolCallingLoop and chatMLFetcher Content attributes (INPUT_MESSAGES, OUTPUT_MESSAGES, TOOL_DEFINITIONS) and user_message span events were unconditionally set via span.setAttribute() in toolCallingLoop.ts and chatMLFetcher.ts. Because ReadableSpan is immutable once created, these attributes leaked to the OTLP exporter even when captureContent was false. Gate all content attribute writes behind `otelService.config.captureContent`, matching the existing pattern in genAiEvents.ts and BYOK providers. Note: With captureContent=false, the Debug Panel will also not show content. This matches the documented behavior in agent_monitoring_arch.md. Fixes https://github.com/microsoft/vscode/issues/307407 --- src/extension/intents/node/toolCallingLoop.ts | 7 +- src/extension/prompt/node/chatMLFetcher.ts | 4 +- .../otel/common/test/contentGating.spec.ts | 186 ++++++++++++++++++ 3 files changed, 191 insertions(+), 6 deletions(-) create mode 100644 src/platform/otel/common/test/contentGating.spec.ts diff --git a/src/extension/intents/node/toolCallingLoop.ts b/src/extension/intents/node/toolCallingLoop.ts index 6ccc5bdf64..356dfd6347 100644 --- a/src/extension/intents/node/toolCallingLoop.ts +++ b/src/extension/intents/node/toolCallingLoop.ts @@ -761,8 +761,8 @@ export abstract class ToolCallingLoop | undefined; if (capiMessages) { // Normalize non-string content (Anthropic arrays, Responses API parts) to strings for OTel schema diff --git a/src/platform/otel/common/test/contentGating.spec.ts b/src/platform/otel/common/test/contentGating.spec.ts new file mode 100644 index 0000000000..c0c69ca028 --- /dev/null +++ b/src/platform/otel/common/test/contentGating.spec.ts @@ -0,0 +1,186 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { GenAiAttr, GenAiOperationName } from '../genAiAttributes'; +import { SpanKind } from '../otelService'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * Tests that content attributes (INPUT_MESSAGES, OUTPUT_MESSAGES, user_message events) + * are properly gated behind captureContent in agent and inference span patterns. + * + * Validates the fix for https://github.com/microsoft/vscode/issues/307407 + * where captureContent=false still leaked content to OTLP via unconditional + * span.setAttribute() calls in toolCallingLoop and chatMLFetcher. + */ +describe('Content gating in agent and inference spans', () => { + describe('agent span pattern (toolCallingLoop)', () => { + it('does NOT set INPUT_MESSAGES or user_message event when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + const span = otel.startSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.AGENT_NAME]: 'copilot', + }, + }); + + // Simulate the gated pattern now used in toolCallingLoop + if (otel.config.captureContent) { + const userMessage = 'fix my code'; + span.setAttribute(GenAiAttr.INPUT_MESSAGES, JSON.stringify([ + { role: 'user', parts: [{ type: 'text', content: userMessage }] } + ])); + span.addEvent('user_message', { content: userMessage }); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); + expect(otel.spans[0].events).toHaveLength(0); + }); + + it('does NOT set OUTPUT_MESSAGES when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + const span = otel.startSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + }, + }); + + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, JSON.stringify([ + { role: 'assistant', parts: [{ type: 'text', content: 'here is the fix' }] } + ])); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeUndefined(); + }); + + it('sets INPUT_MESSAGES, OUTPUT_MESSAGES, and user_message event when captureContent is true', () => { + const otel = new CapturingOTelService({ captureContent: true }); + + const span = otel.startSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.AGENT_NAME]: 'copilot', + }, + }); + + const userMessage = 'fix my code'; + const expectedInput = JSON.stringify([ + { role: 'user', parts: [{ type: 'text', content: userMessage }] } + ]); + const expectedOutput = JSON.stringify([ + { role: 'assistant', parts: [{ type: 'text', content: 'here is the fix' }] } + ]); + + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, expectedInput); + span.addEvent('user_message', { content: userMessage }); + span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, expectedOutput); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBe(expectedInput); + expect(otel.spans[0].attributes[GenAiAttr.OUTPUT_MESSAGES]).toBe(expectedOutput); + expect(otel.spans[0].events).toHaveLength(1); + expect(otel.spans[0].events[0].name).toBe('user_message'); + }); + + it('does NOT set TOOL_DEFINITIONS when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + const span = otel.startSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.AGENT_NAME]: 'copilot', + }, + }); + + // TOOL_DEFINITIONS is classified as opt-in content in genAiAttributes.ts + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.TOOL_DEFINITIONS, JSON.stringify([ + { type: 'function', name: 'readFile', description: 'Read a file' } + ])); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeUndefined(); + }); + + it('still sets non-content attributes when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + const span = otel.startSpan('invoke_agent copilot', { + kind: SpanKind.INTERNAL, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, + [GenAiAttr.AGENT_NAME]: 'copilot', + }, + }); + + // Non-content attributes like AGENT_NAME should always be set + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.AGENT_NAME]).toBe('copilot'); + }); + }); + + describe('inference span pattern (chatMLFetcher)', () => { + it('does NOT set INPUT_MESSAGES when captureContent is false', () => { + const otel = new CapturingOTelService({ captureContent: false }); + + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }, + }); + + // Simulate the gated pattern now used in chatMLFetcher + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, JSON.stringify([ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Hello' }, + ])); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); + }); + + it('sets INPUT_MESSAGES when captureContent is true', () => { + const otel = new CapturingOTelService({ captureContent: true }); + + const span = otel.startSpan('chat gpt-4o', { + kind: SpanKind.CLIENT, + attributes: { + [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, + [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', + }, + }); + + const expectedMessages = JSON.stringify([ + { role: 'system', content: 'You are a helpful assistant.' }, + { role: 'user', content: 'Hello' }, + ]); + + if (otel.config.captureContent) { + span.setAttribute(GenAiAttr.INPUT_MESSAGES, expectedMessages); + } + span.end(); + + expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBe(expectedMessages); + }); + }); +}); From a6323f3929ea8ea4ba93a24ca2af65c8fcfe3b89 Mon Sep 17 00:00:00 2001 From: Junya Yamaguchi Date: Fri, 3 Apr 2026 11:06:18 +0000 Subject: [PATCH 2/2] fix(otel): gate all content attributes behind captureContent - chatMLFetcher: gate USER_REQUEST, SYSTEM_INSTRUCTIONS, OUTPUT_MESSAGES, and REASONING_CONTENT behind captureContent (previously only INPUT_MESSAGES was gated) - Replace inline-duplicating unit tests in contentGating.spec.ts with integration tests that exercise the real ToolCallingLoop code path, ensuring guards cannot be silently removed without test failures - Add CapturingOTelService config smoke tests --- .../node/toolCallingLoopContentGating.spec.ts | 248 ++++++++++++++++++ src/extension/prompt/node/chatMLFetcher.ts | 8 +- .../otel/common/test/contentGating.spec.ts | 184 +------------ 3 files changed, 266 insertions(+), 174 deletions(-) create mode 100644 src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts diff --git a/src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts b/src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts new file mode 100644 index 0000000000..afdd75d62b --- /dev/null +++ b/src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts @@ -0,0 +1,248 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { Raw } from '@vscode/prompt-tsx'; +import { afterEach, beforeEach, describe, expect, it } from 'vitest'; +import type { ChatRequest, LanguageModelChat, LanguageModelToolInformation } from 'vscode'; +import { ChatFetchResponseType, ChatResponse } from '../../../../platform/chat/common/commonTypes'; +import { toTextPart } from '../../../../platform/chat/common/globalStringUtils'; +import { IEndpointProvider } from '../../../../platform/endpoint/common/endpointProvider'; +import { IChatEndpoint, IEmbeddingsEndpoint } from '../../../../platform/networking/common/networking'; +import { GenAiAttr, GenAiOperationName } from '../../../../platform/otel/common/genAiAttributes'; +import { IOTelService } from '../../../../platform/otel/common/otelService'; +import { CapturingOTelService } from '../../../../platform/otel/common/test/capturingOTelService'; +import { CancellationTokenSource } from '../../../../util/vs/base/common/cancellation'; +import { Event } from '../../../../util/vs/base/common/event'; +import { DisposableStore } from '../../../../util/vs/base/common/lifecycle'; +import { generateUuid } from '../../../../util/vs/base/common/uuid'; +import { IInstantiationService } from '../../../../util/vs/platform/instantiation/common/instantiation'; +import { Conversation, Turn } from '../../../prompt/common/conversation'; +import { IBuildPromptContext } from '../../../prompt/common/intents'; +import { IBuildPromptResult, nullRenderPromptResult } from '../../../prompt/node/intents'; +import { createExtensionUnitTestingServices } from '../../../test/node/services'; +import { IToolCallingLoopOptions, ToolCallingLoop } from '../../node/toolCallingLoop'; + +class MockEndpointProvider implements IEndpointProvider { + declare readonly _serviceBrand: undefined; + readonly onDidModelsRefresh = Event.None; + + async getAllCompletionModels() { return []; } + async getAllChatEndpoints() { return [this._createEndpoint()]; } + async getChatEndpoint() { return this._createEndpoint(); } + async getEmbeddingsEndpoint(): Promise { throw new Error('Not implemented'); } + + private _createEndpoint(): IChatEndpoint { + return { + model: 'gpt-4o', + modelProvider: 'github', + family: 'gpt-4o', + name: 'test-endpoint', + version: '1.0', + maxOutputTokens: 4096, + modelMaxPromptTokens: 128000, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: true, + isDefault: true, + isFallback: false, + policy: 'enabled' as const, + urlOrRequestMetadata: 'mock://endpoint', + tokenizer: 'cl100k_base', + acquireTokenizer: () => ({ + countMessagesTokens: async () => 100, + countMessageTokens: async () => 10, + countToolTokens: async () => 50, + encode: () => [], + free: () => { }, + }), + } as unknown as IChatEndpoint; + } +} + +/** + * Integration tests that exercise the real ToolCallingLoop code path to verify + * content attributes are properly gated behind captureContent. + * + * Unlike the original contentGating.spec.ts (which duplicated the if-check inline), + * these tests call through ToolCallingLoop.run() so they will fail if the + * captureContent guards are removed from the production code. + */ + +class ContentGatingTestToolCallingLoop extends ToolCallingLoop { + protected override async buildPrompt(_buildPromptContext: IBuildPromptContext): Promise { + return { + ...nullRenderPromptResult(), + messages: [{ role: Raw.ChatRole.User, content: [toTextPart('fix my code')] }], + }; + } + + protected override async getAvailableTools(): Promise { + return [ + { name: 'readFile', description: 'Read a file from the workspace', inputSchema: {}, tags: [], source: undefined }, + { name: 'writeFile', description: 'Write content to a file', inputSchema: {}, tags: [], source: undefined }, + ]; + } + + protected override async fetch(): Promise { + return { + type: ChatFetchResponseType.Success, + value: 'Here is the fix for your code.', + requestId: 'req-123', + serverRequestId: undefined, + usage: { + prompt_tokens: 50, + completion_tokens: 10, + total_tokens: 60, + }, + resolvedModel: 'gpt-4o', + }; + } +} + +const chatPanelLocation: ChatRequest['location'] = 1; + +function createMockChatRequest(overrides: Partial = {}): ChatRequest { + return { + prompt: 'fix my code', + command: undefined, + references: [], + location: chatPanelLocation, + location2: undefined, + attempt: 0, + enableCommandDetection: false, + isParticipantDetected: false, + toolReferences: [], + toolInvocationToken: {} as ChatRequest['toolInvocationToken'], + model: { family: 'test' } as LanguageModelChat, + tools: new Map(), + id: generateUuid(), + sessionId: generateUuid(), + sessionResource: {} as ChatRequest['sessionResource'], + hasHooksEnabled: false, + ...overrides, + } satisfies ChatRequest; +} + +function createConversation(prompt: string): Conversation { + return new Conversation(generateUuid(), [ + new Turn(generateUuid(), { type: 'user', message: prompt }), + ]); +} + +describe('ToolCallingLoop content gating (integration)', () => { + let disposables: DisposableStore; + let tokenSource: CancellationTokenSource; + + beforeEach(() => { + disposables = new DisposableStore(); + tokenSource = new CancellationTokenSource(); + disposables.add(tokenSource); + }); + + afterEach(() => { + disposables.dispose(); + }); + + function createLoopWithOTel(captureContent: boolean) { + const otel = new CapturingOTelService({ captureContent }); + const serviceCollection = disposables.add(createExtensionUnitTestingServices()); + serviceCollection.define(IOTelService, otel); + serviceCollection.define(IEndpointProvider, new MockEndpointProvider()); + const accessor = serviceCollection.createTestingAccessor(); + disposables.add(accessor); + const instantiationService = accessor.get(IInstantiationService); + + const request = createMockChatRequest(); + const loop = instantiationService.createInstance( + ContentGatingTestToolCallingLoop, + { + conversation: createConversation(request.prompt), + toolCallLimit: 1, + request, + }, + ); + disposables.add(loop); + return { otel, loop }; + } + + it('does NOT set content attributes on agent span when captureContent is false', async () => { + const { otel, loop } = createLoopWithOTel(false); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + expect(agentSpan).toBeDefined(); + + // Content attributes must NOT be set + expect(agentSpan.attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); + expect(agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeUndefined(); + expect(agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeUndefined(); + expect(agentSpan.events.filter(e => e.name === 'user_message')).toHaveLength(0); + + // Non-content attributes should still be set + expect(agentSpan.attributes[GenAiAttr.AGENT_NAME]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.OPERATION_NAME]).toBe(GenAiOperationName.INVOKE_AGENT); + }); + + it('sets content attributes on agent span when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + expect(agentSpan).toBeDefined(); + + // Content attributes must be set + expect(agentSpan.attributes[GenAiAttr.INPUT_MESSAGES]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeDefined(); + + // user_message event should be emitted + const userMessageEvents = agentSpan.events.filter(e => e.name === 'user_message'); + expect(userMessageEvents).toHaveLength(1); + }); + + it('INPUT_MESSAGES contains the user prompt when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const inputMessages = agentSpan.attributes[GenAiAttr.INPUT_MESSAGES] as string; + expect(inputMessages).toContain('fix my code'); + }); + + it('OUTPUT_MESSAGES contains the response text when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const outputMessages = agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES] as string; + expect(outputMessages).toContain('Here is the fix for your code.'); + }); + + it('TOOL_DEFINITIONS contains the available tools when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const toolDefs = agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS] as string; + expect(toolDefs).toContain('readFile'); + expect(toolDefs).toContain('writeFile'); + }); + + it('user_message event content matches user prompt when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const userMessageEvent = agentSpan.events.find(e => e.name === 'user_message'); + expect(userMessageEvent?.attributes?.content).toBe('fix my code'); + }); +}); diff --git a/src/extension/prompt/node/chatMLFetcher.ts b/src/extension/prompt/node/chatMLFetcher.ts index f7265df846..dd88255f64 100644 --- a/src/extension/prompt/node/chatMLFetcher.ts +++ b/src/extension/prompt/node/chatMLFetcher.ts @@ -243,8 +243,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { // Tag span with debug name so orphaned spans (title, progressMessages, etc.) are identifiable otelInferenceSpan?.setAttribute(GenAiAttr.AGENT_NAME, debugName); - // Extract and set structured prompt sections for the debug panel - if (otelInferenceSpan) { + // Extract and set structured prompt sections — gated by captureContent to prevent OTLP leakage + if (otelInferenceSpan && this._otelService.config.captureContent) { // Support both Chat Completions API (messages) and Responses API (input) formats const capiMessages = (requestBody.messages ?? requestBody.input) as ReadonlyArray<{ role?: string; content?: string | unknown[] }> | undefined; // User request: last user-role message @@ -391,8 +391,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { : {}), }); } - // Always capture response content for the debug panel - if (otelInferenceSpan && result.type === ChatFetchResponseType.Success) { + // Capture response content — gated by captureContent to prevent OTLP leakage + if (otelInferenceSpan && this._otelService.config.captureContent && result.type === ChatFetchResponseType.Success) { const responseText = streamRecorder.deltas.map(d => d.text).join(''); const toolCalls = streamRecorder.deltas .filter(d => d.copilotToolCalls?.length) diff --git a/src/platform/otel/common/test/contentGating.spec.ts b/src/platform/otel/common/test/contentGating.spec.ts index c0c69ca028..4cc9df4dd7 100644 --- a/src/platform/otel/common/test/contentGating.spec.ts +++ b/src/platform/otel/common/test/contentGating.spec.ts @@ -4,183 +4,27 @@ *--------------------------------------------------------------------------------------------*/ import { describe, expect, it } from 'vitest'; -import { GenAiAttr, GenAiOperationName } from '../genAiAttributes'; -import { SpanKind } from '../otelService'; import { CapturingOTelService } from './capturingOTelService'; /** - * Tests that content attributes (INPUT_MESSAGES, OUTPUT_MESSAGES, user_message events) - * are properly gated behind captureContent in agent and inference span patterns. + * CapturingOTelService correctly exposes the captureContent config flag. * - * Validates the fix for https://github.com/microsoft/vscode/issues/307407 - * where captureContent=false still leaked content to OTLP via unconditional - * span.setAttribute() calls in toolCallingLoop and chatMLFetcher. + * Integration tests that exercise the real ToolCallingLoop code path live in + * src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts. */ -describe('Content gating in agent and inference spans', () => { - describe('agent span pattern (toolCallingLoop)', () => { - it('does NOT set INPUT_MESSAGES or user_message event when captureContent is false', () => { - const otel = new CapturingOTelService({ captureContent: false }); - - const span = otel.startSpan('invoke_agent copilot', { - kind: SpanKind.INTERNAL, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, - [GenAiAttr.AGENT_NAME]: 'copilot', - }, - }); - - // Simulate the gated pattern now used in toolCallingLoop - if (otel.config.captureContent) { - const userMessage = 'fix my code'; - span.setAttribute(GenAiAttr.INPUT_MESSAGES, JSON.stringify([ - { role: 'user', parts: [{ type: 'text', content: userMessage }] } - ])); - span.addEvent('user_message', { content: userMessage }); - } - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); - expect(otel.spans[0].events).toHaveLength(0); - }); - - it('does NOT set OUTPUT_MESSAGES when captureContent is false', () => { - const otel = new CapturingOTelService({ captureContent: false }); - - const span = otel.startSpan('invoke_agent copilot', { - kind: SpanKind.INTERNAL, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, - }, - }); - - if (otel.config.captureContent) { - span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, JSON.stringify([ - { role: 'assistant', parts: [{ type: 'text', content: 'here is the fix' }] } - ])); - } - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeUndefined(); - }); - - it('sets INPUT_MESSAGES, OUTPUT_MESSAGES, and user_message event when captureContent is true', () => { - const otel = new CapturingOTelService({ captureContent: true }); - - const span = otel.startSpan('invoke_agent copilot', { - kind: SpanKind.INTERNAL, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, - [GenAiAttr.AGENT_NAME]: 'copilot', - }, - }); - - const userMessage = 'fix my code'; - const expectedInput = JSON.stringify([ - { role: 'user', parts: [{ type: 'text', content: userMessage }] } - ]); - const expectedOutput = JSON.stringify([ - { role: 'assistant', parts: [{ type: 'text', content: 'here is the fix' }] } - ]); - - if (otel.config.captureContent) { - span.setAttribute(GenAiAttr.INPUT_MESSAGES, expectedInput); - span.addEvent('user_message', { content: userMessage }); - span.setAttribute(GenAiAttr.OUTPUT_MESSAGES, expectedOutput); - } - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBe(expectedInput); - expect(otel.spans[0].attributes[GenAiAttr.OUTPUT_MESSAGES]).toBe(expectedOutput); - expect(otel.spans[0].events).toHaveLength(1); - expect(otel.spans[0].events[0].name).toBe('user_message'); - }); - - it('does NOT set TOOL_DEFINITIONS when captureContent is false', () => { - const otel = new CapturingOTelService({ captureContent: false }); - - const span = otel.startSpan('invoke_agent copilot', { - kind: SpanKind.INTERNAL, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, - [GenAiAttr.AGENT_NAME]: 'copilot', - }, - }); - - // TOOL_DEFINITIONS is classified as opt-in content in genAiAttributes.ts - if (otel.config.captureContent) { - span.setAttribute(GenAiAttr.TOOL_DEFINITIONS, JSON.stringify([ - { type: 'function', name: 'readFile', description: 'Read a file' } - ])); - } - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeUndefined(); - }); - - it('still sets non-content attributes when captureContent is false', () => { - const otel = new CapturingOTelService({ captureContent: false }); - - const span = otel.startSpan('invoke_agent copilot', { - kind: SpanKind.INTERNAL, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.INVOKE_AGENT, - [GenAiAttr.AGENT_NAME]: 'copilot', - }, - }); - - // Non-content attributes like AGENT_NAME should always be set - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.AGENT_NAME]).toBe('copilot'); - }); +describe('CapturingOTelService captureContent config', () => { + it('defaults captureContent to false when OTEL is enabled without explicit flag', () => { + const otel = new CapturingOTelService(); + expect(otel.config.captureContent).toBe(false); }); - describe('inference span pattern (chatMLFetcher)', () => { - it('does NOT set INPUT_MESSAGES when captureContent is false', () => { - const otel = new CapturingOTelService({ captureContent: false }); - - const span = otel.startSpan('chat gpt-4o', { - kind: SpanKind.CLIENT, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, - [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', - }, - }); - - // Simulate the gated pattern now used in chatMLFetcher - if (otel.config.captureContent) { - span.setAttribute(GenAiAttr.INPUT_MESSAGES, JSON.stringify([ - { role: 'system', content: 'You are a helpful assistant.' }, - { role: 'user', content: 'Hello' }, - ])); - } - span.end(); - - expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); - }); - - it('sets INPUT_MESSAGES when captureContent is true', () => { - const otel = new CapturingOTelService({ captureContent: true }); - - const span = otel.startSpan('chat gpt-4o', { - kind: SpanKind.CLIENT, - attributes: { - [GenAiAttr.OPERATION_NAME]: GenAiOperationName.CHAT, - [GenAiAttr.REQUEST_MODEL]: 'gpt-4o', - }, - }); - - const expectedMessages = JSON.stringify([ - { role: 'system', content: 'You are a helpful assistant.' }, - { role: 'user', content: 'Hello' }, - ]); - - if (otel.config.captureContent) { - span.setAttribute(GenAiAttr.INPUT_MESSAGES, expectedMessages); - } - span.end(); + it('respects captureContent=true override', () => { + const otel = new CapturingOTelService({ captureContent: true }); + expect(otel.config.captureContent).toBe(true); + }); - expect(otel.spans[0].attributes[GenAiAttr.INPUT_MESSAGES]).toBe(expectedMessages); - }); + it('respects captureContent=false override', () => { + const otel = new CapturingOTelService({ captureContent: false }); + expect(otel.config.captureContent).toBe(false); }); });