diff --git a/src/extension/intents/node/toolCallingLoop.ts b/src/extension/intents/node/toolCallingLoop.ts index 6ccc5bdf64..356dfd6347 100644 --- a/src/extension/intents/node/toolCallingLoop.ts +++ b/src/extension/intents/node/toolCallingLoop.ts @@ -761,8 +761,8 @@ export abstract class ToolCallingLoop { throw new Error('Not implemented'); } + + private _createEndpoint(): IChatEndpoint { + return { + model: 'gpt-4o', + modelProvider: 'github', + family: 'gpt-4o', + name: 'test-endpoint', + version: '1.0', + maxOutputTokens: 4096, + modelMaxPromptTokens: 128000, + supportsToolCalls: true, + supportsVision: false, + supportsPrediction: false, + showInModelPicker: true, + isDefault: true, + isFallback: false, + policy: 'enabled' as const, + urlOrRequestMetadata: 'mock://endpoint', + tokenizer: 'cl100k_base', + acquireTokenizer: () => ({ + countMessagesTokens: async () => 100, + countMessageTokens: async () => 10, + countToolTokens: async () => 50, + encode: () => [], + free: () => { }, + }), + } as unknown as IChatEndpoint; + } +} + +/** + * Integration tests that exercise the real ToolCallingLoop code path to verify + * content attributes are properly gated behind captureContent. + * + * Unlike the original contentGating.spec.ts (which duplicated the if-check inline), + * these tests call through ToolCallingLoop.run() so they will fail if the + * captureContent guards are removed from the production code. + */ + +class ContentGatingTestToolCallingLoop extends ToolCallingLoop { + protected override async buildPrompt(_buildPromptContext: IBuildPromptContext): Promise { + return { + ...nullRenderPromptResult(), + messages: [{ role: Raw.ChatRole.User, content: [toTextPart('fix my code')] }], + }; + } + + protected override async getAvailableTools(): Promise { + return [ + { name: 'readFile', description: 'Read a file from the workspace', inputSchema: {}, tags: [], source: undefined }, + { name: 'writeFile', description: 'Write content to a file', inputSchema: {}, tags: [], source: undefined }, + ]; + } + + protected override async fetch(): Promise { + return { + type: ChatFetchResponseType.Success, + value: 'Here is the fix for your code.', + requestId: 'req-123', + serverRequestId: undefined, + usage: { + prompt_tokens: 50, + completion_tokens: 10, + total_tokens: 60, + }, + resolvedModel: 'gpt-4o', + }; + } +} + +const chatPanelLocation: ChatRequest['location'] = 1; + +function createMockChatRequest(overrides: Partial = {}): ChatRequest { + return { + prompt: 'fix my code', + command: undefined, + references: [], + location: chatPanelLocation, + location2: undefined, + attempt: 0, + enableCommandDetection: false, + isParticipantDetected: false, + toolReferences: [], + toolInvocationToken: {} as ChatRequest['toolInvocationToken'], + model: { family: 'test' } as LanguageModelChat, + tools: new Map(), + id: generateUuid(), + sessionId: generateUuid(), + sessionResource: {} as ChatRequest['sessionResource'], + hasHooksEnabled: false, + ...overrides, + } satisfies ChatRequest; +} + +function createConversation(prompt: string): Conversation { + return new Conversation(generateUuid(), [ + new Turn(generateUuid(), { type: 'user', message: prompt }), + ]); +} + +describe('ToolCallingLoop content gating (integration)', () => { + let disposables: DisposableStore; + let tokenSource: CancellationTokenSource; + + beforeEach(() => { + disposables = new DisposableStore(); + tokenSource = new CancellationTokenSource(); + disposables.add(tokenSource); + }); + + afterEach(() => { + disposables.dispose(); + }); + + function createLoopWithOTel(captureContent: boolean) { + const otel = new CapturingOTelService({ captureContent }); + const serviceCollection = disposables.add(createExtensionUnitTestingServices()); + serviceCollection.define(IOTelService, otel); + serviceCollection.define(IEndpointProvider, new MockEndpointProvider()); + const accessor = serviceCollection.createTestingAccessor(); + disposables.add(accessor); + const instantiationService = accessor.get(IInstantiationService); + + const request = createMockChatRequest(); + const loop = instantiationService.createInstance( + ContentGatingTestToolCallingLoop, + { + conversation: createConversation(request.prompt), + toolCallLimit: 1, + request, + }, + ); + disposables.add(loop); + return { otel, loop }; + } + + it('does NOT set content attributes on agent span when captureContent is false', async () => { + const { otel, loop } = createLoopWithOTel(false); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + expect(agentSpan).toBeDefined(); + + // Content attributes must NOT be set + expect(agentSpan.attributes[GenAiAttr.INPUT_MESSAGES]).toBeUndefined(); + expect(agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeUndefined(); + expect(agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeUndefined(); + expect(agentSpan.events.filter(e => e.name === 'user_message')).toHaveLength(0); + + // Non-content attributes should still be set + expect(agentSpan.attributes[GenAiAttr.AGENT_NAME]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.OPERATION_NAME]).toBe(GenAiOperationName.INVOKE_AGENT); + }); + + it('sets content attributes on agent span when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + expect(agentSpan).toBeDefined(); + + // Content attributes must be set + expect(agentSpan.attributes[GenAiAttr.INPUT_MESSAGES]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES]).toBeDefined(); + expect(agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS]).toBeDefined(); + + // user_message event should be emitted + const userMessageEvents = agentSpan.events.filter(e => e.name === 'user_message'); + expect(userMessageEvents).toHaveLength(1); + }); + + it('INPUT_MESSAGES contains the user prompt when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const inputMessages = agentSpan.attributes[GenAiAttr.INPUT_MESSAGES] as string; + expect(inputMessages).toContain('fix my code'); + }); + + it('OUTPUT_MESSAGES contains the response text when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const outputMessages = agentSpan.attributes[GenAiAttr.OUTPUT_MESSAGES] as string; + expect(outputMessages).toContain('Here is the fix for your code.'); + }); + + it('TOOL_DEFINITIONS contains the available tools when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const toolDefs = agentSpan.attributes[GenAiAttr.TOOL_DEFINITIONS] as string; + expect(toolDefs).toContain('readFile'); + expect(toolDefs).toContain('writeFile'); + }); + + it('user_message event content matches user prompt when captureContent is true', async () => { + const { otel, loop } = createLoopWithOTel(true); + + await loop.run(undefined, tokenSource.token); + + const agentSpan = otel.findSpans('invoke_agent')[0]; + const userMessageEvent = agentSpan.events.find(e => e.name === 'user_message'); + expect(userMessageEvent?.attributes?.content).toBe('fix my code'); + }); +}); diff --git a/src/extension/prompt/node/chatMLFetcher.ts b/src/extension/prompt/node/chatMLFetcher.ts index 6e54d31919..dd88255f64 100644 --- a/src/extension/prompt/node/chatMLFetcher.ts +++ b/src/extension/prompt/node/chatMLFetcher.ts @@ -243,8 +243,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { // Tag span with debug name so orphaned spans (title, progressMessages, etc.) are identifiable otelInferenceSpan?.setAttribute(GenAiAttr.AGENT_NAME, debugName); - // Extract and set structured prompt sections for the debug panel - if (otelInferenceSpan) { + // Extract and set structured prompt sections — gated by captureContent to prevent OTLP leakage + if (otelInferenceSpan && this._otelService.config.captureContent) { // Support both Chat Completions API (messages) and Responses API (input) formats const capiMessages = (requestBody.messages ?? requestBody.input) as ReadonlyArray<{ role?: string; content?: string | unknown[] }> | undefined; // User request: last user-role message @@ -278,8 +278,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { } } - // Always capture full request content for the debug panel - if (otelInferenceSpan) { + // Capture full request content — gated by captureContent to prevent OTLP leakage + if (otelInferenceSpan && this._otelService.config.captureContent) { const capiMessages = (requestBody.messages ?? requestBody.input) as ReadonlyArray<{ role?: string; content?: string | unknown[] }> | undefined; if (capiMessages) { // Normalize non-string content (Anthropic arrays, Responses API parts) to strings for OTel schema @@ -391,8 +391,8 @@ export class ChatMLFetcherImpl extends AbstractChatMLFetcher { : {}), }); } - // Always capture response content for the debug panel - if (otelInferenceSpan && result.type === ChatFetchResponseType.Success) { + // Capture response content — gated by captureContent to prevent OTLP leakage + if (otelInferenceSpan && this._otelService.config.captureContent && result.type === ChatFetchResponseType.Success) { const responseText = streamRecorder.deltas.map(d => d.text).join(''); const toolCalls = streamRecorder.deltas .filter(d => d.copilotToolCalls?.length) diff --git a/src/platform/otel/common/test/contentGating.spec.ts b/src/platform/otel/common/test/contentGating.spec.ts new file mode 100644 index 0000000000..4cc9df4dd7 --- /dev/null +++ b/src/platform/otel/common/test/contentGating.spec.ts @@ -0,0 +1,30 @@ +/*--------------------------------------------------------------------------------------------- + * Copyright (c) Microsoft Corporation. All rights reserved. + * Licensed under the MIT License. See License.txt in the project root for license information. + *--------------------------------------------------------------------------------------------*/ + +import { describe, expect, it } from 'vitest'; +import { CapturingOTelService } from './capturingOTelService'; + +/** + * CapturingOTelService correctly exposes the captureContent config flag. + * + * Integration tests that exercise the real ToolCallingLoop code path live in + * src/extension/intents/test/node/toolCallingLoopContentGating.spec.ts. + */ +describe('CapturingOTelService captureContent config', () => { + it('defaults captureContent to false when OTEL is enabled without explicit flag', () => { + const otel = new CapturingOTelService(); + expect(otel.config.captureContent).toBe(false); + }); + + it('respects captureContent=true override', () => { + const otel = new CapturingOTelService({ captureContent: true }); + expect(otel.config.captureContent).toBe(true); + }); + + it('respects captureContent=false override', () => { + const otel = new CapturingOTelService({ captureContent: false }); + expect(otel.config.captureContent).toBe(false); + }); +});