Skip to content

Fix: count all tool tokens in budget including deferred tools #4990

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
bhavyaus wants to merge 1 commit into from
Closed

Fix: count all tool tokens in budget including deferred tools #4990

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 4 additions & 37 deletions src/extension/intents/node/agentIntent.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,6 @@ import { IEditLogService } from '../../../platform/multiFileEdit/common/editLogS
import { CUSTOM_TOOL_SEARCH_NAME, isAnthropicCustomToolSearchEnabled, isAnthropicToolSearchEnabled } from '../../../platform/networking/common/anthropic';
import { IChatEndpoint } from '../../../platform/networking/common/networking';
import { modelsWithoutResponsesContextManagement } from '../../../platform/networking/common/openai';
import { IToolDeferralService } from '../../../platform/networking/common/toolDeferralService';
import { INotebookService } from '../../../platform/notebook/common/notebookService';
import { GenAiMetrics } from '../../../platform/otel/common/genAiMetrics';
import { IOTelService } from '../../../platform/otel/common/otelService';
Expand Down Expand Up @@ -379,7 +378,6 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
@IExperimentationService private readonly expService: IExperimentationService,
@IAutomodeService private readonly automodeService: IAutomodeService,
@IOTelService override readonly otelService: IOTelService,
Copy link

Copilot AI Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

AgentIntentInvocation’s constructor no longer accepts IToolDeferralService, but several subclasses (e.g. AskAgentIntentInvocation/EditCode2IntentInvocation/NotebookEditorIntentInvocation) still inject toolDeferralService and pass it to super(...). This will fail TypeScript compilation due to an argument count mismatch. Update those subclasses to remove the extra DI parameter + super argument (and remove now-unused imports) to keep constructors consistent.

Suggested change
@IOTelService override readonly otelService: IOTelService,
@IOTelService override readonly otelService: IOTelService,
_toolDeferralService?: unknown,

Copilot uses AI. Check for mistakes.
@IToolDeferralService private readonly toolDeferralService: IToolDeferralService,
) {
super(intent, location, endpoint, request, intentOptions, instantiationService, codeMapperService, envService, promptPathRepresentationService, endpointProvider, workspaceService, toolsService, configurationService, editLogService, commandService, telemetryService, notebookService, otelService);
}
Expand All @@ -405,15 +403,8 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
}

const tools = promptContext.tools?.availableTools;
// When Anthropic tool search is enabled, deferred tools are sent with
// defer_loading: true and don't count against the context window until
// the model loads them via tool_search. Only count non-deferred tools
// so the budget isn't artificially reduced.
const toolSearchEnabled = isAnthropicToolSearchEnabled(this.endpoint, this.configurationService);
const effectiveTools = tools && toolSearchEnabled
? tools.filter(t => this.toolDeferralService.isNonDeferredTool(t.name))
: tools;
const toolTokens = effectiveTools?.length ? await this.endpoint.acquireTokenizer().countToolTokens(effectiveTools) : 0;
const toolTokens = tools?.length ? await this.endpoint.acquireTokenizer().countToolTokens(tools) : 0;

const summarizeThresholdOverride = this.configurationService.getConfig<number | undefined>(ConfigKey.Advanced.SummarizeAgentConversationHistoryThreshold);
if (typeof summarizeThresholdOverride === 'number' && summarizeThresholdOverride < 100 && summarizeThresholdOverride > 0) {
Expand Down Expand Up @@ -441,7 +432,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
const safeBudget = useTruncation ? Number.MAX_SAFE_INTEGER : messageBudget;
const endpoint = toolTokens > 0 ? this.endpoint.cloneWithTokenOverride(safeBudget) : this.endpoint;

this.logService.debug(`AgentIntent: rendering with budget=${safeBudget} (baseBudget: ${baseBudget}, toolTokens: ${toolTokens}${toolSearchEnabled ? `, totalTools: ${tools?.length ?? 0}, nonDeferredTools: ${effectiveTools?.length ?? 0}` : ''}), summarizationEnabled=${summarizationEnabled}`);
this.logService.debug(`AgentIntent: rendering with budget=${safeBudget} (baseBudget: ${baseBudget}, toolTokens: ${toolTokens}, totalTools: ${tools?.length ?? 0}, toolSearchEnabled: ${toolSearchEnabled}), summarizationEnabled=${summarizationEnabled}`);
let result: RenderPromptResult;
const props: AgentPromptProps = {
endpoint,
Expand Down Expand Up @@ -595,7 +586,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
const renderer = PromptRenderer.create(this.instantiationService, this.endpoint, this.prompt, {
...renderProps,
endpoint: this.endpoint,
promptContext: this._buildSummarizationPromptContext(renderProps.promptContext),
promptContext: renderProps.promptContext,
triggerSummarize: true,
});
return await renderer.render(progress, token);
Expand Down Expand Up @@ -868,7 +859,7 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
const bgRenderer = PromptRenderer.create(this.instantiationService, this.endpoint, this.prompt, {
...snapshotProps,
endpoint: this.endpoint,
promptContext: this._buildSummarizationPromptContext(snapshotProps.promptContext),
promptContext: snapshotProps.promptContext,
triggerSummarize: true,
summarizationSource: 'background',
});
Expand Down Expand Up @@ -978,30 +969,6 @@ export class AgentIntentInvocation extends EditCodeIntentInvocation implements I
));
}

/**
* Build a promptContext for summarization that filters availableTools to
* non-deferred tools when Anthropic tool search is enabled. Deferred tool
* schemas are unnecessary in the summarization prompt (which uses
* tool_choice: 'none') and can push the prompt over the token budget.
*/
private _buildSummarizationPromptContext(promptContext: IBuildPromptContext): IBuildPromptContext {
if (!promptContext.tools?.availableTools) {
return promptContext;
}
const toolSearchEnabled = isAnthropicToolSearchEnabled(this.endpoint, this.configurationService);
if (!toolSearchEnabled) {
return promptContext;
}
const nonDeferredTools = promptContext.tools.availableTools.filter(t => this.toolDeferralService.isNonDeferredTool(t.name));
return {
...promptContext,
tools: {
...promptContext.tools,
availableTools: nonDeferredTools,
},
};
}

/**
* Record a background compaction failure on the current turn's metadata,
* matching how foreground compaction records its failures.
Expand Down
19 changes: 16 additions & 3 deletions src/extension/prompts/node/agent/summarizedConversationHistory.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -170,7 +170,7 @@ export class ConversationHistorySummarizationPrompt extends PromptElement<Conver
</SystemMessage>
{history}
{this.props.workingNotebook && <WorkingNotebookSummary priority={this.props.priority - 2} notebook={this.props.workingNotebook} />}
<UserMessage>
<UserMessage priority={this.props.priority}>
Summarize the conversation history so far, paying special attention to the most recent agent commands and tool results that triggered this summarization. Structure your summary using the enhanced format provided in the system message.<br />
{isOpus && <>
<br />
Expand Down Expand Up @@ -664,7 +664,18 @@ class ConversationHistorySummarizer {

private async getSummary(mode: SummaryMode, propsInfo: ISummarizedConversationHistoryInfo): Promise<SummarizationResult> {
const stopwatch = new StopWatch(false);
const endpoint = this.props.endpoint;

// In Full mode, tools are sent alongside the summarization prompt with
// tool_choice: 'none'. Reserve budget for them so the rendered messages
// plus tools don't exceed the model's context window.
const tools = this.props.tools;
const toolTokens = mode === SummaryMode.Full && tools?.length
? await this.props.endpoint.acquireTokenizer().countToolTokens(tools)
: 0;
const endpoint = toolTokens > 0
? this.props.endpoint.cloneWithTokenOverride(
Math.max(1, Math.floor((this.props.endpoint.modelMaxPromptTokens - toolTokens) * 0.9)))
: this.props.endpoint;
Comment on lines +668 to +678
Copy link

Copilot AI Apr 6, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

modelMaxPromptTokens telemetry is documented as “the … budget used for the summarization prompt rendering”, but the value sent is this.props.endpoint.modelMaxPromptTokens (the pre-reservation budget). Since getSummary may clone the endpoint with a reduced token budget after reserving tool tokens, telemetry will be misleading. Consider reporting the effective budget actually used for rendering (e.g. the cloned endpoint’s modelMaxPromptTokens / computed message budget), or report both original and effective budgets.

Copilot uses AI. Check for mistakes.

let summarizationPrompt: ChatMessage[];
const associatedRequestId = this.props.promptContext.conversation?.getLatestTurn().id;
Expand Down Expand Up @@ -865,7 +876,8 @@ class ConversationHistorySummarizer {
"promptTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens, server side counted", "isMeasurement": true },
"promptCacheTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of prompt tokens hitting cache as reported by server", "isMeasurement": true },
"responseTokenCount": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "Number of generated tokens", "isMeasurement": true },
"source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the summarization was triggered as a background or foreground operation." }
"source": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "Whether the summarization was triggered as a background or foreground operation." },
"modelMaxPromptTokens": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The modelMaxPromptTokens budget used for the summarization prompt rendering." }
}
*/
this.telemetryService.sendMSFTTelemetryEvent('summarizedConversationHistory', {
Expand All @@ -891,6 +903,7 @@ class ConversationHistorySummarizer {
promptTokenCount: usage?.prompt_tokens,
promptCacheTokenCount: usage?.prompt_tokens_details?.cached_tokens,
responseTokenCount: usage?.completion_tokens,
modelMaxPromptTokens: this.props.endpoint.modelMaxPromptTokens,
});
}
}
Expand Down
47 changes: 47 additions & 0 deletions src/extension/prompts/node/agent/test/summarization.spec.tsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -440,6 +440,53 @@ suite('Agent Summarization', () => {
}
});

test('simple mode summarization with small token budget renders zero messages (repro for No messages provided)', async () => {
// Repro for: "Prompt failed validation with the reason: No messages provided"
//
// Root cause: when modelMaxPromptTokens is small enough that the summarization
// prompt content exceeds the budget, prompt-tsx prunes all child elements.
// After pruning, toChatMessages() silently skips messages whose content is
// empty (isEmpty check), producing an empty messages array — without throwing
// BudgetExceededError. The downstream makeChatRequest2 then hits the
// isValidChatPayload check: "No messages provided".
const instaService = accessor.get(IInstantiationService);
const endpoint = instaService.createInstance(MockEndpoint, 'claude-sonnet');
endpoint.modelMaxPromptTokens = 5; // So small that even a single short message cannot fit

const toolCallRounds = [
new ToolCallRound('ok', [createEditFileToolCall(1)]),
new ToolCallRound('ok 2', [createEditFileToolCall(2)]),
];

const turn = new Turn('turnId', { type: 'user', message: 'hello' });
const testConversation = new Conversation('sessionId', [turn]);

const promptContext: IBuildPromptContext = {
chatVariables: new ChatVariablesCollection([]),
history: [],
query: 'edit this file',
toolCallRounds,
toolCallResults: createEditFileToolResult(1, 2),
tools,
conversation: testConversation,
};

const baseProps = {
priority: 1,
endpoint,
location: ChatLocation.Panel,
promptContext,
maxToolResultLength: Infinity,
};

const propsInfo = instaService.createInstance(SummarizedConversationHistoryPropsBuilder).getProps(baseProps);
const renderer = PromptRenderer.create(instaService, endpoint, ConversationHistorySummarizationPrompt, { ...propsInfo.props, simpleMode: true });
const result = await renderer.render();

// prompt-tsx prunes all content and silently drops empty messages → 0 messages
expect(result.messages.length).toBe(0);
});

test('failure metadata on turn prevents repeated foreground summarization attempts', async () => {
// This test verifies the contract that agentIntent.ts relies on:
// after a foreground summarization failure, setting SummarizedConversationHistoryMetadata
Expand Down
Loading