Cleanup auto with better telemetry + newer API shape (#4374)

* Cleanup auto with better telemetry + newer API shape

* Update src/platform/endpoint/node/automodeService.ts

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* Fix tests

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

Author: lramos15

src/platform/endpoint/node/automodeService.ts

@@ -194,7 +194,7 @@ export class AutomodeService extends Disposable implements IAutomodeService {
 			throw new Error('No auto mode endpoints provided.');
 		}
 
-		const conversationId = getConversationId(chatRequest);
+		const conversationId = chatRequest?.sessionResource?.toString() ?? chatRequest?.sessionId ?? 'unknown';
 		const entry = this._autoModelCache.get(conversationId);
 
 		// Acquire token bank: reuse from cache or take from reserve pool
@@ -212,34 +212,63 @@ export class AutomodeService extends Disposable implements IAutomodeService {
 
 		let selectedModel: IChatEndpoint | undefined;
 		let lastRoutedPrompt = entry?.lastRoutedPrompt;
+		let routerFallbackReason: string | undefined;
 
 		// Try router-based model selection (skip for vision requests to avoid unnecessary latency)
-		if (!hasImage(chatRequest) && this._isRouterEnabled(chatRequest)) {
+		if (hasImage(chatRequest)) {
+			routerFallbackReason = 'hasImage';
+		} else if (this._isRouterEnabled(chatRequest)) {
 			const prompt = chatRequest?.prompt?.trim();
 			// Only route when the prompt has changed since the last decision, to avoid
 			// redundant calls during tool-calling iterations with the same prompt.
-			if (prompt?.length && (!entry || entry.lastRoutedPrompt !== prompt)) {
+			if (!prompt?.length) {
+				routerFallbackReason = 'emptyPrompt';
+			} else if (entry && entry.lastRoutedPrompt === prompt) {
+				// Prompt hasn't changed since the last router decision — skip the
+				// router call but fall through to the endpoint reuse/recreate path
+				// so the endpoint is rebuilt if the session token has changed.
+				// Router fallback reason isn't set here because we don't want telemetry for this case
+			} else {
 				try {
 					const result = await this._routerDecisionFetcher.getRouterDecision(prompt, token.session_token, token.available_models);
-					if (entry?.endpoint) {
+					if (!result.candidate_models.length) {
+						routerFallbackReason = 'emptyCandidateList';
+					} else if (entry?.endpoint) {
 						// Prefer a same-provider model from the router's candidate list
 						selectedModel = this._findSameProviderModel(entry.endpoint.modelProvider, result.candidate_models, knownEndpoints);
 					}
-					selectedModel ??= knownEndpoints.find(e => e.model === result.chosen_model);
+					if (!routerFallbackReason) {
+						selectedModel ??= knownEndpoints.find(e => e.model === result.candidate_models[0]);
+					}
 					if (selectedModel) {
 						lastRoutedPrompt = prompt;
 						if (result.sticky_override) {
-							this._logService.trace(`[AutomodeService] Sticky routing override: confidence=${(result.confidence * 100).toFixed(1)}%, label=${result.predicted_label}, router_model=${result.chosen_model}, actual_model=${selectedModel.model}`);
+							this._logService.trace(`[AutomodeService] Sticky routing override: confidence=${(result.confidence * 100).toFixed(1)}%, label=${result.predicted_label}, router_model=${result.candidate_models[0]}, actual_model=${selectedModel.model}`);
 						}
+					} else {
+						routerFallbackReason = 'noMatchingEndpoint';
 					}
 				} catch (e) {
 					this._logService.error(`Failed to get routed model for conversation ${conversationId}:`, (e as Error).message);
+					routerFallbackReason = 'routerError';
 				}
 			}
 		}
 
 		// Default model selection when router was skipped or failed
 		if (!selectedModel) {
+			if (routerFallbackReason) {
+				/* __GDPR__
+					"automode.routerFallback" : {
+						"owner": "lramos15",
+						"comment": "Reports when the auto mode router is skipped or fails and falls back to default model selection",
+						"reason": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "comment": "The reason the router was skipped or failed (hasImage, noMatchingEndpoint, routerError)" }
+					}
+				*/
+				this._telemetryService.sendMSFTTelemetryEvent('automode.routerFallback', {
+					reason: routerFallbackReason,
+				});
+			}
 			// Pick a model: prefer same provider when refreshing, otherwise first available
 			if (entry?.endpoint) {
 				selectedModel = this._findSameProviderModel(entry.endpoint.modelProvider, token.available_models, knownEndpoints);
@@ -336,18 +365,6 @@ export class AutomodeService extends Disposable implements IAutomodeService {
 	}
 }
 
-/**
- * Get the conversation ID from the chat request. This is representative of a single chat thread
- * @param chatRequest The chat request object.
- * @returns The conversation ID or 'unknown' if not available.
- */
-function getConversationId(chatRequest: ChatRequest | undefined): string {
-	if (!chatRequest) {
-		return 'unknown';
-	}
-	return chatRequest?.sessionId || 'unknown';
-}
-
 function hasImage(chatRequest: ChatRequest | undefined): boolean {
 	if (!chatRequest || !chatRequest.references) {
 		return false;

src/platform/endpoint/node/routerDecisionFetcher.ts

@@ -6,7 +6,6 @@
 import { RequestType } from '@vscode/copilot-api';
 import { Codicon } from '../../../util/vs/base/common/codicons';
 import { IAuthenticationService } from '../../authentication/common/authentication';
-import { IValidator, vArray, vBoolean, vEnum, vNumber, vObj, vRequired, vString } from '../../configuration/common/validator';
 import { ILogService } from '../../log/common/logService';
 import { Response } from '../../networking/common/fetcherService';
 import { IRequestLogger, LoggedRequestKind } from '../../requestLogger/node/requestLogger';
@@ -17,7 +16,6 @@ export interface RouterDecisionResponse {
 	predicted_label: 'needs_reasoning' | 'no_reasoning';
 	confidence: number;
 	latency_ms: number;
-	chosen_model: string;
 	candidate_models: string[];
 	scores: {
 		needs_reasoning: number;
@@ -26,18 +24,6 @@ export interface RouterDecisionResponse {
 	sticky_override?: boolean;
 }
 
-const routerDecisionResponseValidator: IValidator<RouterDecisionResponse> = vObj({
-	predicted_label: vRequired(vEnum('needs_reasoning', 'no_reasoning')),
-	confidence: vRequired(vNumber()),
-	latency_ms: vRequired(vNumber()),
-	chosen_model: vRequired(vString()),
-	candidate_models: vRequired(vArray(vString())),
-	scores: vRequired(vObj({
-		needs_reasoning: vRequired(vNumber()),
-		no_reasoning: vRequired(vNumber())
-	})),
-	sticky_override: vBoolean()
-});
 
 /**
  * Fetches routing decisions from a classification API to determine which model should handle a query.
@@ -75,12 +61,9 @@ export class RouterDecisionFetcher {
 		}
 
 		const text = await response.text();
-		const { content: result, error: validationError } = routerDecisionResponseValidator.validate(JSON.parse(text));
-		if (validationError) {
-			throw new Error(`Invalid router decision response: ${validationError.message}`);
-		}
+		const result: RouterDecisionResponse = JSON.parse(text);
 		const e2eLatencyMs = Date.now() - startTime;
-		this._logService.trace(`[RouterDecisionFetcher] Prediction: ${result.predicted_label}, model: ${result.chosen_model} (confidence: ${(result.confidence * 100).toFixed(1)}%, scores: needs_reasoning=${(result.scores.needs_reasoning * 100).toFixed(1)}%, no_reasoning=${(result.scores.no_reasoning * 100).toFixed(1)}%) (latency_ms: ${result.latency_ms}, e2e_latency_ms: ${e2eLatencyMs}, candidate models: ${result.candidate_models.join(', ')}, sticky_override: ${result.sticky_override ?? false})`);
+		this._logService.trace(`[RouterDecisionFetcher] Prediction: ${result.predicted_label}, (confidence: ${(result.confidence * 100).toFixed(1)}%, scores: needs_reasoning=${(result.scores.needs_reasoning * 100).toFixed(1)}%, no_reasoning=${(result.scores.no_reasoning * 100).toFixed(1)}%) (latency_ms: ${result.latency_ms}, e2e_latency_ms: ${e2eLatencyMs}, candidate models: ${result.candidate_models.join(', ')}, sticky_override: ${result.sticky_override ?? false})`);
 
 		this._requestLogger.addEntry({
 			type: LoggedRequestKind.MarkdownContentRequest,
@@ -91,10 +74,9 @@ export class RouterDecisionFetcher {
 				`# Auto Mode Router Decision`,
 				`## Result`,
 				`- **Predicted Label**: ${result.predicted_label}`,
-				`- **Chosen Model**: ${result.chosen_model}`,
 				`- **Confidence**: ${(result.confidence * 100).toFixed(1)}%`,
-			`- **Sticky Override**: ${result.sticky_override ?? false}`,
-			`## Scores`,
+				`- **Sticky Override**: ${result.sticky_override ?? false}`,
+				`## Scores`,
 				`- **Needs Reasoning**: ${(result.scores.needs_reasoning * 100).toFixed(1)}%`,
 				`- **No Reasoning**: ${(result.scores.no_reasoning * 100).toFixed(1)}%`,
 				`## Latency`,
@@ -112,7 +94,6 @@ export class RouterDecisionFetcher {
 				"owner": "lramos15",
 				"comment": "Reports the routing decision made by the auto mode router API",
 				"predictedLabel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The predicted classification label (needs_reasoning or no_reasoning)" },
-				"chosenModel": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "comment": "The model selected by the router" },
 				"confidence": { "classification": "SystemMetaData", "purpose": "FeatureInsight", "isMeasurement": true, "comment": "The confidence score of the routing decision" },
 				"latencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The latency of the router API call in milliseconds" },
 				"e2eLatencyMs": { "classification": "SystemMetaData", "purpose": "PerformanceAndHealth", "isMeasurement": true, "comment": "The end-to-end latency of the router request in milliseconds, including network overhead" }
@@ -121,7 +102,6 @@ export class RouterDecisionFetcher {
 		this._telemetryService.sendMSFTTelemetryEvent('automode.routerDecision',
 			{
 				predictedLabel: result.predicted_label,
-				chosenModel: result.chosen_model,
 			},
 			{
 				confidence: result.confidence,