PR

Author: pwang347

.github/workflows/chat-perf.yml

@@ -71,9 +71,9 @@ jobs:
             libnotify-bin libkrb5-dev \
             xvfb sqlite3 \
             libnss3 libatk1.0-0 libatk-bridge2.0-0 \
-            libcups2 libdrm2 libxcomposite1 libxdamage1 \
+            libcups2t64 libdrm2 libxcomposite1 libxdamage1 \
             libxrandr2 libgbm1 libpango-1.0-0 libcairo2 \
-            libasound2 libxshmfence1 libgtk-3-0
+            libasound2t64 libxshmfence1 libgtk-3-0
 
       - name: Install dependencies
         run: npm ci

build/filters.ts

@@ -162,6 +162,7 @@ export const copyrightFilter = Object.freeze<string[]>([
 	'**',
 	'!**/*.desktop',
 	'!**/*.json',
+	'!**/*.jsonc',
 	'!**/*.jsonl',
 	'!**/*.html',
 	'!**/*.template',

scripts/chat-simulation/common/perf-scenarios.js

@@ -15,7 +15,7 @@
 const path = require('path');
 const { ScenarioBuilder, registerScenario } = require('./mock-llm-server');
 
-const ROOT = path.join(__dirname, '..', '..', '..');
+const FIXTURES_DIR = path.join(__dirname, '..', 'fixtures');
 
 // -- Content-only scenarios ---------------------------------------------------
 
@@ -232,14 +232,14 @@ const TOOL_CALL_SCENARIOS = {
 	// a real agent gathering context before answering.
 	'tool-read-file': /** @type {import('./mock-llm-server').MultiTurnScenario} */ ((() => {
 		const filesToRead = [
-			'src/vs/base/common/lifecycle.ts',
-			'src/vs/base/common/event.ts',
-			'src/vs/base/common/uri.ts',
-			'src/vs/base/common/errors.ts',
-			'src/vs/base/common/async.ts',
-			'src/vs/base/common/strings.ts',
-			'src/vs/base/common/arrays.ts',
-			'src/vs/base/common/types.ts',
+			'lifecycle.ts',
+			'event.ts',
+			'uri.ts',
+			'errors.ts',
+			'async.ts',
+			'strings.ts',
+			'arrays.ts',
+			'types.ts',
 		];
 		// Round 1: parallel read of first 4 files
 		// Round 2: parallel read of next 4 files
@@ -251,14 +251,14 @@ const TOOL_CALL_SCENARIOS = {
 					kind: 'tool-calls',
 					toolCalls: filesToRead.slice(0, 4).map(f => ({
 						toolNamePattern: /read.?file/i,
-						arguments: { filePath: path.join(ROOT, f), startLine: 1, endLine: 50 },
+						arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 50 },
 					})),
 				},
 				{
 					kind: 'tool-calls',
 					toolCalls: filesToRead.slice(4).map(f => ({
 						toolNamePattern: /read.?file/i,
-						arguments: { filePath: path.join(ROOT, f), startLine: 1, endLine: 50 },
+						arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 50 },
 					})),
 				},
 				{
@@ -296,9 +296,9 @@ const TOOL_CALL_SCENARIOS = {
 	// a real agent reading context and making multiple edits.
 	'tool-edit-file': /** @type {import('./mock-llm-server').MultiTurnScenario} */ ((() => {
 		const readFiles = [
-			'src/vs/base/common/lifecycle.ts',
-			'src/vs/base/common/event.ts',
-			'src/vs/base/common/errors.ts',
+			'lifecycle.ts',
+			'event.ts',
+			'errors.ts',
 		];
 		return {
 			type: 'multi-turn',
@@ -308,7 +308,7 @@ const TOOL_CALL_SCENARIOS = {
 					kind: 'tool-calls',
 					toolCalls: readFiles.map(f => ({
 						toolNamePattern: /read.?file/i,
-						arguments: { filePath: path.join(ROOT, f), startLine: 1, endLine: 40 },
+						arguments: { filePath: path.join(FIXTURES_DIR, f), startLine: 1, endLine: 40 },
 					})),
 				},
 				// Round 2: edit 2 files in parallel
@@ -318,7 +318,7 @@ const TOOL_CALL_SCENARIOS = {
 						{
 							toolNamePattern: /replace.?string|apply.?patch|insert.?edit/i,
 							arguments: {
-								filePath: path.join(ROOT, 'src/vs/base/common/lifecycle.ts'),
+								filePath: path.join(FIXTURES_DIR, 'lifecycle.ts'),
 								oldString: '// perf-benchmark-marker',
 								newString: '// perf-benchmark-marker (updated)',
 								explanation: 'Update the benchmark marker comment in lifecycle.ts',
@@ -327,7 +327,7 @@ const TOOL_CALL_SCENARIOS = {
 						{
 							toolNamePattern: /replace.?string|apply.?patch|insert.?edit/i,
 							arguments: {
-								filePath: path.join(ROOT, 'src/vs/base/common/event.ts'),
+								filePath: path.join(FIXTURES_DIR, 'event.ts'),
 								oldString: '// perf-benchmark-marker',
 								newString: '// perf-benchmark-marker (updated)',
 								explanation: 'Update the benchmark marker comment in event.ts',
@@ -399,7 +399,7 @@ const MULTI_TURN_SCENARIOS = {
 					{
 						toolNamePattern: /read.?file/i,
 						arguments: {
-							filePath: path.join(ROOT, 'src/vs/base/common/lifecycle.ts'),
+							filePath: path.join(FIXTURES_DIR, 'lifecycle.ts'),
 							offset: 1,
 							limit: 50,
 						},

scripts/chat-simulation/common/utils.js

@@ -18,6 +18,22 @@ const { execSync, spawn } = require('child_process');
 const ROOT = path.join(__dirname, '..', '..', '..');
 const DATA_DIR = path.join(ROOT, '.chat-simulation-data');
 
+// -- Config loading ----------------------------------------------------------
+
+/** @param {string} text */
+function stripJsoncComments(text) { return text.replace(/\/\/.*/g, '').replace(/\/\*[\s\S]*?\*\//g, ''); }
+
+/**
+ * Load a namespaced section from config.jsonc.
+ * @param {string} section - Top-level key (e.g. 'perfRegression', 'memLeaks')
+ * @returns {Record<string, any>}
+ */
+function loadConfig(section) {
+	const raw = fs.readFileSync(path.join(__dirname, '..', 'config.jsonc'), 'utf-8');
+	const config = JSON.parse(stripJsoncComments(raw));
+	return config[section] ?? {};
+}
+
 // -- Electron path resolution ------------------------------------------------
 
 function getElectronPath() {
@@ -625,6 +641,7 @@ module.exports = {
 	ROOT,
 	DATA_DIR,
 	METRIC_DEFS,
+	loadConfig,
 	getElectronPath,
 	isVersionString,
 	resolveBuild,

scripts/chat-simulation/config.jsonc

@@ -0,0 +1,19 @@
+{
+	"perfRegression": {
+		// VS Code version, "insiders", or a commit hash (7-40 hex chars)
+		"baselineBuild": "1.115.0",
+
+		// Number of benchmark iterations per scenario
+		"runsPerScenario": 5,
+
+		// Fraction above baseline that triggers a regression (0.2 = 20%)
+		"regressionThreshold": 0.2
+	},
+	"memLeaks": {
+		// Number of chat messages to send during the leak check
+		"messages": 10,
+
+		// Max acceptable heap growth per message in MB
+		"leakThresholdMB": 2
+	}
+}

scripts/chat-simulation/fixtures/arrays.ts

@@ -0,0 +1,84 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// perf-benchmark-marker
+
+/**
+ * Fixture for chat-simulation benchmarks.
+ * Simplified from src/vs/base/common/arrays.ts for stable perf testing.
+ */
+
+export function coalesce<T>(array: ReadonlyArray<T | undefined | null>): T[] {
+	return array.filter((e): e is T => e !== undefined && e !== null);
+}
+
+export function groupBy<T>(data: ReadonlyArray<T>, groupFn: (element: T) => string): { [key: string]: T[] } {
+	const result: { [key: string]: T[] } = {};
+	for (const element of data) {
+		const key = groupFn(element);
+		(result[key] ??= []).push(element);
+	}
+	return result;
+}
+
+export function distinct<T>(array: ReadonlyArray<T>, keyFn: (t: T) => any = t => t): T[] {
+	const seen = new Set<any>();
+	return array.filter(element => {
+		const key = keyFn(element);
+		if (seen.has(key)) { return false; }
+		seen.add(key);
+		return true;
+	});
+}
+
+export function firstOrDefault<T>(array: ReadonlyArray<T>): T | undefined;
+export function firstOrDefault<T>(array: ReadonlyArray<T>, defaultValue: T): T;
+export function firstOrDefault<T>(array: ReadonlyArray<T>, defaultValue?: T): T | undefined {
+	return array.length > 0 ? array[0] : defaultValue;
+}
+
+export function lastOrDefault<T>(array: ReadonlyArray<T>): T | undefined;
+export function lastOrDefault<T>(array: ReadonlyArray<T>, defaultValue: T): T;
+export function lastOrDefault<T>(array: ReadonlyArray<T>, defaultValue?: T): T | undefined {
+	return array.length > 0 ? array[array.length - 1] : defaultValue;
+}
+
+export function binarySearch<T>(array: ReadonlyArray<T>, key: T, comparator: (a: T, b: T) => number): number {
+	let low = 0;
+	let high = array.length - 1;
+	while (low <= high) {
+		const mid = ((low + high) / 2) | 0;
+		const comp = comparator(array[mid], key);
+		if (comp < 0) { low = mid + 1; }
+		else if (comp > 0) { high = mid - 1; }
+		else { return mid; }
+	}
+	return -(low + 1);
+}
+
+export function insertSorted<T>(array: T[], element: T, comparator: (a: T, b: T) => number): void {
+	const idx = binarySearch(array, element, comparator);
+	const insertIdx = idx < 0 ? ~idx : idx;
+	array.splice(insertIdx, 0, element);
+}
+
+export function flatten<T>(arr: T[][]): T[] {
+	return ([] as T[]).concat(...arr);
+}
+
+export function range(to: number): number[];
+export function range(from: number, to: number): number[];
+export function range(arg: number, to?: number): number[] {
+	const from = to !== undefined ? arg : 0;
+	const end = to !== undefined ? to : arg;
+	const result: number[] = [];
+	for (let i = from; i < end; i++) { result.push(i); }
+	return result;
+}
+
+export function tail<T>(array: T[]): [T[], T] {
+	if (array.length === 0) { throw new Error('Invalid tail call'); }
+	return [array.slice(0, array.length - 1), array[array.length - 1]];
+}

scripts/chat-simulation/fixtures/async.ts

@@ -0,0 +1,132 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+// perf-benchmark-marker
+
+/**
+ * Fixture for chat-simulation benchmarks.
+ * Simplified from src/vs/base/common/async.ts for stable perf testing.
+ */
+
+import { IDisposable } from './lifecycle';
+import { CancellationError } from './errors';
+
+export class Throttler {
+	private activePromise: Promise<any> | null = null;
+	private queuedPromiseFactory: (() => Promise<any>) | null = null;
+
+	queue<T>(promiseFactory: () => Promise<T>): Promise<T> {
+		if (this.activePromise) {
+			this.queuedPromiseFactory = promiseFactory;
+			return this.activePromise as Promise<T>;
+		}
+		this.activePromise = promiseFactory();
+		return this.activePromise.finally(() => {
+			this.activePromise = null;
+			if (this.queuedPromiseFactory) {
+				const factory = this.queuedPromiseFactory;
+				this.queuedPromiseFactory = null;
+				return this.queue(factory);
+			}
+		});
+	}
+}
+
+export class Delayer<T> implements IDisposable {
+	private timeout: any;
+	private task: (() => T | Promise<T>) | null = null;
+
+	constructor(public defaultDelay: number) { }
+
+	trigger(task: () => T | Promise<T>, delay: number = this.defaultDelay): Promise<T> {
+		this.task = task;
+		this.cancelTimeout();
+		return new Promise<T>((resolve, reject) => {
+			this.timeout = setTimeout(() => {
+				this.timeout = null;
+				try { resolve(this.task!()); } catch (e) { reject(e); }
+				this.task = null;
+			}, delay);
+		});
+	}
+
+	private cancelTimeout(): void {
+		if (this.timeout !== null) {
+			clearTimeout(this.timeout);
+			this.timeout = null;
+		}
+	}
+
+	dispose(): void {
+		this.cancelTimeout();
+	}
+}
+
+export class RunOnceScheduler implements IDisposable {
+	private runner: (() => void) | null;
+	private timeout: any;
+
+	constructor(runner: () => void, private delay: number) {
+		this.runner = runner;
+	}
+
+	schedule(delay = this.delay): void {
+		this.cancel();
+		this.timeout = setTimeout(() => {
+			this.timeout = null;
+			this.runner?.();
+		}, delay);
+	}
+
+	cancel(): void {
+		if (this.timeout !== null) {
+			clearTimeout(this.timeout);
+			this.timeout = null;
+		}
+	}
+
+	isScheduled(): boolean { return this.timeout !== null; }
+
+	dispose(): void {
+		this.cancel();
+		this.runner = null;
+	}
+}
+
+export class Queue<T> {
+	private readonly queue: Array<() => Promise<T>> = [];
+	private running = false;
+
+	async enqueue(factory: () => Promise<T>): Promise<T> {
+		return new Promise<T>((resolve, reject) => {
+			this.queue.push(() => factory().then(resolve, reject));
+			if (!this.running) { this.processQueue(); }
+		});
+	}
+
+	private async processQueue(): Promise<void> {
+		this.running = true;
+		while (this.queue.length > 0) {
+			const task = this.queue.shift()!;
+			await task();
+		}
+		this.running = false;
+	}
+
+	get size(): number { return this.queue.length; }
+}
+
+export function timeout(millis: number): Promise<void> {
+	return new Promise<void>(resolve => setTimeout(resolve, millis));
+}
+
+export async function retry<T>(task: () => Promise<T>, delay: number, retries: number): Promise<T> {
+	let lastError: Error | undefined;
+	for (let i = 0; i < retries; i++) {
+		try { return await task(); }
+		catch (error) { lastError = error as Error; await timeout(delay); }
+	}
+	throw lastError;
+}