Skip to content

Commit ae07692

Browse files
Copilotjoshspicer
Copilot
and
joshspicer
authored
Implement comprehensive telemetry tracking for remote Copilot feature (#7179)
* Initial plan * Implement telemetry for remote Copilot feature Co-authored-by: joshspicer <23246594+joshspicer@users.noreply.github.com> * Add comprehensive tests for remote Copilot telemetry implementation Co-authored-by: joshspicer <23246594+joshspicer@users.noreply.github.com> * updates * pass telemetry * remove * tidy up * add onExp * add tool general error tracking * Fix TypeScript compilation errors: add followup to destructuring Co-authored-by: joshspicer <23246594+joshspicer@users.noreply.github.com> --------- Co-authored-by: copilot-swe-agent[bot] <198982749+Copilot@users.noreply.github.com> Co-authored-by: joshspicer <23246594+joshspicer@users.noreply.github.com>
1 parent 70bb6c9 commit ae07692

File tree

10 files changed

+140
-49
lines changed

10 files changed

+140
-49
lines changed

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -516,7 +516,8 @@
516516
"default": false,
517517
"markdownDescription": "%githubPullRequests.codingAgent.uiIntegration.description%",
518518
"tags": [
519-
"experimental"
519+
"experimental",
520+
"onExP"
520521
]
521522
},
522523
"githubPullRequests.experimental.notificationsMarkPullRequests": {

‎src/extension.ts‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -239,7 +239,7 @@ async function init(
239239

240240
registerPostCommitCommandsProvider(reposManager, git);
241241

242-
initChat(context, credentialStore, reposManager, copilotRemoteAgentManager);
242+
initChat(context, credentialStore, reposManager, copilotRemoteAgentManager, telemetry);
243243
context.subscriptions.push(vscode.window.registerUriHandler(new UriHandler(reposManager, telemetry, context)));
244244

245245
// Make sure any compare changes tabs, which come from the create flow, are closed.
@@ -250,11 +250,11 @@ async function init(
250250
telemetry.sendTelemetryEvent('startup');
251251
}
252252

253-
function initChat(context: vscode.ExtensionContext, credentialStore: CredentialStore, reposManager: RepositoriesManager, copilotRemoteManager: CopilotRemoteAgentManager) {
253+
function initChat(context: vscode.ExtensionContext, credentialStore: CredentialStore, reposManager: RepositoriesManager, copilotRemoteManager: CopilotRemoteAgentManager, telemetry: ExperimentationTelemetry) {
254254
const createParticipant = () => {
255255
const chatParticipantState = new ChatParticipantState();
256256
context.subscriptions.push(new ChatParticipant(context, chatParticipantState));
257-
registerTools(context, credentialStore, reposManager, chatParticipantState, copilotRemoteManager);
257+
registerTools(context, credentialStore, reposManager, chatParticipantState, copilotRemoteManager, telemetry);
258258
};
259259

260260
const chatEnabled = () => vscode.workspace.getConfiguration(PR_SETTINGS_NAMESPACE).get<boolean>(EXPERIMENTAL_CHAT, false);
@@ -408,7 +408,7 @@ async function deferredActivate(context: vscode.ExtensionContext, showPRControll
408408

409409
Logger.debug('Creating tree view.', 'Activation');
410410

411-
const copilotRemoteAgentManager = new CopilotRemoteAgentManager(credentialStore, reposManager);
411+
const copilotRemoteAgentManager = new CopilotRemoteAgentManager(credentialStore, reposManager, telemetry);
412412
context.subscriptions.push(copilotRemoteAgentManager);
413413

414414
const prTree = new PullRequestsTreeDataProvider(telemetry, context, reposManager, copilotRemoteAgentManager);

‎src/github/copilotApi.ts‎

Lines changed: 47 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import JSZip from 'jszip';
88
import * as vscode from 'vscode';
99
import { AuthProvider } from '../common/authentication';
1010
import Logger from '../common/logger';
11+
import { ITelemetry } from '../common/telemetry';
1112
import { CredentialStore } from './credentials';
1213
import { LoggingOctokit } from './loggingOctokit';
1314
import { hasEnterpriseUri } from './utils';
@@ -37,7 +38,7 @@ export interface RemoteAgentJobResponse {
3738
export class CopilotApi {
3839
protected static readonly ID = 'copilotApi';
3940

40-
constructor(private octokit: LoggingOctokit, private token: string) { }
41+
constructor(private octokit: LoggingOctokit, private token: string, private telemetry: ITelemetry) { }
4142

4243
private get baseUrl(): string {
4344
return 'https://api.githubcopilot.com';
@@ -50,27 +51,52 @@ export class CopilotApi {
5051
): Promise<RemoteAgentJobResponse> {
5152
const repoSlug = `${owner}/${name}`;
5253
const apiUrl = `${this.baseUrl}/agents/swe/v0/jobs/${repoSlug}`;
53-
const response = await fetch(apiUrl, {
54-
method: 'POST',
55-
headers: {
56-
'Copilot-Integration-Id': 'copilot-developer-dev',
57-
'Authorization': `Bearer ${this.token}`,
58-
'Content-Type': 'application/json',
59-
'Accept': 'application/json'
60-
},
61-
body: JSON.stringify(payload)
62-
});
63-
if (!response.ok) {
64-
throw new Error(await this.formatRemoteAgentJobError(response.status, repoSlug, response));
54+
let status: number | undefined;
55+
try {
56+
const response = await fetch(apiUrl, {
57+
method: 'POST',
58+
headers: {
59+
'Copilot-Integration-Id': 'copilot-developer-dev',
60+
'Authorization': `Bearer ${this.token}`,
61+
'Content-Type': 'application/json',
62+
'Accept': 'application/json'
63+
},
64+
body: JSON.stringify(payload)
65+
});
66+
67+
status = response.status;
68+
if (!response.ok) {
69+
throw new Error(await this.formatRemoteAgentJobError(status, repoSlug, response));
70+
}
71+
72+
const data = await response.json();
73+
this.validateRemoteAgentJobResponse(data);
74+
/*
75+
__GDPR__
76+
"remoteAgent.postRemoteAgentJob" : {
77+
"status" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
78+
}
79+
*/
80+
this.telemetry.sendTelemetryEvent('remoteAgent.postRemoteAgentJob', {
81+
status: status.toString(),
82+
});
83+
return data;
84+
} catch (error) {
85+
/* __GDPR__
86+
"remoteAgent.postRemoteAgentJob" : {
87+
"status" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
88+
}
89+
*/
90+
this.telemetry.sendTelemetryErrorEvent('remoteAgent.postRemoteAgentJob', {
91+
status: status?.toString() || '999',
92+
});
93+
throw error;
6594
}
66-
const data = await response.json();
67-
this.validateRemoteAgentJobResponse(data);
68-
return data;
6995
}
7096

71-
7297
// https://github.com/github/sweagentd/blob/371ea6db280b9aecf790ccc20660e39a7ecb8d1c/internal/api/jobapi/handler.go#L110-L120
7398
private async formatRemoteAgentJobError(status: number, repoSlug: string, response: Response): Promise<string> {
99+
Logger.error(`Error in remote agent job: ${await response.text()}`, CopilotApi.ID);
74100
switch (status) {
75101
case 400:
76102
return vscode.l10n.t('Bad request');
@@ -85,10 +111,9 @@ export class CopilotApi {
85111
case 409:
86112
return vscode.l10n.t('A coding agent pull request already exists');
87113
case 500:
88-
Logger.error(`Server error in remote agent job: ${await response.text()}`, CopilotApi.ID);
89-
return vscode.l10n.t('Server error. Please try again later.');
114+
return vscode.l10n.t('Server error. Please see logs for details.');
90115
default:
91-
return vscode.l10n.t('Error: {0}', status);
116+
return vscode.l10n.t('Error: {0}. Please see logs for details', status);
92117
}
93118
}
94119

@@ -200,7 +225,7 @@ export interface SessionInfo {
200225
error: string | null;
201226
}
202227

203-
export async function getCopilotApi(credentialStore: CredentialStore, authProvider?: AuthProvider): Promise<CopilotApi | undefined> {
228+
export async function getCopilotApi(credentialStore: CredentialStore, telemetry: ITelemetry, authProvider?: AuthProvider): Promise<CopilotApi | undefined> {
204229
if (!authProvider) {
205230
if (credentialStore.isAuthenticated(AuthProvider.githubEnterprise) && hasEnterpriseUri()) {
206231
authProvider = AuthProvider.githubEnterprise;
@@ -217,5 +242,5 @@ export async function getCopilotApi(credentialStore: CredentialStore, authProvid
217242
}
218243

219244
const { token } = await github.octokit.api.auth() as { token: string };
220-
return new CopilotApi(github.octokit, token);
245+
return new CopilotApi(github.octokit, token, telemetry);
221246
}

‎src/github/copilotRemoteAgent.ts‎

Lines changed: 51 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,16 @@
55

66
import vscode from 'vscode';
77
import { Repository } from '../api/api';
8-
import { AuthProvider } from '../common/authentication';
98
import { COPILOT_LOGINS } from '../common/copilot';
109
import { commands } from '../common/executeCommands';
1110
import { Disposable } from '../common/lifecycle';
1211
import Logger from '../common/logger';
1312
import { GitHubRemote } from '../common/remote';
1413
import { CODING_AGENT, CODING_AGENT_AUTO_COMMIT_AND_PUSH, CODING_AGENT_ENABLED } from '../common/settingKeys';
14+
import { ITelemetry } from '../common/telemetry';
1515
import { toOpenPullRequestWebviewUri } from '../common/uri';
1616
import { OctokitCommon } from './common';
17-
import { CopilotApi, RemoteAgentJobPayload, SessionInfo } from './copilotApi';
17+
import { CopilotApi, getCopilotApi, RemoteAgentJobPayload, SessionInfo } from './copilotApi';
1818
import { CopilotPRWatcher, CopilotStateModel } from './copilotPrWatcher';
1919
import { CredentialStore } from './credentials';
2020
import { FolderRepositoryManager } from './folderRepositoryManager';
@@ -59,7 +59,7 @@ export class CopilotRemoteAgentManager extends Disposable {
5959
private readonly _onDidCreatePullRequest = this._register(new vscode.EventEmitter<number>());
6060
readonly onDidCreatePullRequest = this._onDidCreatePullRequest.event;
6161

62-
constructor(private credentialStore: CredentialStore, public repositoriesManager: RepositoriesManager) {
62+
constructor(private credentialStore: CredentialStore, public repositoriesManager: RepositoriesManager, private telemetry: ITelemetry) {
6363
super();
6464
this._register(this.credentialStore.onDidChangeSessions((e: vscode.AuthenticationSessionsChangeEvent) => {
6565
if (e.provider.id === 'github') {
@@ -104,12 +104,7 @@ export class CopilotRemoteAgentManager extends Disposable {
104104
}
105105

106106
private async initializeCopilotApi(): Promise<CopilotApi | undefined> {
107-
const gh = await this.credentialStore.getHubOrLogin(AuthProvider.github);
108-
const { token } = await gh?.octokit.api.auth() as { token: string };
109-
if (!token || !gh?.octokit) {
110-
return;
111-
}
112-
return new CopilotApi(gh.octokit, token);
107+
return await getCopilotApi(this.credentialStore, this.telemetry);
113108
}
114109

115110
enabled(): boolean {
@@ -264,21 +259,48 @@ export class CopilotRemoteAgentManager extends Disposable {
264259
if (!args) {
265260
return;
266261
}
262+
const { userPrompt, summary, source, followup } = args;
263+
264+
/* __GDPR__
265+
"remoteAgent.command.args" : {
266+
"source" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
267+
"isFollowup" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
268+
"userPromptLength" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
269+
"summaryLength" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
270+
}
271+
*/
272+
this.telemetry.sendTelemetryEvent('remoteAgent.command.args', {
273+
source: source?.toString() || 'unknown',
274+
isFollowup: !!followup ? 'true' : 'false',
275+
userPromptLength: userPrompt.length.toString(),
276+
summaryLength: summary ? summary.length.toString() : '0'
277+
});
267278

268-
const { userPrompt, summary, source } = args;
269279
if (!userPrompt || userPrompt.trim().length === 0) {
270280
return;
271281
}
272282

273283
const repoInfo = await this.repoInfo();
274284
if (!repoInfo) {
285+
/* __GDPR__
286+
"remoteAgent.command.result" : {
287+
"reason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
288+
}
289+
*/
290+
this.telemetry.sendTelemetryErrorEvent('remoteAgent.command.result', { reason: 'noRepositoryInfo' });
275291
return;
276292
}
277293
const { repository, owner, repo } = repoInfo;
278294

279295
const repoName = `${owner}/${repo}`;
280296
const hasChanges = repository.state.workingTreeChanges.length > 0 || repository.state.indexChanges.length > 0;
281297
const learnMoreCb = async () => {
298+
/* __GDPR__
299+
"remoteAgent.command.result" : {
300+
"reason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
301+
}
302+
*/
303+
this.telemetry.sendTelemetryErrorEvent('remoteAgent.command.result', { reason: 'learnMore' });
282304
vscode.env.openExternal(vscode.Uri.parse('https://docs.github.com/copilot/using-github-copilot/coding-agent'));
283305
};
284306

@@ -299,6 +321,12 @@ export class CopilotRemoteAgentManager extends Disposable {
299321
);
300322

301323
if (!modalResult) {
324+
/* __GDPR__
325+
"remoteAgent.command.result" : {
326+
"reason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
327+
}
328+
*/
329+
this.telemetry.sendTelemetryErrorEvent('remoteAgent.command.result', { reason: 'cancel' });
302330
return;
303331
}
304332

@@ -338,11 +366,24 @@ export class CopilotRemoteAgentManager extends Disposable {
338366
);
339367

340368
if (result.state !== 'success') {
369+
/* __GDPR__
370+
"remoteAgent.command.result" : {
371+
"reason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
372+
}
373+
*/
374+
this.telemetry.sendTelemetryErrorEvent('remoteAgent.command.result', { reason: 'invocationFailure' });
341375
vscode.window.showErrorMessage(result.error);
342376
return;
343377
}
344378

345379
const { webviewUri, link, number } = result;
380+
381+
this.telemetry.sendTelemetryEvent('remoteAgent.command', {
382+
source: source || 'unknown',
383+
hasFollowup: (!!followup).toString(),
384+
outcome: 'success'
385+
});
386+
346387
vscode.commands.executeCommand('vscode.open', webviewUri);
347388

348389
// allow-any-unicode-next-line

‎src/github/pullRequestOverview.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -497,7 +497,7 @@ export class PullRequestOverviewPanel extends IssueOverviewPanel<PullRequestMode
497497
if (message.args.event !== EventType.CopilotStarted) {
498498
return this._replyMessage(message, { success: false, error: 'Invalid event type' });
499499
} else {
500-
const copilotApi = await getCopilotApi(this._folderRepositoryManager.credentialStore, this._item.remote.authProviderId);
500+
const copilotApi = await getCopilotApi(this._folderRepositoryManager.credentialStore, this._telemetry, this._item.remote.authProviderId);
501501
if (copilotApi) {
502502
const session = (await copilotApi.getAllSessions(this._item.id))[0];
503503
if (session.state !== 'completed') {

‎src/lm/tools/copilotRemoteAgentTool.ts‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55

66

77
import * as vscode from 'vscode';
8+
import { ITelemetry } from '../../common/telemetry';
89
import { CopilotRemoteAgentManager } from '../../github/copilotRemoteAgent';
910

1011
export interface CopilotRemoteAgentToolParameters {
@@ -22,7 +23,7 @@ export interface CopilotRemoteAgentToolParameters {
2223
export class CopilotRemoteAgentTool implements vscode.LanguageModelTool<CopilotRemoteAgentToolParameters> {
2324
public static readonly toolId = 'github-pull-request_copilot-coding-agent';
2425

25-
constructor(private manager: CopilotRemoteAgentManager) { }
26+
constructor(private manager: CopilotRemoteAgentManager, private telemetry: ITelemetry) { }
2627

2728
async prepareInvocation(options: vscode.LanguageModelToolInvocationPrepareOptions<CopilotRemoteAgentToolParameters>): Promise<vscode.PreparedToolInvocation> {
2829
const { title, existingPullRequest } = options.input;
@@ -35,6 +36,12 @@ export class CopilotRemoteAgentTool implements vscode.LanguageModelTool<CopilotR
3536

3637
const targetRepo = await this.manager.repoInfo();
3738
const autoPushEnabled = this.manager.autoCommitAndPushEnabled();
39+
40+
/* __GDPR__
41+
"remoteAgent.tool.prepare" : {}
42+
*/
43+
this.telemetry.sendTelemetryEvent('copilot.remoteAgent.tool.prepare', {});
44+
3845
return {
3946
pastTenseMessage: vscode.l10n.t('Launched coding agent'),
4047
invocationMessage: vscode.l10n.t('Launching coding agent'),
@@ -63,6 +70,17 @@ export class CopilotRemoteAgentTool implements vscode.LanguageModelTool<CopilotR
6370
]);
6471
}
6572

73+
/* __GDPR__
74+
"remoteAgent.tool.invoke" : {
75+
"hasExistingPR" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" },
76+
"hasBody" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
77+
},
78+
*/
79+
this.telemetry.sendTelemetryEvent('copilot.remoteAgent.tool.invoke', {
80+
hasExistingPR: existingPullRequest ? 'true' : 'false',
81+
hasBody: body ? 'true' : 'false'
82+
});
83+
6684
if (existingPullRequest) {
6785
const pullRequestNumber = parseInt(existingPullRequest, 10);
6886
if (isNaN(pullRequestNumber)) {
@@ -78,6 +96,12 @@ export class CopilotRemoteAgentTool implements vscode.LanguageModelTool<CopilotR
7896

7997
const result = await this.manager.invokeRemoteAgent(title, body);
8098
if (result.state === 'error') {
99+
/* __GDPR__
100+
"remoteAgent.tool.invoke" : {
101+
"reason" : { "classification": "SystemMetaData", "purpose": "FeatureInsight" }
102+
}
103+
*/
104+
this.telemetry.sendTelemetryErrorEvent('copilot.remoteAgent.tool.error', { reason: 'invocationError' });
81105
throw new Error(result.error);
82106
}
83107
return new vscode.LanguageModelToolResult([

‎src/lm/tools/tools.ts‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
'use strict';
66

77
import * as vscode from 'vscode';
8+
import { ITelemetry } from '../../common/telemetry';
89
import { CopilotRemoteAgentManager } from '../../github/copilotRemoteAgent';
910
import { CredentialStore } from '../../github/credentials';
1011
import { RepositoriesManager } from '../../github/repositoriesManager';
@@ -19,12 +20,12 @@ import { SuggestFixTool } from './suggestFixTool';
1920
import { IssueSummarizationTool } from './summarizeIssueTool';
2021
import { NotificationSummarizationTool } from './summarizeNotificationsTool';
2122

22-
export function registerTools(context: vscode.ExtensionContext, credentialStore: CredentialStore, repositoriesManager: RepositoriesManager, chatParticipantState: ChatParticipantState, copilotRemoteAgentManager: CopilotRemoteAgentManager) {
23+
export function registerTools(context: vscode.ExtensionContext, credentialStore: CredentialStore, repositoriesManager: RepositoriesManager, chatParticipantState: ChatParticipantState, copilotRemoteAgentManager: CopilotRemoteAgentManager, telemetry: ITelemetry) {
2324
registerFetchingTools(context, credentialStore, repositoriesManager, chatParticipantState);
2425
registerSummarizationTools(context);
2526
registerSuggestFixTool(context, credentialStore, repositoriesManager, chatParticipantState);
2627
registerSearchTools(context, credentialStore, repositoriesManager, chatParticipantState);
27-
registerCopilotAgentTools(context, copilotRemoteAgentManager);
28+
registerCopilotAgentTools(context, copilotRemoteAgentManager, telemetry);
2829
context.subscriptions.push(vscode.lm.registerTool(ActivePullRequestTool.toolId, new ActivePullRequestTool(repositoriesManager, copilotRemoteAgentManager)));
2930
}
3031

@@ -42,8 +43,8 @@ function registerSuggestFixTool(context: vscode.ExtensionContext, credentialStor
4243
context.subscriptions.push(vscode.lm.registerTool(SuggestFixTool.toolId, new SuggestFixTool(credentialStore, repositoriesManager, chatParticipantState)));
4344
}
4445

45-
function registerCopilotAgentTools(context: vscode.ExtensionContext, copilotRemoteAgentManager: CopilotRemoteAgentManager) {
46-
context.subscriptions.push(vscode.lm.registerTool(CopilotRemoteAgentTool.toolId, new CopilotRemoteAgentTool(copilotRemoteAgentManager)));
46+
function registerCopilotAgentTools(context: vscode.ExtensionContext, copilotRemoteAgentManager: CopilotRemoteAgentManager, telemetry: ITelemetry) {
47+
context.subscriptions.push(vscode.lm.registerTool(CopilotRemoteAgentTool.toolId, new CopilotRemoteAgentTool(copilotRemoteAgentManager, telemetry)));
4748
}
4849

4950
function registerSearchTools(context: vscode.ExtensionContext, credentialStore: CredentialStore, repositoriesManager: RepositoriesManager, chatParticipantState: ChatParticipantState) {

0 commit comments

Comments
 (0)