feat: Download content exclusions from GitHub

Author: dividedmind

src/lib/exclusionUtils.ts

@@ -0,0 +1,75 @@
+import { warn } from 'node:console';
+import { writeFile } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+import { isNativeError } from 'node:util/types';
+
+import vscode from 'vscode';
+
+import fetchGHExclusions from './fetchGHExclusions';
+
+const exclusionPath = join(homedir(), '.appmap', 'navie', 'global-ignore');
+
+export async function downloadContentExclusions(githubToken: string) {
+  const exclusions = await fetchGHExclusions(githubToken, 'all');
+  const paths = exclusions.flatMap(({ rules }) => rules.flatMap(({ paths }) => paths));
+  await writeFile(exclusionPath, paths.join('\n'));
+  exclusionsDownloaded = true;
+}
+
+let exclusionsDownloaded = false;
+
+export async function ensureExclusionsDownloaded() {
+  if (exclusionsDownloaded) return;
+  try {
+    const githubToken = await getGHToken();
+    if (!githubToken) throw new Error('Failed to obtain GitHub token');
+    await downloadContentExclusions(githubToken);
+  } catch (e) {
+    exclusionDownloadError(e);
+    throw e;
+  }
+}
+
+export async function tryDownloadingExclusions() {
+  if (exclusionsDownloaded) return;
+  const githubToken = await getGHToken(true);
+  if (githubToken)
+    try {
+      downloadContentExclusions(githubToken);
+    } catch {
+      // pass, it will be retried
+    }
+}
+
+export async function getGHToken(silent = false): Promise<string | undefined> {
+  const provider = vscode.workspace
+    .getConfiguration()
+    .get('github.copilot.advanced.authProvider', 'github');
+  const session = await vscode.authentication.getSession(provider, [], {
+    silent,
+    createIfNone: !silent,
+  });
+  if (session) return session.accessToken;
+}
+
+function exclusionDownloadError(error: unknown) {
+  const title = 'Failed to download content exclusion policy from GitHub';
+  const errorText = isNativeError(error) ? error.message : String(error);
+
+  warn(`Failed to download content exclusions: ${errorText}`);
+  if (exclusionDownloadError.wait) return;
+
+  const showDetails = () => {
+    vscode.window.showErrorMessage(title, {
+      detail: `${errorText}\nPlease try again later.`,
+      modal: true,
+    });
+  };
+  if (exclusionDownloadError.wait === null) showDetails(); // first time always show the details
+  else vscode.window.showErrorMessage(title, {}, 'More...').then((x) => x && showDetails());
+
+  exclusionDownloadError.wait = true;
+  setTimeout(() => (exclusionDownloadError.wait = false), 5000).unref();
+}
+exclusionDownloadError.wait = null as boolean | null;

src/lib/fetchGHExclusions.ts

@@ -0,0 +1,37 @@
+import fetch from 'node-fetch';
+
+export interface GitHubContentExclusion {
+  rules: Rule[];
+  last_updated_at: string;
+  scope: Scope;
+}
+
+interface Rule {
+  source: {
+    name: string;
+    type: string;
+  };
+  paths: string[];
+}
+
+type Scope = 'all' | 'repo';
+
+export default async function fetchGHExclusions(
+  githubToken: string,
+  scope: Scope = 'all',
+  repos?: string[]
+): Promise<GitHubContentExclusion[]> {
+  const url = new URL('https://api.github.com/copilot_internal/content_exclusion');
+  url.searchParams.set('scope', scope);
+  if (repos) url.searchParams.set('repos', repos.join(','));
+  const response = await fetch(url.toString(), {
+    headers: {
+      Accept: 'application/json',
+      Authorization: `Bearer ${githubToken}`,
+    },
+  });
+  if (response.status === 404) return [];
+  if (!response.ok) throw new Error(`Failed to fetch exclusions: ${response.statusText}`);
+
+  return response.json();
+}

src/services/chatCompletion.ts

@@ -16,6 +16,7 @@ import vscode, {
 } from 'vscode';
 
 import ExtensionSettings from '../configuration/extensionSettings';
+import { ensureExclusionsDownloaded, tryDownloadingExclusions } from '../lib/exclusionUtils';
 import once from '../lib/once';
 
 const debug = debuglog('appmap-vscode:chat-completion');
@@ -35,6 +36,7 @@ export default class ChatCompletion implements Disposable {
     public readonly key = randomKey(),
     public readonly host = '127.0.0.1'
   ) {
+    tryDownloadingExclusions();
     this.server = createServer(async (req, res) => {
       try {
         await this.handleRequest(req, res);
@@ -134,6 +136,8 @@ export default class ChatCompletion implements Disposable {
   }
 
   async handleRequest(req: IncomingMessage, res: ServerResponse): Promise<void> {
+    await ensureExclusionsDownloaded();
+
     if (req.method !== 'POST') {
       res.writeHead(405);
       res.end();

test/unit/lib/fetchGHExclusions.test.ts

@@ -0,0 +1,50 @@
+import { expect } from 'chai';
+import nock from 'nock';
+
+import fetchGHExclusions, { GitHubContentExclusion } from '../../../src/lib/fetchGHExclusions';
+
+describe('fetchGHExclusions', () => {
+  const githubToken = 'test-token';
+  const apiUrl = 'https://api.github.com';
+
+  afterEach(() => {
+    nock.cleanAll();
+  });
+
+  it('should fetch exclusions successfully', async () => {
+    const mockResponse: GitHubContentExclusion[] = [
+      {
+        rules: [{ source: { name: 'test', type: 'type' }, paths: ['path1', 'path2'] }],
+        last_updated_at: '2023-01-01T00:00:00Z',
+        scope: 'all',
+      },
+    ];
+
+    nock(apiUrl)
+      .get('/copilot_internal/content_exclusion')
+      .query({ scope: 'all' })
+      .reply(200, mockResponse);
+
+    const exclusions = await fetchGHExclusions(githubToken, 'all');
+    expect(exclusions).to.deep.equal(mockResponse);
+  });
+
+  it('should return an empty array for 404 response', async () => {
+    nock(apiUrl).get('/copilot_internal/content_exclusion').query({ scope: 'all' }).reply(404);
+
+    const exclusions = await fetchGHExclusions(githubToken, 'all');
+    expect(exclusions).to.deep.equal([]);
+  });
+
+  it('should throw an error for non-200 and non-404 responses', async () => {
+    nock(apiUrl).get('/copilot_internal/content_exclusion').query({ scope: 'all' }).reply(500);
+
+    try {
+      await fetchGHExclusions(githubToken, 'all');
+      throw new Error('Expected fetchGHExclusions to throw an error');
+    } catch (err) {
+      expect(err).to.be.an.instanceOf(Error);
+      expect((err as Error).message).to.equal('Failed to fetch exclusions: Internal Server Error');
+    }
+  });
+});

test/unit/services/chatCompletion.test.ts

@@ -15,6 +15,7 @@ import { workspace } from 'vscode';
 import ChatCompletion from '../../../src/services/chatCompletion';
 import MockExtensionContext from '../../mocks/mockExtensionContext';
 import { addMockChatModel, resetModelMocks } from '../mock/vscode/lm';
+import * as ExclusionUtils from '../../../src/lib/exclusionUtils';
 
 const mockModel: LanguageModelChat = {
   id: 'test-model',
@@ -34,6 +35,7 @@ describe('ChatCompletion', () => {
 
   beforeEach(async () => {
     addMockChatModel(mockModel);
+    sinon.stub(ExclusionUtils, 'ensureExclusionsDownloaded').resolves();
 
     await ChatCompletion.refreshModels();
   });