Add new GitHub API fields to scenario files

This uses a script to add the new `stargazers_count` and `updated_at` to
the scenario files. This is done by using the GitHub API to get the
information for each repo and then updating the scenario file.

The `updated_at` values are not completely representative since they are
the `updated_at` at time of running the script, rather than at the time
the variant analysis was run. However, this should not really matter in
practice. An alternative for scanned repositories might be getting the
creation time of the `database_commit_sha` commit.

Author: koesie10

extensions/ql-vscode/package-lock.json

@@ -1312,6 +1312,7 @@
     "@babel/core": "^7.18.13",
     "@babel/plugin-transform-modules-commonjs": "^7.18.6",
     "@faker-js/faker": "^7.5.0",
+    "@octokit/plugin-throttling": "^4.3.2",
     "@storybook/addon-actions": "^6.5.10",
     "@storybook/addon-essentials": "^6.5.10",
     "@storybook/addon-interactions": "^6.5.10",

extensions/ql-vscode/package.json

@@ -0,0 +1,128 @@
+/**
+ * This scripts helps after adding a new field in the GitHub API. You will
+ * need to modify this script to add the new field to the scenarios. This
+ * is just a template and should not be used as-is since it has already been
+ * applied.
+ *
+ * Depending on the actual implementation of the script, you might run into
+ * rate limits. If that happens, you can set a `GITHUB_TOKEN` environment
+ * variable. For example, use: ``export GITHUB_TOKEN=`gh auth token```.
+ *
+ * Usage: npx ts-node scripts/add-fields-to-scenarios.ts
+ */
+
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+import { Octokit, type RestEndpointMethodTypes } from '@octokit/rest';
+import { throttling } from '@octokit/plugin-throttling';
+
+import { getFiles } from './util/files';
+import type { GitHubApiRequest } from '../src/mocks/gh-api-request';
+import { isGetVariantAnalysisRequest } from '../src/mocks/gh-api-request';
+import { VariantAnalysis } from '../src/remote-queries/gh-api/variant-analysis';
+import { RepositoryWithMetadata } from '../src/remote-queries/gh-api/repository';
+
+const extensionDirectory = path.resolve(__dirname, '..');
+const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
+
+// Make sure we don't run into rate limits by automatically waiting until we can
+// make another request.
+const MyOctokit = Octokit.plugin(throttling);
+
+const auth = process.env.GITHUB_TOKEN;
+
+const octokit = new MyOctokit({
+  auth,
+  throttle: {
+    onRateLimit: (retryAfter: number, options: any, octokit: Octokit): boolean => {
+      octokit.log.warn(
+        `Request quota exhausted for request ${options.method} ${options.url}. Retrying after ${retryAfter} seconds!`
+      );
+
+      return true;
+    },
+    onSecondaryRateLimit: (_retryAfter: number, options: any, octokit: Octokit): void => {
+      octokit.log.warn(
+        `SecondaryRateLimit detected for request ${options.method} ${options.url}`
+      );
+    },
+  }
+});
+const repositories = new Map<number, RestEndpointMethodTypes['repos']['get']['response']['data']>();
+
+async function addFieldsToRepository(repository: RepositoryWithMetadata) {
+  if (!repositories.has(repository.id)) {
+    const [owner, repo] = repository.full_name.split('/');
+
+    const apiRepository = await octokit.repos.get({
+      owner,
+      repo,
+    });
+
+    repositories.set(repository.id, apiRepository.data);
+  }
+
+  const apiRepository = repositories.get(repository.id)!;
+
+  repository.stargazers_count = apiRepository.stargazers_count;
+  repository.updated_at = apiRepository.updated_at;
+}
+
+async function addFieldsToScenarios() {
+  if (!(await fs.pathExists(scenariosDirectory))) {
+    console.error('Scenarios directory does not exist: ' + scenariosDirectory);
+    return;
+  }
+
+  for await (const file of getFiles(scenariosDirectory)) {
+    if (!file.endsWith('.json')) {
+      continue;
+    }
+
+    const data: GitHubApiRequest = await fs.readJson(file);
+
+    if (!isGetVariantAnalysisRequest(data)) {
+      continue;
+    }
+
+    if (!data.response.body || !('controller_repo' in data.response.body)) {
+      continue;
+    }
+
+    console.log(`Adding fields to '${path.relative(scenariosDirectory, file)}'`);
+
+    const variantAnalysis = data.response.body as VariantAnalysis;
+
+    if (variantAnalysis.scanned_repositories) {
+      for (const item of variantAnalysis.scanned_repositories) {
+        await addFieldsToRepository(item.repository);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.access_mismatch_repos) {
+      for (const item of variantAnalysis.skipped_repositories.access_mismatch_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.no_codeql_db_repos) {
+      for (const item of variantAnalysis.skipped_repositories.no_codeql_db_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    if (variantAnalysis.skipped_repositories?.over_limit_repos) {
+      for (const item of variantAnalysis.skipped_repositories.over_limit_repos.repositories) {
+        await addFieldsToRepository(item);
+      }
+    }
+
+    await fs.writeJson(file, data, { spaces: 2 });
+  }
+}
+
+addFieldsToScenarios().catch(e => {
+  console.error(e);
+  process.exit(2);
+});

extensions/ql-vscode/scripts/add-fields-to-scenarios.ts

@@ -4,6 +4,8 @@ import * as path from 'path';
 import Ajv from 'ajv';
 import * as tsj from 'ts-json-schema-generator';
 
+import { getFiles } from './util/files';
+
 const extensionDirectory = path.resolve(__dirname, '..');
 const rootDirectory = path.resolve(extensionDirectory, '../..');
 const scenariosDirectory = path.resolve(extensionDirectory, 'src/mocks/scenarios');
@@ -60,19 +62,6 @@ async function lintScenarios() {
   }
 }
 
-// https://stackoverflow.com/a/45130990
-async function* getFiles(dir: string): AsyncGenerator<string> {
-  const dirents = await fs.readdir(dir, { withFileTypes: true });
-  for (const dirent of dirents) {
-    const res = path.resolve(dir, dirent.name);
-    if (dirent.isDirectory()) {
-      yield* getFiles(res);
-    } else {
-      yield res;
-    }
-  }
-}
-
 lintScenarios().catch(e => {
   console.error(e);
   process.exit(2);

extensions/ql-vscode/scripts/lint-scenarios.ts

@@ -2,5 +2,9 @@
   "$schema": "https://json.schemastore.org/tsconfig",
   "extends": "../tsconfig.json",
   "include": ["**/*.ts"],
-  "exclude": []
+  "exclude": [],
+  "compilerOptions": {
+    "rootDir": "..",
+    "noEmit": true
+  }
 }

extensions/ql-vscode/scripts/tsconfig.json

@@ -0,0 +1,15 @@
+import * as fs from 'fs-extra';
+import * as path from 'path';
+
+// https://stackoverflow.com/a/45130990
+export async function* getFiles(dir: string): AsyncGenerator<string> {
+  const dirents = await fs.readdir(dir, { withFileTypes: true });
+  for (const dirent of dirents) {
+    const res = path.resolve(dir, dirent.name);
+    if (dirent.isDirectory()) {
+      yield* getFiles(res);
+    } else {
+      yield res;
+    }
+  }
+}

extensions/ql-vscode/scripts/util/files.ts

@@ -105,7 +105,9 @@
             "id": 206444,
             "name": "hive",
             "full_name": "apache/hive",
-            "private": false
+            "private": false,
+            "stargazers_count": 4523,
+            "updated_at": "2022-11-02T10:04:02Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 81841,
@@ -116,7 +118,9 @@
             "id": 20753500,
             "name": "ng-nice",
             "full_name": "angular-cn/ng-nice",
-            "private": false
+            "private": false,
+            "stargazers_count": 192,
+            "updated_at": "2022-03-17T08:34:30Z"
           },
           "analysis_status": "in_progress"
         },
@@ -125,7 +129,9 @@
             "id": 23418517,
             "name": "hadoop",
             "full_name": "apache/hadoop",
-            "private": false
+            "private": false,
+            "stargazers_count": 13030,
+            "updated_at": "2022-11-02T08:23:58Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 66895,
@@ -136,7 +142,9 @@
             "id": 236095576,
             "name": "backstage",
             "full_name": "backstage/backstage",
-            "private": false
+            "private": false,
+            "stargazers_count": 19033,
+            "updated_at": "2022-11-02T10:25:24Z"
           },
           "analysis_status": "in_progress"
         },
@@ -145,7 +153,9 @@
             "id": 257485422,
             "name": "vite",
             "full_name": "vitejs/vite",
-            "private": false
+            "private": false,
+            "stargazers_count": 49064,
+            "updated_at": "2022-11-02T11:29:22Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 702,

extensions/ql-vscode/src/mocks/scenarios/problem-query-success/11-getVariantAnalysis.json

@@ -105,7 +105,9 @@
             "id": 206444,
             "name": "hive",
             "full_name": "apache/hive",
-            "private": false
+            "private": false,
+            "stargazers_count": 4523,
+            "updated_at": "2022-11-02T10:04:02Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 81841,
@@ -116,7 +118,9 @@
             "id": 20753500,
             "name": "ng-nice",
             "full_name": "angular-cn/ng-nice",
-            "private": false
+            "private": false,
+            "stargazers_count": 192,
+            "updated_at": "2022-03-17T08:34:30Z"
           },
           "analysis_status": "in_progress"
         },
@@ -125,7 +129,9 @@
             "id": 23418517,
             "name": "hadoop",
             "full_name": "apache/hadoop",
-            "private": false
+            "private": false,
+            "stargazers_count": 13030,
+            "updated_at": "2022-11-02T08:23:58Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 66895,
@@ -136,7 +142,9 @@
             "id": 236095576,
             "name": "backstage",
             "full_name": "backstage/backstage",
-            "private": false
+            "private": false,
+            "stargazers_count": 19033,
+            "updated_at": "2022-11-02T10:25:24Z"
           },
           "analysis_status": "in_progress"
         },
@@ -145,7 +153,9 @@
             "id": 257485422,
             "name": "vite",
             "full_name": "vitejs/vite",
-            "private": false
+            "private": false,
+            "stargazers_count": 49064,
+            "updated_at": "2022-11-02T11:29:22Z"
           },
           "analysis_status": "succeeded",
           "artifact_size_in_bytes": 702,