Merge pull request #2658 from github/nora-koen/data-extensions-editor-without-ql-submodule

Remove submodules dependency from data extension editor

Author: norascheuch

extensions/ql-vscode/src/data-extensions-editor/auto-model-usages-query.ts

@@ -6,12 +6,14 @@ import { QueryRunner } from "../query-server";
 import { DatabaseItem } from "../databases/local-databases";
 import { interpretResultsSarif } from "../query-results";
 import { ProgressCallback } from "../common/vscode/progress";
+import { Mode } from "./shared/mode";
 
 type Options = {
   cliServer: CodeQLCliServer;
   queryRunner: QueryRunner;
   databaseItem: DatabaseItem;
   queryStorageDir: string;
+  queryDir: string;
 
   progress: ProgressCallback;
 };
@@ -23,6 +25,7 @@ export async function getAutoModelUsages({
   queryRunner,
   databaseItem,
   queryStorageDir,
+  queryDir,
   progress,
 }: Options): Promise<UsageSnippetsBySignature> {
   const maxStep = 1500;
@@ -32,11 +35,12 @@ export async function getAutoModelUsages({
   // This will re-run the query that was already run when opening the data extensions editor. This
   // might be unnecessary, but this makes it really easy to get the path to the BQRS file which we
   // need to interpret the results.
-  const queryResult = await runQuery("applicationModeQuery", {
+  const queryResult = await runQuery(Mode.Application, {
     cliServer,
     queryRunner,
     queryStorageDir,
     databaseItem,
+    queryDir,
     progress: (update) =>
       progress({
         maxStep,

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -9,7 +9,17 @@ import { join } from "path";
 import { App } from "../common/app";
 import { withProgress } from "../common/vscode/progress";
 import { pickExtensionPack } from "./extension-pack-picker";
-import { showAndLogErrorMessage } from "../common/logging";
+import {
+  showAndLogErrorMessage,
+  showAndLogExceptionWithTelemetry,
+} from "../common/logging";
+import { dir } from "tmp-promise";
+import { fetchExternalApiQueries } from "./queries";
+import { telemetryListener } from "../common/vscode/telemetry";
+import { redactableError } from "../common/errors";
+import { extLogger } from "../common/logging/vscode";
+import { isQueryLanguage } from "../common/query-language";
+import { setUpPack } from "./external-api-usage-query";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
 
@@ -60,10 +70,14 @@ export class DataExtensionsEditorModule {
           return;
         }
 
-        if (!SUPPORTED_LANGUAGES.includes(db.language)) {
+        const language = db.language;
+        if (
+          !SUPPORTED_LANGUAGES.includes(language) ||
+          !isQueryLanguage(language)
+        ) {
           void showAndLogErrorMessage(
             this.app.logger,
-            `The data extensions editor is not supported for ${db.language} databases.`,
+            `The data extensions editor is not supported for ${language} databases.`,
           );
           return;
         }
@@ -99,13 +113,29 @@ export class DataExtensionsEditorModule {
               return;
             }
 
+            const query = fetchExternalApiQueries[language];
+            if (!query) {
+              void showAndLogExceptionWithTelemetry(
+                extLogger,
+                telemetryListener,
+                redactableError`No external API usage query found for language ${language}`,
+              );
+              return;
+            }
+
+            // Create new temporary directory for query files and pack dependencies
+            const queryDir = (await dir({ unsafeCleanup: true })).path;
+            await setUpPack(queryDir, query, language);
+            await this.cliServer.packInstall(queryDir);
+
             const view = new DataExtensionsEditorView(
               this.ctx,
               this.app,
               this.databaseManager,
               this.cliServer,
               this.queryRunner,
               this.queryStorageDir,
+              queryDir,
               db,
               modelFile,
             );

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -71,6 +71,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly cliServer: CodeQLCliServer,
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
+    private readonly queryDir: string,
     private readonly databaseItem: DatabaseItem,
     private readonly extensionPack: ExtensionPack,
     private mode: Mode = Mode.Application,
@@ -248,19 +249,15 @@ export class DataExtensionsEditorView extends AbstractWebview<
       async (progress) => {
         try {
           const cancellationTokenSource = new CancellationTokenSource();
-          const queryResult = await runQuery(
-            this.mode === Mode.Framework
-              ? "frameworkModeQuery"
-              : "applicationModeQuery",
-            {
-              cliServer: this.cliServer,
-              queryRunner: this.queryRunner,
-              databaseItem: this.databaseItem,
-              queryStorageDir: this.queryStorageDir,
-              progress: (update) => progress({ ...update, maxStep: 1500 }),
-              token: cancellationTokenSource.token,
-            },
-          );
+          const queryResult = await runQuery(this.mode, {
+            cliServer: this.cliServer,
+            queryRunner: this.queryRunner,
+            databaseItem: this.databaseItem,
+            queryStorageDir: this.queryStorageDir,
+            queryDir: this.queryDir,
+            progress: (update) => progress({ ...update, maxStep: 1500 }),
+            token: cancellationTokenSource.token,
+          });
           if (!queryResult) {
             return;
           }
@@ -432,6 +429,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
           cliServer: this.cliServer,
           queryRunner: this.queryRunner,
           queryStorageDir: this.queryStorageDir,
+          queryDir: this.queryDir,
           databaseItem: this.databaseItem,
           progress: (update) => progress({ ...update, maxStep }),
         });
@@ -512,6 +510,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
         this.cliServer,
         this.queryRunner,
         this.queryStorageDir,
+        this.queryDir,
         addedDatabase,
         modelFile,
         Mode.Framework,

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -43,6 +43,10 @@ export async function pickExtensionPack(
 
   // Get all existing extension packs in the workspace
   const additionalPacks = getOnDiskWorkspaceFolders();
+  // the CLI doesn't check packs in the .github folder, so we need to add it manually
+  if (additionalPacks.length === 1) {
+    additionalPacks.push(`${additionalPacks[0]}/.github`);
+  }
   const extensionPacksInfo = await cliServer.resolveQlpacks(
     additionalPacks,
     true,

extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts

@@ -1,71 +1,44 @@
 import { CoreCompletedQuery, QueryRunner } from "../query-server";
-import { dir } from "tmp-promise";
-import { writeFile } from "fs-extra";
-import { dump as dumpYaml } from "js-yaml";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { extLogger } from "../common/logging/vscode";
 import { showAndLogExceptionWithTelemetry, TeeLogger } from "../common/logging";
-import { isQueryLanguage } from "../common/query-language";
 import { CancellationToken } from "vscode";
 import { CodeQLCliServer } from "../codeql-cli/cli";
 import { DatabaseItem } from "../databases/local-databases";
 import { ProgressCallback } from "../common/vscode/progress";
-import { fetchExternalApiQueries } from "./queries";
 import { QueryResultType } from "../query-server/new-messages";
-import { join } from "path";
 import { redactableError } from "../common/errors";
 import { telemetryListener } from "../common/vscode/telemetry";
+import { join } from "path";
+import { Mode } from "./shared/mode";
+import { writeFile } from "fs-extra";
 import { Query } from "./queries/query";
+import { QueryLanguage } from "../common/query-language";
+import { dump } from "js-yaml";
 
 type RunQueryOptions = {
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">;
   queryRunner: Pick<QueryRunner, "createQueryRun" | "logger">;
   databaseItem: Pick<DatabaseItem, "contents" | "databaseUri" | "language">;
   queryStorageDir: string;
+  queryDir: string;
 
   progress: ProgressCallback;
   token: CancellationToken;
 };
 
-export async function runQuery(
-  queryName: keyof Omit<Query, "dependencies">,
-  {
-    cliServer,
-    queryRunner,
-    databaseItem,
-    queryStorageDir,
-    progress,
-    token,
-  }: RunQueryOptions,
-): Promise<CoreCompletedQuery | undefined> {
-  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
-  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
-  // This is intentionally not pretty code, as it will be removed soon.
-  // For a reference of what this should do in the future, see the previous implementation in
-  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
-
-  if (!isQueryLanguage(databaseItem.language)) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`Unsupported database language ${databaseItem.language}`,
-    );
-    return;
-  }
-
-  const query = fetchExternalApiQueries[databaseItem.language];
-  if (!query) {
-    void showAndLogExceptionWithTelemetry(
-      extLogger,
-      telemetryListener,
-      redactableError`No external API usage query found for language ${databaseItem.language}`,
+export async function setUpPack(
+  queryDir: string,
+  query: Query,
+  language: QueryLanguage,
+) {
+  Object.values(Mode).map(async (mode) => {
+    const queryFile = join(
+      queryDir,
+      `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
     );
-    return;
-  }
-
-  const queryDir = (await dir({ unsafeCleanup: true })).path;
-  const queryFile = join(queryDir, "FetchExternalApis.ql");
-  await writeFile(queryFile, query[queryName], "utf8");
+    await writeFile(queryFile, query[`${mode}ModeQuery`], "utf8");
+  });
 
   if (query.dependencies) {
     for (const [filename, contents] of Object.entries(query.dependencies)) {
@@ -78,18 +51,42 @@ export async function runQuery(
     name: "codeql/external-api-usage",
     version: "0.0.0",
     dependencies: {
-      [`codeql/${databaseItem.language}-all`]: "*",
+      [`codeql/${language}-all`]: "*",
     },
   };
 
   const qlpackFile = join(queryDir, "codeql-pack.yml");
-  await writeFile(qlpackFile, dumpYaml(syntheticQueryPack), "utf8");
+  await writeFile(qlpackFile, dump(syntheticQueryPack), "utf8");
+}
+
+export async function runQuery(
+  mode: Mode,
+  {
+    cliServer,
+    queryRunner,
+    databaseItem,
+    queryStorageDir,
+    queryDir,
+    progress,
+    token,
+  }: RunQueryOptions,
+): Promise<CoreCompletedQuery | undefined> {
+  // The below code is temporary to allow for rapid prototyping of the queries. Once the queries are stabilized, we will
+  // move these queries into the `github/codeql` repository and use them like any other contextual (e.g. AST) queries.
+  // This is intentionally not pretty code, as it will be removed soon.
+  // For a reference of what this should do in the future, see the previous implementation in
+  // https://github.com/github/vscode-codeql/blob/089d3566ef0bc67d9b7cc66e8fd6740b31c1c0b0/extensions/ql-vscode/src/data-extensions-editor/external-api-usage-query.ts#L33-L72
 
   const additionalPacks = getOnDiskWorkspaceFolders();
   const extensionPacks = Object.keys(
     await cliServer.resolveQlpacks(additionalPacks, true),
   );
 
+  const queryFile = join(
+    queryDir,
+    `FetchExternalApis${mode.charAt(0).toUpperCase() + mode.slice(1)}Mode.ql`,
+  );
+
   const queryRun = queryRunner.createQueryRun(
     databaseItem.databaseUri.fsPath,
     {

extensions/ql-vscode/test/vscode-tests/no-workspace/data-extensions-editor/extension-pack-picker.test.ts

@@ -99,7 +99,10 @@ describe("pickExtensionPack", () => {
       name: "codeql-custom-queries-java",
       index: 0,
     };
-    additionalPacks = [Uri.file(tmpDir).fsPath];
+    additionalPacks = [
+      Uri.file(tmpDir).fsPath,
+      `${Uri.file(tmpDir).fsPath}/.github`,
+    ];
     workspaceFoldersSpy = jest
       .spyOn(workspace, "workspaceFolders", "get")
       .mockReturnValue([workspaceFolder]);