Merge pull request #2521 from github/koesie10/auto-create-model-files

Automatically create different model files per library

Author: koesie10

extensions/ql-vscode/src/common/interface-types.ts

@@ -532,6 +532,7 @@ export interface OpenExtensionPackMessage {
 
 export interface OpenModelFileMessage {
   t: "openModelFile";
+  library: string;
 }
 
 export interface SaveModeledMethods {

…nsions/ql-vscode/src/view/common/path.ts extensions/ql-vscode/src/common/path.tsextensions/ql-vscode/src/view/common/path.ts renamed to extensions/ql-vscode/src/common/path.ts extensions/ql-vscode/src/view/common/path.ts renamed to extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -8,7 +8,7 @@ import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { App } from "../common/app";
 import { withProgress } from "../common/vscode/progress";
-import { pickExtensionPackModelFile } from "./extension-pack-picker";
+import { pickExtensionPack } from "./extension-pack-picker";
 import { showAndLogErrorMessage } from "../common/logging";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
@@ -78,7 +78,7 @@ export class DataExtensionsEditorModule {
               return;
             }
 
-            const modelFile = await pickExtensionPackModelFile(
+            const modelFile = await pickExtensionPack(
               this.cliServer,
               db,
               this.app.logger,

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -6,6 +6,7 @@ import {
   window,
   workspace,
 } from "vscode";
+import { join } from "path";
 import { RequestError } from "@octokit/request-error";
 import {
   AbstractWebview,
@@ -21,7 +22,7 @@ import {
   showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
 } from "../common/logging";
-import { outputFile, pathExists, readFile } from "fs-extra";
+import { outputFile, readFile } from "fs-extra";
 import { load as loadYaml } from "js-yaml";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { CodeQLCliServer } from "../codeql-cli/cli";
@@ -34,17 +35,22 @@ import { showResolvableLocation } from "../databases/local-databases/locations";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
 import { redactableError } from "../common/errors";
 import { readQueryResults, runQuery } from "./external-api-usage-query";
-import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import {
+  createDataExtensionYamlsPerLibrary,
+  createFilenameForLibrary,
+  loadDataExtensionYaml,
+} from "./yaml";
 import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
-import { ExtensionPackModelFile } from "./shared/extension-pack";
+import { ExtensionPack } from "./shared/extension-pack";
 import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
 import {
   createAutoModelRequest,
   parsePredictedClassifications,
 } from "./auto-model";
 import { showLlmGeneration } from "../config";
 import { getAutoModelUsages } from "./auto-model-usages-query";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
@@ -58,7 +64,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly modelFile: ExtensionPackModelFile,
+    private readonly extensionPack: ExtensionPack,
   ) {
     super(ctx);
   }
@@ -95,13 +101,18 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "openExtensionPack":
         await this.app.commands.execute(
           "revealInExplorer",
-          Uri.file(this.modelFile.extensionPack.path),
+          Uri.file(this.extensionPack.path),
         );
 
         break;
       case "openModelFile":
         await window.showTextDocument(
-          await workspace.openTextDocument(this.modelFile.filename),
+          await workspace.openTextDocument(
+            join(
+              this.extensionPack.path,
+              createFilenameForLibrary(msg.library),
+            ),
+          ),
         );
 
         break;
@@ -147,8 +158,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.postMessage({
       t: "setDataExtensionEditorViewState",
       viewState: {
-        extensionPackModelFile: this.modelFile,
-        modelFileExists: await pathExists(this.modelFile.filename),
+        extensionPack: this.extensionPack,
         showLlmButton: showLlmGeneration(),
       },
     });
@@ -178,39 +188,55 @@ export class DataExtensionsEditorView extends AbstractWebview<
     externalApiUsages: ExternalApiUsage[],
     modeledMethods: Record<string, ModeledMethod>,
   ): Promise<void> {
-    const yaml = createDataExtensionYaml(
+    const yamls = createDataExtensionYamlsPerLibrary(
       this.databaseItem.language,
       externalApiUsages,
       modeledMethods,
     );
 
-    await outputFile(this.modelFile.filename, yaml);
+    for (const [filename, yaml] of Object.entries(yamls)) {
+      await outputFile(join(this.extensionPack.path, filename), yaml);
+    }
 
-    void this.app.logger.log(
-      `Saved data extension YAML to ${this.modelFile.filename}`,
-    );
+    void this.app.logger.log(`Saved data extension YAML`);
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
-      if (!(await pathExists(this.modelFile.filename))) {
-        return;
+      const extensions = await this.cliServer.resolveExtensions(
+        this.extensionPack.path,
+        getOnDiskWorkspaceFolders(),
+      );
+
+      const modelFiles = new Set<string>();
+
+      if (this.extensionPack.path in extensions.data) {
+        for (const extension of extensions.data[this.extensionPack.path]) {
+          modelFiles.add(extension.file);
+        }
       }
 
-      const yaml = await readFile(this.modelFile.filename, "utf8");
+      const existingModeledMethods: Record<string, ModeledMethod> = {};
 
-      const data = loadYaml(yaml, {
-        filename: this.modelFile.filename,
-      });
+      for (const modelFile of modelFiles) {
+        const yaml = await readFile(modelFile, "utf8");
 
-      const existingModeledMethods = loadDataExtensionYaml(data);
+        const data = loadYaml(yaml, {
+          filename: modelFile,
+        });
 
-      if (!existingModeledMethods) {
-        void showAndLogErrorMessage(
-          this.app.logger,
-          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
-        );
-        return;
+        const modeledMethods = loadDataExtensionYaml(data);
+        if (!modeledMethods) {
+          void showAndLogErrorMessage(
+            this.app.logger,
+            `Failed to parse data extension YAML ${modelFile}.`,
+          );
+          continue;
+        }
+
+        for (const [key, value] of Object.entries(modeledMethods)) {
+          existingModeledMethods[key] = value;
+        }
       }
 
       await this.postMessage({
@@ -220,9 +246,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     } catch (e: unknown) {
       void showAndLogErrorMessage(
         this.app.logger,
-        `Unable to read data extension YAML ${
-          this.modelFile.filename
-        }: ${getErrorMessage(e)}`,
+        `Unable to read data extension YAML: ${getErrorMessage(e)}`,
       );
     }
   }

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -1,17 +1,15 @@
-import { join, relative, resolve, sep } from "path";
+import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
-import { minimatch } from "minimatch";
 import { CancellationToken, window } from "vscode";
 import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ProgressCallback } from "../common/vscode/progress";
 import { DatabaseItem } from "../databases/local-databases";
 import { getQlPackPath, QLPACK_FILENAMES } from "../common/ql";
 import { getErrorMessage } from "../common/helpers-pure";
-import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+import { ExtensionPack } from "./shared/extension-pack";
 import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
-import { containsPath } from "../common/files";
 import { disableAutoNameExtensionPack } from "../config";
 import {
   autoNameExtensionPack,
@@ -27,42 +25,7 @@ import {
 
 const maxStep = 3;
 
-export async function pickExtensionPackModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
-  logger: NotificationLogger,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<ExtensionPackModelFile | undefined> {
-  const extensionPack = await pickExtensionPack(
-    cliServer,
-    databaseItem,
-    logger,
-    progress,
-    token,
-  );
-  if (!extensionPack) {
-    return undefined;
-  }
-
-  const modelFile = await pickModelFile(
-    cliServer,
-    databaseItem,
-    extensionPack,
-    progress,
-    token,
-  );
-  if (!modelFile) {
-    return;
-  }
-
-  return {
-    filename: modelFile,
-    extensionPack,
-  };
-}
-
-async function pickExtensionPack(
+export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   logger: NotificationLogger,
@@ -190,69 +153,6 @@ async function pickExtensionPack(
   return extensionPackOption.extensionPack;
 }
 
-async function pickModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<string | undefined> {
-  // Find the existing model files in the extension pack
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensions = await cliServer.resolveExtensions(
-    extensionPack.path,
-    additionalPacks,
-  );
-
-  const modelFiles = new Set<string>();
-
-  if (extensionPack.path in extensions.data) {
-    for (const extension of extensions.data[extensionPack.path]) {
-      modelFiles.add(extension.file);
-    }
-  }
-
-  if (modelFiles.size === 0) {
-    return pickNewModelFile(databaseItem, extensionPack, token);
-  }
-
-  const fileOptions: Array<{ label: string; file: string | null }> = [];
-  for (const file of modelFiles) {
-    fileOptions.push({
-      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
-      file,
-    });
-  }
-  fileOptions.push({
-    label: "Create new model file",
-    file: null,
-  });
-
-  progress({
-    message: "Choosing model file...",
-    step: 3,
-    maxStep,
-  });
-
-  const fileOption = await window.showQuickPick(
-    fileOptions,
-    {
-      title: "Select model file to use",
-    },
-    token,
-  );
-
-  if (!fileOption) {
-    return undefined;
-  }
-
-  if (fileOption.file) {
-    return fileOption.file;
-  }
-
-  return pickNewModelFile(databaseItem, extensionPack, token);
-}
-
 async function pickNewExtensionPack(
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   token: CancellationToken,
@@ -428,49 +328,6 @@ async function writeExtensionPack(
   return extensionPack;
 }
 
-async function pickNewModelFile(
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  token: CancellationToken,
-) {
-  const filename = await window.showInputBox(
-    {
-      title: "Enter the name of the new model file",
-      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
-      validateInput: async (value: string): Promise<string | undefined> => {
-        if (value === "") {
-          return "File name must not be empty";
-        }
-
-        const path = resolve(extensionPack.path, value);
-
-        if (await pathExists(path)) {
-          return "File already exists";
-        }
-
-        if (!containsPath(extensionPack.path, path)) {
-          return "File must be in the extension pack";
-        }
-
-        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
-          minimatch(value, pattern, { matchBase: true }),
-        );
-        if (!matchesPattern) {
-          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
-        }
-
-        return undefined;
-      },
-    },
-    token,
-  );
-  if (!filename) {
-    return undefined;
-  }
-
-  return resolve(extensionPack.path, filename);
-}
-
 async function readExtensionPack(path: string): Promise<ExtensionPack> {
   const qlpackPath = await getQlPackPath(path);
   if (!qlpackPath) {

extensions/ql-vscode/src/data-extensions-editor/shared/extension-pack.ts

@@ -8,8 +8,3 @@ export interface ExtensionPack {
   extensionTargets: Record<string, string>;
   dataExtensions: string[];
 }
-
-export interface ExtensionPackModelFile {
-  filename: string;
-  extensionPack: ExtensionPack;
-}