Automatically create different model files per library

This will remove the user input for a model file and will instead create
1 model file per library (JAR/DLL). The model filename will be based on
the JAR/DLL name, but will remove the version number and the file
extension. It will also normalize the name.

These files will be created automatically, and the editor now also reads
in all files contained in an extension pack to read the modeled methods.
This could result in duplicates if the user has created a different file
to contain the same modeled methods, but this is an edge-case that we're
explicitly not handling.

Author: koesie10

extensions/ql-vscode/src/common/interface-types.ts

@@ -532,6 +532,7 @@ export interface OpenExtensionPackMessage {
 
 export interface OpenModelFileMessage {
   t: "openModelFile";
+  library: string;
 }
 
 export interface SaveModeledMethods {

…nsions/ql-vscode/src/view/common/path.ts extensions/ql-vscode/src/common/path.tsextensions/ql-vscode/src/view/common/path.ts renamed to extensions/ql-vscode/src/common/path.ts extensions/ql-vscode/src/view/common/path.ts renamed to extensions/ql-vscode/src/common/path.ts

@@ -19,3 +19,11 @@ export const basename = (path: string): string => {
   const index = path.lastIndexOf("\\");
   return index === -1 ? path : path.slice(index + 1);
 };
+
+// Returns the extension of a path, including the leading dot.
+export const extname = (path: string): string => {
+  const name = basename(path);
+
+  const index = name.lastIndexOf(".");
+  return index === -1 ? "" : name.slice(index);
+};

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-module.ts

@@ -8,7 +8,7 @@ import { ensureDir } from "fs-extra";
 import { join } from "path";
 import { App } from "../common/app";
 import { withProgress } from "../common/vscode/progress";
-import { pickExtensionPackModelFile } from "./extension-pack-picker";
+import { pickExtensionPack } from "./extension-pack-picker";
 import { showAndLogErrorMessage } from "../common/logging";
 
 const SUPPORTED_LANGUAGES: string[] = ["java", "csharp"];
@@ -78,7 +78,7 @@ export class DataExtensionsEditorModule {
               return;
             }
 
-            const modelFile = await pickExtensionPackModelFile(
+            const modelFile = await pickExtensionPack(
               this.cliServer,
               db,
               this.app.logger,

extensions/ql-vscode/src/data-extensions-editor/data-extensions-editor-view.ts

@@ -6,6 +6,7 @@ import {
   window,
   workspace,
 } from "vscode";
+import { join } from "path";
 import { RequestError } from "@octokit/request-error";
 import {
   AbstractWebview,
@@ -21,7 +22,7 @@ import {
   showAndLogExceptionWithTelemetry,
   showAndLogErrorMessage,
 } from "../common/logging";
-import { outputFile, pathExists, readFile } from "fs-extra";
+import { outputFile, readFile } from "fs-extra";
 import { load as loadYaml } from "js-yaml";
 import { DatabaseItem, DatabaseManager } from "../databases/local-databases";
 import { CodeQLCliServer } from "../codeql-cli/cli";
@@ -34,17 +35,22 @@ import { showResolvableLocation } from "../databases/local-databases/locations";
 import { decodeBqrsToExternalApiUsages } from "./bqrs";
 import { redactableError } from "../common/errors";
 import { readQueryResults, runQuery } from "./external-api-usage-query";
-import { createDataExtensionYaml, loadDataExtensionYaml } from "./yaml";
+import {
+  createDataExtensionYamlsPerLibrary,
+  createFilenameForLibrary,
+  loadDataExtensionYaml,
+} from "./yaml";
 import { ExternalApiUsage } from "./external-api-usage";
 import { ModeledMethod } from "./modeled-method";
-import { ExtensionPackModelFile } from "./shared/extension-pack";
+import { ExtensionPack } from "./shared/extension-pack";
 import { autoModel, ModelRequest, ModelResponse } from "./auto-model-api";
 import {
   createAutoModelRequest,
   parsePredictedClassifications,
 } from "./auto-model";
 import { showLlmGeneration } from "../config";
 import { getAutoModelUsages } from "./auto-model-usages-query";
+import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 
 export class DataExtensionsEditorView extends AbstractWebview<
   ToDataExtensionsEditorMessage,
@@ -58,7 +64,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     private readonly queryRunner: QueryRunner,
     private readonly queryStorageDir: string,
     private readonly databaseItem: DatabaseItem,
-    private readonly modelFile: ExtensionPackModelFile,
+    private readonly extensionPack: ExtensionPack,
   ) {
     super(ctx);
   }
@@ -95,13 +101,18 @@ export class DataExtensionsEditorView extends AbstractWebview<
       case "openExtensionPack":
         await this.app.commands.execute(
           "revealInExplorer",
-          Uri.file(this.modelFile.extensionPack.path),
+          Uri.file(this.extensionPack.path),
         );
 
         break;
       case "openModelFile":
         await window.showTextDocument(
-          await workspace.openTextDocument(this.modelFile.filename),
+          await workspace.openTextDocument(
+            join(
+              this.extensionPack.path,
+              createFilenameForLibrary(msg.library),
+            ),
+          ),
         );
 
         break;
@@ -147,8 +158,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     await this.postMessage({
       t: "setDataExtensionEditorViewState",
       viewState: {
-        extensionPackModelFile: this.modelFile,
-        modelFileExists: await pathExists(this.modelFile.filename),
+        extensionPack: this.extensionPack,
         showLlmButton: showLlmGeneration(),
       },
     });
@@ -178,39 +188,55 @@ export class DataExtensionsEditorView extends AbstractWebview<
     externalApiUsages: ExternalApiUsage[],
     modeledMethods: Record<string, ModeledMethod>,
   ): Promise<void> {
-    const yaml = createDataExtensionYaml(
+    const yamls = createDataExtensionYamlsPerLibrary(
       this.databaseItem.language,
       externalApiUsages,
       modeledMethods,
     );
 
-    await outputFile(this.modelFile.filename, yaml);
+    for (const [filename, yaml] of Object.entries(yamls)) {
+      await outputFile(join(this.extensionPack.path, filename), yaml);
+    }
 
-    void this.app.logger.log(
-      `Saved data extension YAML to ${this.modelFile.filename}`,
-    );
+    void this.app.logger.log(`Saved data extension YAML`);
   }
 
   protected async loadExistingModeledMethods(): Promise<void> {
     try {
-      if (!(await pathExists(this.modelFile.filename))) {
-        return;
+      const extensions = await this.cliServer.resolveExtensions(
+        this.extensionPack.path,
+        getOnDiskWorkspaceFolders(),
+      );
+
+      const modelFiles = new Set<string>();
+
+      if (this.extensionPack.path in extensions.data) {
+        for (const extension of extensions.data[this.extensionPack.path]) {
+          modelFiles.add(extension.file);
+        }
       }
 
-      const yaml = await readFile(this.modelFile.filename, "utf8");
+      const existingModeledMethods: Record<string, ModeledMethod> = {};
 
-      const data = loadYaml(yaml, {
-        filename: this.modelFile.filename,
-      });
+      for (const modelFile of modelFiles) {
+        const yaml = await readFile(modelFile, "utf8");
 
-      const existingModeledMethods = loadDataExtensionYaml(data);
+        const data = loadYaml(yaml, {
+          filename: modelFile,
+        });
 
-      if (!existingModeledMethods) {
-        void showAndLogErrorMessage(
-          this.app.logger,
-          `Failed to parse data extension YAML ${this.modelFile.filename}.`,
-        );
-        return;
+        const modeledMethods = loadDataExtensionYaml(data);
+        if (!modeledMethods) {
+          void showAndLogErrorMessage(
+            this.app.logger,
+            `Failed to parse data extension YAML ${modelFile}.`,
+          );
+          continue;
+        }
+
+        for (const [key, value] of Object.entries(modeledMethods)) {
+          existingModeledMethods[key] = value;
+        }
       }
 
       await this.postMessage({
@@ -220,9 +246,7 @@ export class DataExtensionsEditorView extends AbstractWebview<
     } catch (e: unknown) {
       void showAndLogErrorMessage(
         this.app.logger,
-        `Unable to read data extension YAML ${
-          this.modelFile.filename
-        }: ${getErrorMessage(e)}`,
+        `Unable to read data extension YAML: ${getErrorMessage(e)}`,
       );
     }
   }

extensions/ql-vscode/src/data-extensions-editor/extension-pack-picker.ts

@@ -1,17 +1,15 @@
-import { join, relative, resolve, sep } from "path";
+import { join } from "path";
 import { outputFile, pathExists, readFile } from "fs-extra";
 import { dump as dumpYaml, load as loadYaml } from "js-yaml";
-import { minimatch } from "minimatch";
 import { CancellationToken, window } from "vscode";
 import { CodeQLCliServer, QlpacksInfo } from "../codeql-cli/cli";
 import { getOnDiskWorkspaceFolders } from "../common/vscode/workspace-folders";
 import { ProgressCallback } from "../common/vscode/progress";
 import { DatabaseItem } from "../databases/local-databases";
 import { getQlPackPath, QLPACK_FILENAMES } from "../common/ql";
 import { getErrorMessage } from "../common/helpers-pure";
-import { ExtensionPack, ExtensionPackModelFile } from "./shared/extension-pack";
+import { ExtensionPack } from "./shared/extension-pack";
 import { NotificationLogger, showAndLogErrorMessage } from "../common/logging";
-import { containsPath } from "../common/files";
 import { disableAutoNameExtensionPack } from "../config";
 import {
   autoNameExtensionPack,
@@ -27,42 +25,7 @@ import {
 
 const maxStep = 3;
 
-export async function pickExtensionPackModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveQlpacks" | "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name" | "language">,
-  logger: NotificationLogger,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<ExtensionPackModelFile | undefined> {
-  const extensionPack = await pickExtensionPack(
-    cliServer,
-    databaseItem,
-    logger,
-    progress,
-    token,
-  );
-  if (!extensionPack) {
-    return undefined;
-  }
-
-  const modelFile = await pickModelFile(
-    cliServer,
-    databaseItem,
-    extensionPack,
-    progress,
-    token,
-  );
-  if (!modelFile) {
-    return;
-  }
-
-  return {
-    filename: modelFile,
-    extensionPack,
-  };
-}
-
-async function pickExtensionPack(
+export async function pickExtensionPack(
   cliServer: Pick<CodeQLCliServer, "resolveQlpacks">,
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   logger: NotificationLogger,
@@ -190,69 +153,6 @@ async function pickExtensionPack(
   return extensionPackOption.extensionPack;
 }
 
-async function pickModelFile(
-  cliServer: Pick<CodeQLCliServer, "resolveExtensions">,
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  progress: ProgressCallback,
-  token: CancellationToken,
-): Promise<string | undefined> {
-  // Find the existing model files in the extension pack
-  const additionalPacks = getOnDiskWorkspaceFolders();
-  const extensions = await cliServer.resolveExtensions(
-    extensionPack.path,
-    additionalPacks,
-  );
-
-  const modelFiles = new Set<string>();
-
-  if (extensionPack.path in extensions.data) {
-    for (const extension of extensions.data[extensionPack.path]) {
-      modelFiles.add(extension.file);
-    }
-  }
-
-  if (modelFiles.size === 0) {
-    return pickNewModelFile(databaseItem, extensionPack, token);
-  }
-
-  const fileOptions: Array<{ label: string; file: string | null }> = [];
-  for (const file of modelFiles) {
-    fileOptions.push({
-      label: relative(extensionPack.path, file).replaceAll(sep, "/"),
-      file,
-    });
-  }
-  fileOptions.push({
-    label: "Create new model file",
-    file: null,
-  });
-
-  progress({
-    message: "Choosing model file...",
-    step: 3,
-    maxStep,
-  });
-
-  const fileOption = await window.showQuickPick(
-    fileOptions,
-    {
-      title: "Select model file to use",
-    },
-    token,
-  );
-
-  if (!fileOption) {
-    return undefined;
-  }
-
-  if (fileOption.file) {
-    return fileOption.file;
-  }
-
-  return pickNewModelFile(databaseItem, extensionPack, token);
-}
-
 async function pickNewExtensionPack(
   databaseItem: Pick<DatabaseItem, "name" | "language">,
   token: CancellationToken,
@@ -428,49 +328,6 @@ async function writeExtensionPack(
   return extensionPack;
 }
 
-async function pickNewModelFile(
-  databaseItem: Pick<DatabaseItem, "name">,
-  extensionPack: ExtensionPack,
-  token: CancellationToken,
-) {
-  const filename = await window.showInputBox(
-    {
-      title: "Enter the name of the new model file",
-      value: `models/${databaseItem.name.replaceAll("/", ".")}.model.yml`,
-      validateInput: async (value: string): Promise<string | undefined> => {
-        if (value === "") {
-          return "File name must not be empty";
-        }
-
-        const path = resolve(extensionPack.path, value);
-
-        if (await pathExists(path)) {
-          return "File already exists";
-        }
-
-        if (!containsPath(extensionPack.path, path)) {
-          return "File must be in the extension pack";
-        }
-
-        const matchesPattern = extensionPack.dataExtensions.some((pattern) =>
-          minimatch(value, pattern, { matchBase: true }),
-        );
-        if (!matchesPattern) {
-          return `File must match one of the patterns in 'dataExtensions' in ${extensionPack.yamlPath}`;
-        }
-
-        return undefined;
-      },
-    },
-    token,
-  );
-  if (!filename) {
-    return undefined;
-  }
-
-  return resolve(extensionPack.path, filename);
-}
-
 async function readExtensionPack(path: string): Promise<ExtensionPack> {
   const qlpackPath = await getQlPackPath(path);
   if (!qlpackPath) {

extensions/ql-vscode/src/data-extensions-editor/shared/extension-pack.ts

@@ -8,8 +8,3 @@ export interface ExtensionPack {
   extensionTargets: Record<string, string>;
   dataExtensions: string[];
 }
-
-export interface ExtensionPackModelFile {
-  filename: string;
-  extensionPack: ExtensionPack;
-}