Resolve database lock contention w/ vscode-codeql

Resolves #117

Fixes a known compatibility issue for databases added, and therefore
locked, via the GitHub.vscode-codeql extension.

The vscode-codeql query server creates .lock files in the cache
directory of every registered CodeQL database, preventing the ql-mcp
server from running CLI commands (codeql_query_run,
codeql_database_analyze) against those same databases.

Add a DatabaseCopier that syncs databases from vscode-codeql storage
into a managed directory under the `vscode-codeql-development-mcp-server`
extension's globalStorage, stripping .lock files from the copy. The
EnvironmentBuilder now sets CODEQL_DATABASES_BASE_DIRS to this managed
directory by default (configurable via codeql-mcp.copyDatabases).

- New DatabaseCopier class with incremental sync (skips unchanged databases)
- StoragePaths.getManagedDatabaseStoragePath() for the managed databases/ dir
- EnvironmentBuilder accepts injectable DatabaseCopierFactory for testability
- codeql-mcp.copyDatabases setting (default: true)
- 11 unit tests for DatabaseCopier (real filesystem operations)
- 15 unit tests for EnvironmentBuilder (updated for copy mode + fallback)
- 3 bridge integration tests (managed dir structure, no .lock files)
- 4 E2E integration tests: inject .lock → copy → codeql_query_run +
  codeql_database_analyze succeed against the lock-free copy

Author: data-douser

extensions/vscode/esbuild.config.js

@@ -35,6 +35,7 @@ const testSuiteConfig = {
   entryPoints: [
     'test/suite/index.ts',
     'test/suite/bridge.integration.test.ts',
+    'test/suite/copydb-e2e.integration.test.ts',
     'test/suite/extension.integration.test.ts',
     'test/suite/mcp-resource-e2e.integration.test.ts',
     'test/suite/mcp-server.integration.test.ts',

extensions/vscode/package.json

@@ -48,33 +48,13 @@
     "configuration": {
       "title": "CodeQL MCP Server",
       "properties": {
-        "codeql-mcp.autoInstall": {
-          "type": "boolean",
-          "default": true,
-          "description": "Automatically install and update the CodeQL Development MCP Server on activation."
-        },
-        "codeql-mcp.serverVersion": {
-          "type": "string",
-          "default": "latest",
-          "description": "The npm version of codeql-development-mcp-server to install. Use 'latest' for the most recent release."
-        },
-        "codeql-mcp.serverCommand": {
-          "type": "string",
-          "default": "node",
-          "description": "Command to launch the MCP server. The default 'node' runs the bundled server. Override to 'npx' to download from npm, or provide a custom path."
-        },
-        "codeql-mcp.serverArgs": {
+        "codeql-mcp.additionalDatabaseDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
-        },
-        "codeql-mcp.watchCodeqlExtension": {
-          "type": "boolean",
-          "default": true,
-          "description": "Watch for CodeQL databases and query results created by the CodeQL extension."
+          "description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS alongside the vscode-codeql storage paths."
         },
         "codeql-mcp.additionalEnv": {
           "type": "object",
@@ -84,29 +64,54 @@
             "type": "string"
           }
         },
-        "codeql-mcp.additionalDatabaseDirs": {
+        "codeql-mcp.additionalMrvaRunResultsDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories to search for CodeQL databases. Appended to CODEQL_DATABASES_BASE_DIRS alongside the vscode-codeql storage paths."
+          "description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
         },
-        "codeql-mcp.additionalMrvaRunResultsDirs": {
+        "codeql-mcp.additionalQueryRunResultsDirs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories containing MRVA run result subdirectories. Appended to CODEQL_MRVA_RUN_RESULTS_DIRS alongside the vscode-codeql variant analysis storage path."
+          "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
         },
-        "codeql-mcp.additionalQueryRunResultsDirs": {
+        "codeql-mcp.autoInstall": {
+          "type": "boolean",
+          "default": true,
+          "description": "Automatically install and update the CodeQL Development MCP Server on activation."
+        },
+        "codeql-mcp.copyDatabases": {
+          "type": "boolean",
+          "default": true,
+          "markdownDescription": "Copy CodeQL databases from the `GitHub.vscode-codeql` extension storage into a managed directory, removing query-server lock files so the MCP server CLI can operate without contention. Disable to use databases in-place (may fail when the CodeQL query server is running)."
+        },
+        "codeql-mcp.serverArgs": {
           "type": "array",
           "items": {
             "type": "string"
           },
           "default": [],
-          "description": "Additional directories containing query run result subdirectories. Appended to CODEQL_QUERY_RUN_RESULTS_DIRS alongside the vscode-codeql query storage path."
+          "description": "Custom arguments for the MCP server command. When empty, the bundled server entry point is used automatically. Set to e.g. ['/path/to/server/dist/codeql-development-mcp-server.js'] for local development."
+        },
+        "codeql-mcp.serverCommand": {
+          "type": "string",
+          "default": "node",
+          "description": "Command to launch the MCP server. The default 'node' runs the bundled server. Override to 'npx' to download from npm, or provide a custom path."
+        },
+        "codeql-mcp.serverVersion": {
+          "type": "string",
+          "default": "latest",
+          "description": "The npm version of codeql-development-mcp-server to install. Use 'latest' for the most recent release."
+        },
+        "codeql-mcp.watchCodeqlExtension": {
+          "type": "boolean",
+          "default": true,
+          "description": "Watch for CodeQL databases and query results created by the CodeQL extension."
         }
       }
     },

extensions/vscode/src/bridge/database-copier.ts

@@ -0,0 +1,147 @@
+import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, unlinkSync } from 'fs';
+import { join } from 'path';
+import type { Logger } from '../common/logger';
+
+/**
+ * Copies CodeQL databases from `GitHub.vscode-codeql` extension storage
+ * to a managed directory, removing `.lock` files that the CodeQL query
+ * server creates in `<dataset>/default/cache/`.
+ *
+ * This avoids lock contention when the `ql-mcp` server runs CLI commands
+ * against databases that are simultaneously registered by the
+ * `vscode-codeql` query server.
+ *
+ * Each database is identified by its top-level directory name (which
+ * contains `codeql-database.yml`). A database is only re-copied when its
+ * source has been modified more recently than the existing copy.
+ */
+export class DatabaseCopier {
+  constructor(
+    private readonly destinationBase: string,
+    private readonly logger: Logger,
+  ) {}
+
+  /**
+   * Synchronise databases from one or more source directories into the
+   * managed destination. Only databases that are newer than the existing
+   * copy (or missing entirely) are re-copied.
+   *
+   * @returns The list of database paths in the managed destination that
+   *          are ready for use (absolute paths).
+   */
+  syncAll(sourceDirs: string[]): string[] {
+    mkdirSync(this.destinationBase, { recursive: true });
+
+    const copied: string[] = [];
+
+    for (const sourceDir of sourceDirs) {
+      if (!existsSync(sourceDir)) {
+        continue;
+      }
+
+      let entries: string[];
+      try {
+        entries = readdirSync(sourceDir);
+      } catch {
+        continue;
+      }
+
+      for (const entry of entries) {
+        const srcDbPath = join(sourceDir, entry);
+        if (!isCodeQLDatabase(srcDbPath)) {
+          continue;
+        }
+
+        const destDbPath = join(this.destinationBase, entry);
+
+        if (this.needsCopy(srcDbPath, destDbPath)) {
+          this.copyDatabase(srcDbPath, destDbPath);
+        }
+
+        copied.push(destDbPath);
+      }
+    }
+
+    return copied;
+  }
+
+  /**
+   * Copy a single database directory, then strip any `.lock` files that
+   * the CodeQL query server may have left behind.
+   */
+  private copyDatabase(src: string, dest: string): void {
+    this.logger.info(`Copying database ${src} → ${dest}`);
+    try {
+      // Remove stale destination if present
+      if (existsSync(dest)) {
+        rmSync(dest, { recursive: true, force: true });
+      }
+
+      cpSync(src, dest, { recursive: true });
+      removeLockFiles(dest);
+      this.logger.info(`Database copied successfully: ${dest}`);
+    } catch (err) {
+      this.logger.error(
+        `Failed to copy database ${src}: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  /**
+   * A copy is needed when the destination does not exist, or the source
+   * `codeql-database.yml` is newer than the destination's.
+   */
+  private needsCopy(src: string, dest: string): boolean {
+    const destYml = join(dest, 'codeql-database.yml');
+    if (!existsSync(destYml)) {
+      return true;
+    }
+
+    const srcYml = join(src, 'codeql-database.yml');
+    try {
+      const srcMtime = statSync(srcYml).mtimeMs;
+      const destMtime = statSync(destYml).mtimeMs;
+      return srcMtime > destMtime;
+    } catch {
+      // If stat fails, re-copy to be safe
+      return true;
+    }
+  }
+}
+
+/** Check whether a directory looks like a CodeQL database. */
+function isCodeQLDatabase(dirPath: string): boolean {
+  try {
+    return statSync(dirPath).isDirectory() && existsSync(join(dirPath, 'codeql-database.yml'));
+  } catch {
+    return false;
+  }
+}
+
+/**
+ * Recursively remove all `.lock` files under the given directory.
+ * These are empty sentinel files created by the CodeQL query server in
+ * `<dataset>/default/cache/.lock`.
+ */
+function removeLockFiles(dir: string): void {
+  let entries: string[];
+  try {
+    entries = readdirSync(dir);
+  } catch {
+    return;
+  }
+
+  for (const entry of entries) {
+    const fullPath = join(dir, entry);
+    try {
+      const stat = statSync(fullPath);
+      if (stat.isDirectory()) {
+        removeLockFiles(fullPath);
+      } else if (entry === '.lock') {
+        unlinkSync(fullPath);
+      }
+    } catch {
+      // Best-effort removal
+    }
+  }
+}

extensions/vscode/src/bridge/environment-builder.ts

@@ -4,6 +4,13 @@ import { DisposableObject } from '../common/disposable';
 import type { Logger } from '../common/logger';
 import type { CliResolver } from '../codeql/cli-resolver';
 import type { StoragePaths } from './storage-paths';
+import { DatabaseCopier } from './database-copier';
+
+/** Factory that creates a DatabaseCopier for a given destination. */
+export type DatabaseCopierFactory = (dest: string, logger: Logger) => DatabaseCopier;
+
+const defaultCopierFactory: DatabaseCopierFactory = (dest, logger) =>
+  new DatabaseCopier(dest, logger);
 
 /**
  * Assembles the environment variables for the MCP server process.
@@ -19,14 +26,17 @@ import type { StoragePaths } from './storage-paths';
  */
 export class EnvironmentBuilder extends DisposableObject {
   private cachedEnv: Record<string, string> | null = null;
+  private readonly copierFactory: DatabaseCopierFactory;
 
   constructor(
     private readonly context: vscode.ExtensionContext,
     private readonly cliResolver: CliResolver,
     private readonly storagePaths: StoragePaths,
     private readonly logger: Logger,
+    copierFactory?: DatabaseCopierFactory,
   ) {
     super();
+    this.copierFactory = copierFactory ?? defaultCopierFactory;
   }
 
   /** Invalidate the cached environment so the next `build()` recomputes. */
@@ -83,9 +93,22 @@ export class EnvironmentBuilder extends DisposableObject {
 
     // Database discovery directories for list_codeql_databases
     // Includes: global storage, workspace storage, and user-configured dirs
-    const dbDirs = [...this.storagePaths.getAllDatabaseStoragePaths()];
+    const sourceDirs = this.storagePaths.getAllDatabaseStoragePaths();
     const userDbDirs = config.get<string[]>('additionalDatabaseDirs', []);
-    dbDirs.push(...userDbDirs);
+
+    // When copyDatabases is enabled, copy databases from vscode-codeql
+    // storage to our own managed directory, removing query-server lock
+    // files so the MCP server CLI can operate without contention.
+    const copyEnabled = config.get<boolean>('copyDatabases', true);
+    let dbDirs: string[];
+    if (copyEnabled) {
+      const managedDir = this.storagePaths.getManagedDatabaseStoragePath();
+      const copier = this.copierFactory(managedDir, this.logger);
+      copier.syncAll(sourceDirs);
+      dbDirs = [managedDir, ...userDbDirs];
+    } else {
+      dbDirs = [...sourceDirs, ...userDbDirs];
+    }
     env.CODEQL_DATABASES_BASE_DIRS = dbDirs.join(':');
 
     // MRVA run results directory for variant analysis discovery

extensions/vscode/src/bridge/storage-paths.ts

@@ -88,6 +88,14 @@ export class StoragePaths extends DisposableObject {
     return join(this.getCodeqlGlobalStoragePath(), 'variant-analyses');
   }
 
+  /**
+   * Directory where the MCP extension stores lock-free copies of databases.
+   * Path: `<our-globalStorageUri>/databases/`
+   */
+  getManagedDatabaseStoragePath(): string {
+    return join(this.context.globalStorageUri.fsPath, 'databases');
+  }
+
   /** The VS Code global storage root (parent of all extension storage dirs). */
   getGlobalStorageRoot(): string {
     return this.vsCodeGlobalStorageRoot;