fix(perf): scope WASM grammar load in engine-parity backfill (#1054) (#1058)

* fix(perf): scope WASM grammar load in engine-parity backfill

The native engine drops files in some build environments (#1054), triggering
a WASM backfill via the worker pool. The pool's first-call overhead is fine
for full builds (amortized over hundreds of files) but dwarfs the actual
parse work for small backfill batches — on slow CI runners, ~1.7s for 4
fixture files in one language.

Add `parseFilesWasmInline`: a main-thread, no-worker parse path that loads
only the grammars matching the input extensions and returns symbols with
`_tree` set so the unified walker in `runAnalyses` populates AST/CFG/dataflow
data downstream. New `parseFilesWasmForBackfill` chooses inline for batches
≤ 16 files, keeping worker isolation for larger batches where tree-sitter
WASM crash protection matters more (#965).

Routes both backfill sites through the new helper:
- `parseFilesAuto`'s per-call inline backfill in `domain/parser.ts`
- `backfillNativeDroppedFiles` in `domain/graph/builder/pipeline.ts`

Refs #1054

Impact: 4 functions changed, 13 affected

* fix(perf): free WASM trees after inline backfill (#1058)

The inline backfill path sets symbols._tree (live web-tree-sitter Tree
backed by WASM linear memory) on every result, but those symbols are
consumed locally for DB row construction in backfillNativeDroppedFiles
and never added to ctx.allSymbols, so the finalize-stage releaseWasmTrees
sweep never frees them. Without explicit cleanup, trees leak WASM memory
until process exit — bounded per run but cumulative across in-process
integration tests.

Adds a cleanup loop after batchInsertNodes that mirrors releaseWasmTrees,
and drops the now-unused parseFilesAuto import.

* docs(parser): explain INLINE_BACKFILL_THRESHOLD rationale (#1058)

Adds context for the 16-file threshold per Claude review feedback:
sized for typical engine-parity drops (recurring HCL case is 4 files);
above it, the worker-pool's IPC + crash-isolation cost is amortized over
enough parse work to be worth paying; below it, the cold-start dominates.

Author: carlos-alm

src/domain/graph/builder/pipeline.ts

@@ -37,7 +37,7 @@ import {
   formatDropExtensionSummary,
   getActiveEngine,
   getInstalledWasmExtensions,
-  parseFilesAuto,
+  parseFilesWasmForBackfill,
 } from '../../parser.js';
 import { setWorkspaces } from '../resolve.js';
 import { PipelineContext } from './context.js';
@@ -793,7 +793,7 @@ async function backfillNativeDroppedFiles(ctx: PipelineContext): Promise<void> {
       `Native orchestrator dropped ${totals['native-extractor-failure']} file(s) in natively-supported languages — likely a Rust extractor bug. Backfilling via WASM: ${formatDropExtensionSummary(byReason['native-extractor-failure'])}`,
     );
   }
-  const wasmResults = await parseFilesAuto(missingAbs, ctx.rootDir, { engine: 'wasm' });
+  const wasmResults = await parseFilesWasmForBackfill(missingAbs, ctx.rootDir);
 
   const rows: unknown[][] = [];
   const exportKeys: unknown[][] = [];
@@ -853,6 +853,27 @@ async function backfillNativeDroppedFiles(ctx: PipelineContext): Promise<void> {
       updateStmt.run(...vals);
     }
   }
+
+  // Free WASM parse trees from the inline backfill path (#1058).
+  // `parseFilesWasmInline` sets `symbols._tree` (a live web-tree-sitter Tree
+  // backed by WASM linear memory) on every result, but these symbols are
+  // consumed locally for DB row construction and never added to
+  // `ctx.allSymbols`, so the finalize-stage `releaseWasmTrees` sweep never
+  // sees them. Without this, trees leak WASM memory until process exit —
+  // bounded per run but cumulative across in-process integration tests.
+  // Mirrors the cleanup discipline established for #931.
+  for (const [, symbols] of wasmResults) {
+    const tree = (symbols as { _tree?: { delete?: () => void } })._tree;
+    if (tree && typeof tree.delete === 'function') {
+      try {
+        tree.delete();
+      } catch {
+        /* ignore cleanup errors */
+      }
+    }
+    (symbols as { _tree?: unknown; _langId?: unknown })._tree = undefined;
+    (symbols as { _tree?: unknown; _langId?: unknown })._langId = undefined;
+  }
 }
 
 // ── Pipeline stages execution ───────────────────────────────────────────

src/domain/parser.ts

@@ -1067,6 +1067,71 @@ async function parseFilesWasm(
   return result;
 }
 
+/**
+ * Files at or below this count use the inline parse path (no worker spawn).
+ *
+ * Sized for typical engine-parity drops: a handful of fixture files in one
+ * or two languages (the recurring HCL case is 4 files). Above this, the
+ * worker-pool's IPC + crash-isolation cost (#965) is amortized over enough
+ * parse work to be worth paying; below it, the ~1–2s cold-start dominates.
+ */
+const INLINE_BACKFILL_THRESHOLD = 16;
+
+/**
+ * Inline WASM parse (no worker) for small file batches.
+ *
+ * Used by the engine-parity backfill path when the native engine drops a
+ * handful of files (typically test fixtures). The worker pool's per-call
+ * IPC + grammar-init overhead can cost 1–2s on slow CI runners — for a
+ * 4-file backfill, that dwarfs the ~10ms of actual parse work.
+ *
+ * Returns symbols with `_tree` set so `runAnalyses` can run AST/CFG/dataflow
+ * visitors via the unified walker (mirrors how WASM-engine results behaved
+ * before the worker pool was introduced).
+ */
+async function parseFilesWasmInline(
+  filePaths: string[],
+  rootDir: string,
+): Promise<Map<string, ExtractorOutput>> {
+  const result = new Map<string, ExtractorOutput>();
+  if (filePaths.length === 0) return result;
+  const parsers = await ensureParsersForFiles(filePaths);
+  for (const filePath of filePaths) {
+    if (!_extToLang.has(path.extname(filePath).toLowerCase())) continue;
+    let code: string;
+    try {
+      code = fs.readFileSync(filePath, 'utf-8');
+    } catch (err: unknown) {
+      warn(`Skipping ${path.relative(rootDir, filePath)}: ${(err as Error).message}`);
+      continue;
+    }
+    const extracted = wasmExtractSymbols(parsers, filePath, code);
+    if (!extracted) continue;
+    const relPath = path.relative(rootDir, filePath).split(path.sep).join('/');
+    const symbols = extracted.symbols as ExtractorOutput & { _tree?: unknown; _langId?: string };
+    symbols._tree = extracted.tree;
+    symbols._langId = extracted.langId;
+    result.set(relPath, symbols);
+  }
+  return result;
+}
+
+/**
+ * Backfill helper: small batches use the inline (main-thread) path; larger
+ * batches keep the worker-pool isolation against tree-sitter WASM crashes
+ * (#965). Threshold matches typical engine-parity drop sizes (a few fixture
+ * files in one or two languages).
+ */
+export async function parseFilesWasmForBackfill(
+  filePaths: string[],
+  rootDir: string,
+): Promise<Map<string, ExtractorOutput>> {
+  if (filePaths.length <= INLINE_BACKFILL_THRESHOLD) {
+    return parseFilesWasmInline(filePaths, rootDir);
+  }
+  return parseFilesWasm(filePaths, rootDir);
+}
+
 /**
  * Parse multiple files in bulk and return a Map<relPath, symbols>.
  */
@@ -1117,7 +1182,7 @@ export async function parseFilesAuto(
   );
   if (dropped.length > 0) {
     warn(`Native engine dropped ${dropped.length} file(s); falling back to WASM for parity`);
-    const wasmResults = await parseFilesWasm(dropped, rootDir);
+    const wasmResults = await parseFilesWasmForBackfill(dropped, rootDir);
     for (const [relPath, symbols] of wasmResults) {
       result.set(relPath, symbols);
     }