feat(index): cross-process lock, unlock CLI, and errors.log (#129)

Acquire <state-dir>/index.lock before indexing with in-process serialization,
fail-fast when another live indexer holds the lock, and append parse failures
to errors.log. Adds codemap unlock for stale lock recovery.

Author: SutuSebastian

.changeset/index-lock-error-log.md

@@ -0,0 +1,5 @@
+---
+"@stainless-code/codemap": minor
+---
+
+Add cross-process index lock with `codemap unlock`, and append parse failures to `<state-dir>/errors.log`.

docs/agents.md

@@ -60,6 +60,8 @@ All integrations reuse the **same** bundled content under **`.agents/`**. Symlin
 
 When the file watcher is off (WSL `/mnt/*` mounts, `CODEMAP_WATCH=0`, etc.), **`codemap agents init --git-hooks`** installs marker-delimited blocks in **`post-commit`**, **`post-merge`**, and **`post-checkout`** that run `( codemap >/dev/null 2>&1 & )` — non-blocking background incremental index. **`--no-git-hooks`** removes only codemap-marked blocks. Interactive init offers hooks automatically when [`watch-policy.ts`](../src/application/watch-policy.ts) would disable the watcher for the project root.
 
+Concurrent indexers (CLI, MCP `--watch`, git hooks) coordinate via **`<state-dir>/index.lock`**. If indexing fails with “Index already running” after a crash, run **`codemap unlock`**. Per-file parse failures append to **`<state-dir>/errors.log`**.
+
 ## Pointer files
 
 Root / Copilot **pointer** files (**`CLAUDE.md`**, **`AGENTS.md`**, **`GEMINI.md`**, **`.github/copilot-instructions.md`**) use a **managed section** between **`<!-- codemap-pointer:begin -->`** and **`<!-- codemap-pointer:end -->`** (HTML comments — usually hidden in rendered Markdown):

docs/plans/agent-surface-delivery.md

@@ -10,11 +10,11 @@
 
 ## Quick resume
 
-| Next action           | Detail                                                                                                                                                                              |
-| --------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| **Review / merge P0** | PR [#126](https://github.com/stainless-code/codemap/pull/126) (MCP instructions + allowlist), PR [#127](https://github.com/stainless-code/codemap/pull/127) (WSL watch + git hooks) |
-| **Start next**        | **PR 3** — [`index-lock-and-error-log`](./index-lock-and-error-log.md) on branch from fresh `main`                                                                                  |
-| **Do not start yet**  | PR 6 (MCP trace tools) until PR 1 + PR 4 land; PR 9 (eval harness) until PR 1 + PR 8                                                                                                |
+| Next action          | Detail                                                                                                                               |
+| -------------------- | ------------------------------------------------------------------------------------------------------------------------------------ |
+| **Review / merge**   | PR 3 — index lock + error log (branch `feat/index-lock`) when open                                                                   |
+| **Start next**       | **PR 4** — trace recipes (`call-path`, `symbol-neighborhood`) or **PR 5** — `affected-tests-recipe` (parallel with 4 after 3 merges) |
+| **Do not start yet** | PR 6 (MCP trace tools) until PR 4 land; PR 9 (eval harness) until PR 8                                                               |
 
 Update the table below when a PR merges or a new branch opens.
 
@@ -26,24 +26,24 @@ Merge each PR to `main` directly. No long-lived integration branch (`feat/agent-
 
 ### Wave 1 — P0 (~1 week wall clock, 2 PRs)
 
-| PR    | Plans bundled                                                                                               | Status | Branch / link                                                                           | Notes                                                                                                                       |
-| ----- | ----------------------------------------------------------------------------------------------------------- | ------ | --------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
-| **1** | [`mcp-server-instructions`](./mcp-server-instructions.md) + [`mcp-tool-allowlist`](./mcp-tool-allowlist.md) | open   | [`feat/mcp-instructions-allowlist`](https://github.com/stainless-code/codemap/pull/126) | Same hot files (`mcp-server.ts`, `agent-content`); ~3–4 days                                                                |
-| **2** | [`wsl-watch-policy`](./wsl-watch-policy.md) → [`git-hook-auto-sync`](./git-hook-auto-sync.md)               | open   | [`feat/wsl-watch-git-hooks`](https://github.com/stainless-code/codemap/pull/127)        | `watch-policy.ts` first; hooks reference it in diagnostics. Lock deferred to PR 3 — note concurrent hook + MCP in PR 2 body |
+| PR    | Plans bundled                                                                                               | Status | Branch / link                                              | Notes                                                                                                                       |
+| ----- | ----------------------------------------------------------------------------------------------------------- | ------ | ---------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------- |
+| **1** | [`mcp-server-instructions`](./mcp-server-instructions.md) + [`mcp-tool-allowlist`](./mcp-tool-allowlist.md) | merged | [#126](https://github.com/stainless-code/codemap/pull/126) | Same hot files (`mcp-server.ts`, `agent-content`); ~3–4 days                                                                |
+| **2** | [`wsl-watch-policy`](./wsl-watch-policy.md) → [`git-hook-auto-sync`](./git-hook-auto-sync.md)               | merged | [#127](https://github.com/stainless-code/codemap/pull/127) | `watch-policy.ts` first; hooks reference it in diagnostics. Lock deferred to PR 3 — note concurrent hook + MCP in PR 2 body |
 
 ### Wave 2 — P1 (~2–3 weeks, parallel tracks)
 
 Max **3 parallel tracks** at once.
 
-| PR    | Plans                                                                                                                         | Status  | Blocked by                          | Parallel with                     |
-| ----- | ----------------------------------------------------------------------------------------------------------------------------- | ------- | ----------------------------------- | --------------------------------- |
-| **3** | [`index-lock-and-error-log`](./index-lock-and-error-log.md) → [`parse-worker-hardening`](./parse-worker-hardening.md) (stack) | planned | PR 2 merged (doc note on hook+lock) | 4, 5                              |
-| **4** | Recipe half of [`mcp-trace-explore-tools`](./mcp-trace-explore-tools.md) (`call-path`, `symbol-neighborhood` SQL + tests)     | planned | —                                   | 3, 5                              |
-| **5** | [`affected-tests-recipe`](./affected-tests-recipe.md)                                                                         | planned | —                                   | 3, 4                              |
-| **6** | MCP half of trace (`trace` / `explore` / `node` tools) + update instructions                                                  | planned | PR 1, PR 4                          | —                                 |
-| **7** | [`field-qualified-search`](./field-qualified-search.md)                                                                       | planned | PR 1                                | 4, 5 if `mcp-server.ts` untouched |
-| **8** | [`agents-init-mcp-wiring`](./agents-init-mcp-wiring.md)                                                                       | planned | PR 1                                | 3–5                               |
-| **9** | [`agent-eval-harness`](./agent-eval-harness.md)                                                                               | planned | PR 1, PR 8, allowlist               | **last P1**                       |
+| PR    | Plans                                                                                                                         | Status  | Blocked by            | Parallel with                     |
+| ----- | ----------------------------------------------------------------------------------------------------------------------------- | ------- | --------------------- | --------------------------------- |
+| **3** | [`index-lock-and-error-log`](./index-lock-and-error-log.md) → [`parse-worker-hardening`](./parse-worker-hardening.md) (stack) | open    | PR 2 merged           | 4, 5                              |
+| **4** | Recipe half of [`mcp-trace-explore-tools`](./mcp-trace-explore-tools.md) (`call-path`, `symbol-neighborhood` SQL + tests)     | planned | —                     | 3, 5                              |
+| **5** | [`affected-tests-recipe`](./affected-tests-recipe.md)                                                                         | planned | —                     | 3, 4                              |
+| **6** | MCP half of trace (`trace` / `explore` / `node` tools) + update instructions                                                  | planned | PR 1, PR 4            | —                                 |
+| **7** | [`field-qualified-search`](./field-qualified-search.md)                                                                       | planned | PR 1                  | 4, 5 if `mcp-server.ts` untouched |
+| **8** | [`agents-init-mcp-wiring`](./agents-init-mcp-wiring.md)                                                                       | planned | PR 1                  | 3–5                               |
+| **9** | [`agent-eval-harness`](./agent-eval-harness.md)                                                                               | planned | PR 1, PR 8, allowlist | **last P1**                       |
 
 **Parallelization constraints**
 

src/application/error-log.ts

@@ -0,0 +1,21 @@
+import { appendFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+
+/** Append-only per-file index failure log inside `<state-dir>/`. */
+export const ERROR_LOG_NAME = "errors.log";
+
+export function errorLogPath(stateDir: string): string {
+  return join(stateDir, ERROR_LOG_NAME);
+}
+
+/** One TSV line: ISO timestamp, file path, reason (tabs flattened). */
+export function appendIndexError(
+  stateDir: string,
+  filePath: string,
+  reason: string,
+): void {
+  mkdirSync(stateDir, { recursive: true });
+  const safeReason = reason.replace(/\t/g, " ").replace(/\n/g, " ");
+  const line = `${new Date().toISOString()}\t${filePath}\t${safeReason}\n`;
+  appendFileSync(errorLogPath(stateDir), line, "utf-8");
+}

src/application/index-engine.ts

@@ -57,6 +57,7 @@ import {
   getFts5Enabled,
   getIncludePatterns,
   getProjectRoot,
+  getStateDir,
   isPathExcluded,
 } from "../runtime";
 import { parseFilesParallel } from "../worker-pool";
@@ -67,6 +68,7 @@ import {
   resolveBindings,
 } from "./bindings-engine";
 import { persistModuleCycles } from "./cycles-engine";
+import { appendIndexError } from "./error-log";
 import { persistFileBarrelFlags } from "./file-graph-flags";
 import { persistJsxElementsAndAttributes } from "./jsx-persist";
 import type { QueryBindValue } from "./query-engine";
@@ -273,6 +275,15 @@ export function getCurrentCommit(): string {
   return result.stdout.toString().trim();
 }
 
+function reportParseError(relPath: string, reason: string): void {
+  console.error(`  Parse error in ${relPath}: ${reason}`);
+  try {
+    appendIndexError(getStateDir(), relPath, reason);
+  } catch {
+    // logging must not fail the index run
+  }
+}
+
 function insertParsedResults(
   db: CodemapDatabase,
   results: ParsedFile[],
@@ -283,7 +294,14 @@ function insertParsedResults(
 
   const transaction = db.transaction(() => {
     for (const parsed of results) {
-      if (parsed.error) continue;
+      if (parsed.error) {
+        reportParseError(parsed.relPath, "file read failed");
+        continue;
+      }
+
+      if (parsed.parseError) {
+        reportParseError(parsed.relPath, parsed.parseError);
+      }
 
       if (parsed.hasSideEffects) {
         parsed.fileRow.has_side_effects = parsed.hasSideEffects;
@@ -364,8 +382,9 @@ function insertParsedResults(
         if (parsed.suppressions?.length)
           insertSuppressions(db, parsed.suppressions);
       } catch (err) {
-        console.error(
-          `  Parse error in ${parsed.relPath}: ${err instanceof Error ? err.message : err}`,
+        reportParseError(
+          parsed.relPath,
+          err instanceof Error ? err.message : String(err),
         );
       }
 
@@ -600,8 +619,9 @@ export async function indexFiles(
           const suppressions = extractSuppressions(source, relPath);
           if (suppressions.length) insertSuppressions(db, suppressions);
         } catch (err) {
-          console.error(
-            `  Parse error in ${relPath}: ${err instanceof Error ? err.message : err}`,
+          reportParseError(
+            relPath,
+            err instanceof Error ? err.message : String(err),
           );
         }
 

src/application/index-lock.test.ts

@@ -0,0 +1,111 @@
+import { afterEach, describe, expect, it } from "bun:test";
+import {
+  existsSync,
+  mkdirSync,
+  mkdtempSync,
+  readFileSync,
+  rmSync,
+  writeFileSync,
+} from "node:fs";
+import { join } from "node:path";
+
+import { appendIndexError, errorLogPath } from "./error-log";
+import {
+  IndexLockHeldError,
+  acquireIndexLock,
+  indexLockPath,
+  isStaleLock,
+  readIndexLock,
+  removeIndexLock,
+} from "./index-lock";
+
+function scratchDir(prefix: string): string {
+  const base = join(process.cwd(), "fixtures", "tmp");
+  mkdirSync(base, { recursive: true });
+  return mkdtempSync(join(base, prefix));
+}
+
+describe("index-lock", () => {
+  let dir: string;
+
+  afterEach(() => {
+    if (dir) rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("acquireIndexLock creates JSON lock and release removes it", () => {
+    dir = scratchDir("index-lock-");
+    const release = acquireIndexLock(dir);
+    const lockPath = indexLockPath(dir);
+    expect(existsSync(lockPath)).toBe(true);
+    const payload = readIndexLock(lockPath);
+    expect(payload?.pid).toBe(process.pid);
+    release();
+    expect(existsSync(lockPath)).toBe(false);
+  });
+
+  it("second acquire fails fast while lock is held", () => {
+    dir = scratchDir("index-lock-held-");
+    const release = acquireIndexLock(dir);
+    try {
+      expect(() => acquireIndexLock(dir)).toThrow(IndexLockHeldError);
+    } finally {
+      release();
+    }
+  });
+
+  it("isStaleLock treats dead PID as stale", () => {
+    expect(
+      isStaleLock({ pid: 999_999_999, started_at: new Date().toISOString() }),
+    ).toBe(true);
+  });
+
+  it("removeIndexLock refuses live lock without force", () => {
+    dir = scratchDir("index-lock-live-");
+    const release = acquireIndexLock(dir);
+    try {
+      expect(() => removeIndexLock(dir)).toThrow(IndexLockHeldError);
+    } finally {
+      release();
+    }
+  });
+
+  it("removeIndexLock with force clears a live lock", () => {
+    dir = scratchDir("index-lock-force-");
+    const release = acquireIndexLock(dir);
+    expect(removeIndexLock(dir, { force: true })).toBe(true);
+    expect(existsSync(indexLockPath(dir))).toBe(false);
+    release();
+  });
+
+  it("auto-steals stale lock on acquire", () => {
+    dir = scratchDir("index-lock-stale-");
+    const lockPath = indexLockPath(dir);
+    mkdirSync(dir, { recursive: true });
+    writeFileSync(
+      lockPath,
+      `${JSON.stringify({
+        pid: 999_999_999,
+        started_at: new Date().toISOString(),
+      })}\n`,
+      "utf-8",
+    );
+    const release = acquireIndexLock(dir);
+    expect(readIndexLock(lockPath)?.pid).toBe(process.pid);
+    release();
+  });
+});
+
+describe("error-log", () => {
+  let dir: string;
+
+  afterEach(() => {
+    if (dir) rmSync(dir, { recursive: true, force: true });
+  });
+
+  it("appendIndexError writes TSV lines", () => {
+    dir = scratchDir("error-log-");
+    appendIndexError(dir, "src/a.ts", "boom");
+    const text = readFileSync(errorLogPath(dir), "utf-8");
+    expect(text).toContain("\tsrc/a.ts\tboom\n");
+  });
+});