Merge pull request giancarloerra#72 from giancarloerra/feat/auto-resume-all-projects

giancarloerra · web-flow · commit db2e03877b2c · 2026-06-11T11:49:30.000+01:00
feat(startup): opt-in auto-resume of all indexed projects
diff --git a/README.md b/README.md
@@ -1216,6 +1216,8 @@ The rest of this section documents the variables themselves. Pass them using whi
 | `SOCRATICODE_PROJECT_ID` | *(none)* | Override the auto-generated project ID. When set, all paths resolve to the same Qdrant collections, allowing multiple directories (e.g. git worktrees of the same repo) to share a single index. Must match `[a-zA-Z0-9_-]+`. Takes precedence over the `projectId` field in `.socraticode.json`. |
 | `SOCRATICODE_BRANCH_AWARE` | `false` | When `true`, append the current git branch name to the project ID, creating separate Qdrant collections per branch. Ignored when `SOCRATICODE_PROJECT_ID` is set or when `projectId` is set in `.socraticode.json`. |
 | `SOCRATICODE_LINKED_PROJECTS` | *(none)* | Comma-separated list of additional project paths to include in cross-project search. Merged with paths from `.socraticode.json`. Non-existent paths are silently skipped. |
+| `SOCRATICODE_AUTO_RESUME` | *(none)* | When set to `all`, server startup auto-resumes the file watcher plus an incremental catch-up update for **every** indexed project that has a stored path, not just the current working directory. Projects resume one at a time (sequentially) to avoid overloading the embedding provider. Skipped projects (directory no longer exists, or indexed before path tracking was added) are logged at warn level. Useful when one MCP server session should keep many canonical checkouts fresh. |
+| `SOCRATICODE_AUTO_RESUME_PROJECTS` | *(none)* | Comma-separated list of project paths to auto-resume on server startup (sequentially), e.g. `/repos/api,/repos/web`. Takes precedence over `SOCRATICODE_AUTO_RESUME`. Paths that do not exist or are not indexed are skipped with a warning. |
 | `SOCRATICODE_LOG_LEVEL` | `info` | Log verbosity: `debug`, `info`, `warn`, `error` |
 | `SOCRATICODE_LOG_FILE` | *(none)* | Absolute path to a log file. When set, all log entries are appended to this file (a session separator is written on each server start). Useful for debugging when the MCP host doesn't surface log notifications. |
 
@@ -1383,6 +1385,11 @@ The watcher will not auto-start if a full index or incremental update is current
 progress, if the project has not been indexed yet, or if another MCP process is already
 watching the same project.
 
+If you work across many indexed repos and want all of them resumed at server startup
+(watcher plus catch-up update), not just the one you opened, see the
+`SOCRATICODE_AUTO_RESUME` and `SOCRATICODE_AUTO_RESUME_PROJECTS` environment variables
+in the [Indexing Behaviour](#indexing-behaviour) table.
+
 ### Can multiple AI agents work on the same codebase at the same time?
 
 Yes — this is a first-class supported workflow. When multiple agents (each running their own MCP server instance) are pointed at the same project directory, they automatically share the same Qdrant index. The first agent to trigger indexing acquires a cross-process lock and builds the index; any other agent that tries to index simultaneously receives current progress instead of starting a duplicate operation. All agents can search concurrently with no coordination needed — Qdrant handles parallel reads natively.
diff --git a/src/services/startup.ts b/src/services/startup.ts
@@ -8,31 +8,46 @@
  * of the MCP stdio transport.
  */
 
+import fs from "node:fs";
+import path from "node:path";
 import { collectionName, projectIdFromPath } from "../config.js";
-import { QDRANT_MODE } from "../constants.js";
+import { QDRANT_COLLECTION_PREFIX, QDRANT_MODE } from "../constants.js";
 import { isDockerAvailable, isQdrantRunning } from "./docker.js";
 import { getIndexingInProgressProjects, getPersistedIndexingStatus, indexProject, requestCancellation, updateProjectIndex } from "./indexer.js";
 import { getLockHolderPid, releaseAllLocks } from "./lock.js";
 import { logger } from "./logger.js";
-import { listCodebaseCollections } from "./qdrant.js";
+import { getProjectMetadata, listCodebaseCollections } from "./qdrant.js";
 import { isWatching, startWatching, stopAllWatchers } from "./watcher.js";
 
 /**
- * Auto-resume for the current project (process.cwd()) on server startup.
+ * Auto-resume indexed projects on server startup.
  *
- * If the project has a completed index:
- *   - Start file watcher
- *   - Run incremental update to catch changes made while the MCP was down
+ * Which projects are resumed is controlled by two opt-in environment
+ * variables (issue #70). Default (both unset) keeps the original behavior:
+ * only the project at process.cwd() is considered.
  *
- * If the project has an incomplete (interrupted) index:
- *   - Run indexProject which skips already-hashed files and embeds the rest,
- *     correctly resuming the full index (supports codebase_stop cancellation)
- *   - Watcher starts automatically after indexProject completes
+ *   - SOCRATICODE_AUTO_RESUME_PROJECTS: comma-separated list of project
+ *     paths to resume. Takes precedence over SOCRATICODE_AUTO_RESUME.
+ *   - SOCRATICODE_AUTO_RESUME=all: resume every indexed project that has a
+ *     stored path in its Qdrant metadata.
+ *
+ * Multi-project modes resume strictly sequentially: each project's catch-up
+ * (incremental update or interrupted-index recovery) completes before the
+ * next starts, so N projects never stampede the embedding provider. Skipped
+ * projects (missing directory, no stored path, not indexed) are logged at
+ * warn level so misconfiguration is visible.
+ *
+ * Per project, if the index is complete: start the file watcher and run an
+ * incremental update to catch changes made while the MCP was down. If the
+ * index is incomplete (interrupted): resume it via indexProject, which skips
+ * already-hashed files and honours codebase_stop cancellation; the watcher
+ * starts after recovery completes.
  *
  * Runs in the background after server startup — non-blocking, non-fatal.
  *
- * @param projectPath - Override project path (defaults to process.cwd()).
- *   Only used by tests — production always uses process.cwd().
+ * @param projectPath - Override project path for the default single-project
+ *   mode (defaults to process.cwd()). Only used by tests — production always
+ *   uses process.cwd().
  */
 export async function autoResumeIndexedProjects(projectPath?: string): Promise<void> {
   try {
@@ -49,7 +64,28 @@ export async function autoResumeIndexedProjects(projectPath?: string): Promise<v
       }
     }
 
-    // Only consider the current project
+    // Read at call time (not module load) so tests and MCP host configs
+    // that set the variables after import are honoured.
+    const explicitList = process.env.SOCRATICODE_AUTO_RESUME_PROJECTS?.trim();
+    const resumeMode = process.env.SOCRATICODE_AUTO_RESUME?.trim();
+
+    if (explicitList) {
+      await resumeProjectList(explicitList);
+      return;
+    }
+
+    if (resumeMode) {
+      if (resumeMode.toLowerCase() === "all") {
+        await resumeAllIndexedProjects();
+        return;
+      }
+      logger.warn("Auto-resume: unrecognized SOCRATICODE_AUTO_RESUME value, using default behavior (current project only)", {
+        value: resumeMode,
+      });
+      // Fall through to the default single-project behavior below.
+    }
+
+    // Default: only consider the current project
     const resolvedPath = projectPath ?? process.cwd();
 
     // If CWD is root or home, the MCP host hasn't opened a specific project yet — skip
@@ -58,92 +94,218 @@ export async function autoResumeIndexedProjects(projectPath?: string): Promise<v
       return;
     }
 
-    const projectId = projectIdFromPath(resolvedPath);
-    const collection = collectionName(projectId);
-
     const collections = await listCodebaseCollections();
-    if (!collections.includes(collection)) {
-      logger.info("Auto-resume: current project not yet indexed, skipping", { projectPath: resolvedPath });
-      return;
+    await resumeProject(resolvedPath, collections, "info");
+  } catch (err) {
+    logger.warn("Auto-resume failed (non-fatal)", {
+      error: err instanceof Error ? err.message : String(err),
+    });
+  }
+}
+
+/**
+ * Resume the explicit comma-separated project list from
+ * SOCRATICODE_AUTO_RESUME_PROJECTS, sequentially. Paths are resolved to
+ * absolute and deduplicated; entries whose directory does not exist are
+ * skipped with a warning. One project failing does not stop the rest.
+ */
+async function resumeProjectList(rawList: string): Promise<void> {
+  const targets = [...new Set(
+    rawList
+      .split(",")
+      .map((p) => p.trim())
+      .filter((p) => p.length > 0)
+      .map((p) => path.resolve(p)),
+  )];
+
+  if (targets.length === 0) {
+    logger.warn("Auto-resume: SOCRATICODE_AUTO_RESUME_PROJECTS is set but contains no paths");
+    return;
+  }
+
+  logger.info("Auto-resume: resuming explicit project list sequentially", { count: targets.length });
+  const collections = await listCodebaseCollections();
+
+  for (const target of targets) {
+    if (!fs.existsSync(target)) {
+      logger.warn("Auto-resume: listed project directory does not exist, skipping", { projectPath: target });
+      continue;
     }
+    try {
+      await resumeProject(target, collections, "warn");
+    } catch (err) {
+      logger.warn("Auto-resume: project resume failed, continuing with remaining projects", {
+        projectPath: target,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+}
 
-    // Check persisted indexing status to detect interrupted indexing
-    const persistedStatus = await getPersistedIndexingStatus(resolvedPath);
+/**
+ * Resume every indexed project found in Qdrant (SOCRATICODE_AUTO_RESUME=all),
+ * sequentially. Project paths come from each collection's stored metadata;
+ * collections without a stored path (indexed before path tracking) and paths
+ * that no longer exist on disk are skipped with a warning.
+ */
+async function resumeAllIndexedProjects(): Promise<void> {
+  const collections = await listCodebaseCollections();
+  const codebasePrefix = `${QDRANT_COLLECTION_PREFIX}codebase_`;
+  const codebaseCollections = collections.filter((c) => c.startsWith(codebasePrefix));
 
-    if (persistedStatus === "in-progress") {
-      // Check if another process is already indexing (orphan from previous session)
-      const orphanPid = await getLockHolderPid(resolvedPath, "index");
-      if (orphanPid !== null) {
-        logger.info("Auto-resume: skipping — another process is still indexing this project", {
-          projectPath: resolvedPath, orphanPid,
-        });
-        return;
-      }
+  if (codebaseCollections.length === 0) {
+    logger.info("Auto-resume: no indexed projects found, nothing to resume");
+    return;
+  }
 
-      // Index was interrupted — resume it via indexProject.
-      // indexProject skips already-hashed files, embeds the rest, and honours
-      // codebase_stop cancellation requests (unlike updateProjectIndex's incremental diff).
-      // Do NOT start watcher yet — it will start after indexing completes.
-      logger.info("Auto-resume: detected incomplete index, resuming indexing", { projectPath: resolvedPath });
-      const resumeOnProgress = (msg: string) => logger.info(msg, { tool: "auto-resume", projectPath: resolvedPath });
-      indexProject(resolvedPath, resumeOnProgress)
-        .then(async (result) => {
-          logger.info("Auto-resume: incomplete index recovery completed", {
-            projectPath: resolvedPath,
-            filesIndexed: result.filesIndexed,
-            chunksCreated: result.chunksCreated,
-            cancelled: result.cancelled,
-          });
-          // Now start the watcher after recovery — only if not cancelled
-          if (!result.cancelled && !isWatching(resolvedPath)) {
-            const started = await startWatching(resolvedPath);
-            if (started) {
-              logger.info("Auto-resume: started file watcher after recovery", { projectPath: resolvedPath });
-            }
-          }
-        })
-        .catch((err) => {
-          logger.warn("Auto-resume: incomplete index recovery failed (non-fatal)", {
-            projectPath: resolvedPath,
-            error: err instanceof Error ? err.message : String(err),
-          });
-        });
+  const targets: string[] = [];
+  const seen = new Set<string>();
+  for (const coll of codebaseCollections) {
+    const metadata = await getProjectMetadata(coll);
+    if (!metadata?.projectPath) {
+      logger.warn("Auto-resume: project has no stored path (indexed before path tracking), cannot auto-resume. Re-index it once to fix this permanently.", {
+        collection: coll,
+      });
+      continue;
+    }
+    const resolved = path.resolve(metadata.projectPath);
+    if (seen.has(resolved)) continue;
+    seen.add(resolved);
+    if (!fs.existsSync(resolved)) {
+      logger.warn("Auto-resume: indexed project directory no longer exists, skipping", {
+        projectPath: resolved,
+        collection: coll,
+      });
+      continue;
+    }
+    targets.push(resolved);
+  }
+
+  logger.info("Auto-resume: resuming all indexed projects sequentially", { count: targets.length });
+
+  for (const target of targets) {
+    try {
+      await resumeProject(target, collections, "warn");
+    } catch (err) {
+      logger.warn("Auto-resume: project resume failed, continuing with remaining projects", {
+        projectPath: target,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+  }
+}
+
+/**
+ * Resume a single project: start its watcher and catch up its index.
+ * Awaits the catch-up work to completion so callers can sequence multiple
+ * projects without overlapping embedding load.
+ *
+ * @param resolvedPath - Absolute project path.
+ * @param collections - Pre-fetched Qdrant collection names (fetched once per
+ *   startup so multi-project modes do not re-query per project).
+ * @param skipLogLevel - "warn" in multi-project modes so misconfigured
+ *   entries are visible; "info" for the default cwd mode (an unindexed cwd
+ *   is normal, not a misconfiguration).
+ */
+async function resumeProject(
+  resolvedPath: string,
+  collections: string[],
+  skipLogLevel: "info" | "warn",
+): Promise<void> {
+  const projectId = projectIdFromPath(resolvedPath);
+  const collection = collectionName(projectId);
+
+  if (!collections.includes(collection)) {
+    if (skipLogLevel === "warn") {
+      logger.warn("Auto-resume: project not yet indexed, skipping", { projectPath: resolvedPath });
+    } else {
+      logger.info("Auto-resume: current project not yet indexed, skipping", { projectPath: resolvedPath });
+    }
+    return;
+  }
+
+  // Check persisted indexing status to detect interrupted indexing
+  const persistedStatus = await getPersistedIndexingStatus(resolvedPath);
+
+  if (persistedStatus === "in-progress") {
+    // Check if another process is already indexing (orphan from previous session)
+    const orphanPid = await getLockHolderPid(resolvedPath, "index");
+    if (orphanPid !== null) {
+      logger.info("Auto-resume: skipping — another process is still indexing this project", {
+        projectPath: resolvedPath, orphanPid,
+      });
       return;
     }
 
-    // Index is complete — start watcher and do incremental catch-up
-    if (!isWatching(resolvedPath)) {
+    // Index was interrupted — resume it via indexProject.
+    // indexProject skips already-hashed files, embeds the rest, and honours
+    // codebase_stop cancellation requests (unlike updateProjectIndex's incremental diff).
+    // Do NOT start watcher yet — it will start after indexing completes.
+    logger.info("Auto-resume: detected incomplete index, resuming indexing", { projectPath: resolvedPath });
+    const resumeOnProgress = (msg: string) => logger.info(msg, { tool: "auto-resume", projectPath: resolvedPath });
+    try {
+      const result = await indexProject(resolvedPath, resumeOnProgress);
+      logger.info("Auto-resume: incomplete index recovery completed", {
+        projectPath: resolvedPath,
+        filesIndexed: result.filesIndexed,
+        chunksCreated: result.chunksCreated,
+        cancelled: result.cancelled,
+      });
+      // Now start the watcher after recovery — only if not cancelled
+      if (!result.cancelled && !isWatching(resolvedPath)) {
+        const started = await startWatching(resolvedPath);
+        if (started) {
+          logger.info("Auto-resume: started file watcher after recovery", { projectPath: resolvedPath });
+        }
+      }
+    } catch (err) {
+      logger.warn("Auto-resume: incomplete index recovery failed (non-fatal)", {
+        projectPath: resolvedPath,
+        error: err instanceof Error ? err.message : String(err),
+      });
+    }
+    return;
+  }
+
+  // Index is complete — start watcher and do incremental catch-up.
+  // Watcher startup failure must not skip the catch-up below: a project
+  // without a live watcher is degraded, but a project with a stale index
+  // is broken for search.
+  if (!isWatching(resolvedPath)) {
+    try {
       const started = await startWatching(resolvedPath);
       if (started) {
         logger.info("Auto-resume: started file watcher", { projectPath: resolvedPath });
       }
+    } catch (err) {
+      logger.warn("Auto-resume: failed to start watcher, continuing with incremental catch-up", {
+        projectPath: resolvedPath,
+        error: err instanceof Error ? err.message : String(err),
+      });
     }
+  }
 
-    // Incremental update in the background — only re-embeds actually changed files
-    // Note: updateProjectIndex now handles code graph rebuild internally
-    updateProjectIndex(resolvedPath)
-      .then(async (result) => {
-        const changed = result.added + result.updated + result.removed;
-        if (changed > 0) {
-          logger.info("Auto-resume: incremental update completed", {
-            projectPath: resolvedPath,
-            added: result.added,
-            updated: result.updated,
-            removed: result.removed,
-            filesChanged: changed,
-          });
-        } else {
-          logger.info("Auto-resume: index is up to date", { projectPath: resolvedPath });
-        }
-      })
-      .catch((err) => {
-        logger.warn("Auto-resume: incremental update failed (non-fatal)", {
-          projectPath: resolvedPath,
-          error: err instanceof Error ? err.message : String(err),
-        });
+  // Incremental update: only re-embeds actually changed files. Awaited so
+  // multi-project resume stays sequential (auto-resume as a whole already
+  // runs in the background, so this blocks nothing user-facing).
+  // Note: updateProjectIndex handles code graph rebuild internally.
+  try {
+    const result = await updateProjectIndex(resolvedPath);
+    const changed = result.added + result.updated + result.removed;
+    if (changed > 0) {
+      logger.info("Auto-resume: incremental update completed", {
+        projectPath: resolvedPath,
+        added: result.added,
+        updated: result.updated,
+        removed: result.removed,
+        filesChanged: changed,
       });
+    } else {
+      logger.info("Auto-resume: index is up to date", { projectPath: resolvedPath });
+    }
   } catch (err) {
-    logger.warn("Auto-resume failed (non-fatal)", {
+    logger.warn("Auto-resume: incremental update failed (non-fatal)", {
+      projectPath: resolvedPath,
       error: err instanceof Error ? err.message : String(err),
     });
   }
diff --git a/tests/unit/startup.test.ts b/tests/unit/startup.test.ts
