fix(backend): add filesystem-first scan to GC to handle orphaned disk resources

Adds cleanupOrphanedDiskResources() which walks the repos/ and index/
directories on disk and removes any entries with no corresponding Repo
record in the database. This handles desyncs caused by DB resets or
cascade deletes that bypass the normal cleanup job flow.

Also adds getRepoIdFromPath() to @sourcebot/shared and
getRepoIdFromShardFileName() to the backend utils as inverse helpers
to the existing getRepoPath() and getShardPrefix() functions.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: brendan-kellam

packages/backend/src/repoIndexManager.ts

@@ -1,18 +1,18 @@
 import * as Sentry from '@sentry/node';
 import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
-import { createLogger, env, getRepoPath, Logger, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared";
+import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared";
 import { DelayedError, Job, Queue, Worker } from "bullmq";
 import { existsSync } from 'fs';
 import { readdir, rm } from 'fs/promises';
 import { Redis } from 'ioredis';
 import micromatch from 'micromatch';
 import Redlock, { ExecutionError } from 'redlock';
-import { INDEX_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js';
+import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js';
 import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig } from './git.js';
 import { captureEvent } from './posthog.js';
 import { PromClient } from './promClient.js';
 import { RepoWithConnections, Settings } from "./types.js";
-import { getAuthCredentialsForRepo, getShardPrefix, measure, setIntervalAsync } from './utils.js';
+import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js';
 import { cleanupTempShards, indexGitRepository } from './zoekt.js';
 
 const LOG_TAG = 'repo-index-manager';
@@ -170,6 +170,8 @@ export class RepoIndexManager {
     }
 
     private async scheduleCleanupJobs() {
+        await this.cleanupOrphanedDiskResources();
+
         const gcGracePeriodMs = new Date(Date.now() - this.settings.repoGarbageCollectionGracePeriodMs);
         const timeoutDate = new Date(Date.now() - this.settings.repoIndexTimeoutMs);
 
@@ -637,6 +639,48 @@ export class RepoIndexManager {
         }
     }
 
+    // Scans the repos and index directories on disk and removes any entries
+    // that have no corresponding Repo record in the database. This handles
+    // edge cases where the DB and disk resources are out of sync.
+    private async cleanupOrphanedDiskResources() {
+        // --- Repo directories ---
+        // Dirs are named by repoId: DATA_CACHE_DIR/repos/<repoId>/
+        if (existsSync(REPOS_CACHE_DIR)) {
+            const entries = await readdir(REPOS_CACHE_DIR);
+            for (const entry of entries) {
+                const repoPath = `${REPOS_CACHE_DIR}/${entry}`;
+                const repoId = getRepoIdFromPath(repoPath);
+                if (repoId === undefined) {
+                    continue;
+                }
+
+                const repo = await this.db.repo.findUnique({ where: { id: repoId } });
+                if (!repo) {
+                    logger.info(`Removing orphaned repo directory with no DB record: ${repoPath}`);
+                    await rm(repoPath, { recursive: true, force: true });
+                }
+            }
+        }
+
+        // --- Index shards ---
+        // Shard files are prefixed with <orgId>_<repoId>: DATA_CACHE_DIR/index/<orgId>_<repoId>_*.zoekt
+        if (existsSync(INDEX_CACHE_DIR)) {
+            const entries = await readdir(INDEX_CACHE_DIR);
+            for (const entry of entries) {
+                const repoId = getRepoIdFromShardFileName(entry);
+                if (repoId === undefined) {
+                    continue;
+                }
+                const repo = await this.db.repo.findUnique({ where: { id: repoId } });
+                if (!repo) {
+                    const shardPath = `${INDEX_CACHE_DIR}/${entry}`;
+                    logger.info(`Removing orphaned index shard with no DB record: ${shardPath}`);
+                    await rm(shardPath, { force: true });
+                }
+            }
+        }
+    }
+
     public async dispose() {
         if (this.interval) {
             clearInterval(this.interval);

packages/backend/src/utils.ts

@@ -57,6 +57,14 @@ export const getShardPrefix = (orgId: number, repoId: number) => {
     return `${orgId}_${repoId}`;
 }
 
+export const getRepoIdFromShardFileName = (fileName: string): number | undefined => {
+    const match = fileName.match(/^(\d+)_(\d+)_/);
+    if (!match) {
+        return undefined;
+    }
+    return parseInt(match[2], 10);
+}
+
 export const fetchWithRetry = async <T>(
     fetchFn: () => Promise<T>,
     identifier: string,

packages/shared/src/index.server.ts

@@ -24,6 +24,7 @@ export {
     loadJsonFile,
     getConfigSettings,
     getRepoPath,
+    getRepoIdFromPath,
 } from "./utils.js";
 export * from "./constants.js";
 export {

packages/shared/src/utils.ts

@@ -87,6 +87,11 @@ export const getConfigSettings = async (configPath?: string): Promise<ConfigSett
     }
 }
 
+export const getRepoIdFromPath = (repoPath: string): number | undefined => {
+    const id = parseInt(path.basename(repoPath), 10);
+    return isNaN(id) ? undefined : id;
+}
+
 export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => {
     // If we are dealing with a local repository, then use that as the path.
     // Mark as read-only since we aren't guaranteed to have write access to the local filesystem.