fix(backend): add filesystem-first scan to GC to handle orphaned disk resources (#973)

* fix(backend): add filesystem-first scan to GC to handle orphaned disk resources

Adds cleanupOrphanedDiskResources() which walks the repos/ and index/
directories on disk and removes any entries with no corresponding Repo
record in the database. This handles desyncs caused by DB resets or
cascade deletes that bypass the normal cleanup job flow.

Also adds getRepoIdFromPath() to @sourcebot/shared and
getRepoIdFromShardFileName() to the backend utils as inverse helpers
to the existing getRepoPath() and getShardPrefix() functions.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>

* wip

---------

Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: brendan-kellam

CHANGELOG.md

@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Fixed
 - Preserve regex and case sensitivity query parameters when loading more search results. [#972](https://github.com/sourcebot-dev/sourcebot/pull/972)
 - Fixed page navigation failing after Next.js 16 upgrade by removing `router.refresh()` calls immediately following `router.push()`. [#974](https://github.com/sourcebot-dev/sourcebot/pull/974)
+- Add filesystem-first GC scan to remove orphaned repo directories and index shards that have no corresponding database record. [#973](https://github.com/sourcebot-dev/sourcebot/pull/973)
+
 
 ## [4.13.1] - 2026-02-28
 

packages/backend/src/index.ts

@@ -66,7 +66,7 @@ const repoIndexManager = new RepoIndexManager(prisma, settings, redis, promClien
 const configManager = new ConfigManager(prisma, connectionManager, env.CONFIG_PATH);
 
 connectionManager.startScheduler();
-repoIndexManager.startScheduler();
+await repoIndexManager.startScheduler();
 
 if (env.EXPERIMENT_EE_PERMISSION_SYNC_ENABLED === 'true' && !hasEntitlement('permission-syncing')) {
     logger.error('Permission syncing is not supported in current plan. Please contact team@sourcebot.dev for assistance.');

packages/backend/src/repoIndexManager.ts

@@ -1,18 +1,18 @@
 import * as Sentry from '@sentry/node';
 import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
-import { createLogger, env, getRepoPath, Logger, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared";
+import { createLogger, env, getRepoPath, Logger, getRepoIdFromPath, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from "@sourcebot/shared";
 import { DelayedError, Job, Queue, Worker } from "bullmq";
 import { existsSync } from 'fs';
 import { readdir, rm } from 'fs/promises';
 import { Redis } from 'ioredis';
 import micromatch from 'micromatch';
 import Redlock, { ExecutionError } from 'redlock';
-import { INDEX_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js';
+import { INDEX_CACHE_DIR, REPOS_CACHE_DIR, WORKER_STOP_GRACEFUL_TIMEOUT_MS } from './constants.js';
 import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName, getLatestCommitTimestamp, getLocalDefaultBranch, getTags, isPathAValidGitRepoRoot, isRepoEmpty, unsetGitConfig, upsertGitConfig } from './git.js';
 import { captureEvent } from './posthog.js';
 import { PromClient } from './promClient.js';
 import { RepoWithConnections, Settings } from "./types.js";
-import { getAuthCredentialsForRepo, getShardPrefix, measure, setIntervalAsync } from './utils.js';
+import { getAuthCredentialsForRepo, getRepoIdFromShardFileName, getShardPrefix, measure, setIntervalAsync } from './utils.js';
 import { cleanupTempShards, indexGitRepository } from './zoekt.js';
 
 const LOG_TAG = 'repo-index-manager';
@@ -96,8 +96,10 @@ export class RepoIndexManager {
         });
     }
 
-    public startScheduler() {
+    public async startScheduler() {
         logger.debug('Starting scheduler');
+        // Cleanup any orphaned disk resources on startup
+        await this.cleanupOrphanedDiskResources();
         this.interval = setIntervalAsync(async () => {
             await this.scheduleIndexJobs();
             await this.scheduleCleanupJobs();
@@ -637,6 +639,71 @@ export class RepoIndexManager {
         }
     }
 
+    // Scans the repos and index directories on disk and removes any entries
+    // that have no corresponding Repo record in the database. This handles
+    // edge cases where the DB and disk resources are out of sync.
+    private async cleanupOrphanedDiskResources() {
+        // --- Repo directories ---
+        // Dirs are named by repoId: DATA_CACHE_DIR/repos/<repoId>/
+        if (existsSync(REPOS_CACHE_DIR)) {
+            const entries = await readdir(REPOS_CACHE_DIR);
+            const repoIdToPath = new Map<number, string>();
+            for (const entry of entries) {
+                const repoPath = `${REPOS_CACHE_DIR}/${entry}`;
+                const repoId = getRepoIdFromPath(repoPath);
+                if (repoId !== undefined) {
+                    repoIdToPath.set(repoId, repoPath);
+                }
+            }
+
+            if (repoIdToPath.size > 0) {
+                const existingRepos = await this.db.repo.findMany({
+                    where: { id: { in: [...repoIdToPath.keys()] } },
+                    select: { id: true },
+                });
+                const existingIds = new Set(existingRepos.map(r => r.id));
+                for (const [repoId, repoPath] of repoIdToPath) {
+                    if (!existingIds.has(repoId)) {
+                        logger.info(`Removing orphaned repo directory with no DB record: ${repoPath}`);
+                        await rm(repoPath, { recursive: true, force: true });
+                    }
+                }
+            }
+        }
+
+        // --- Index shards ---
+        // Shard files are prefixed with <orgId>_<repoId>: DATA_CACHE_DIR/index/<orgId>_<repoId>_*.zoekt
+        if (existsSync(INDEX_CACHE_DIR)) {
+            const entries = await readdir(INDEX_CACHE_DIR);
+            const repoIdToShards = new Map<number, string[]>();
+            for (const entry of entries) {
+                const repoId = getRepoIdFromShardFileName(entry);
+                if (repoId !== undefined) {
+                    const shards = repoIdToShards.get(repoId) ?? [];
+                    shards.push(entry);
+                    repoIdToShards.set(repoId, shards);
+                }
+            }
+
+            if (repoIdToShards.size > 0) {
+                const existingRepos = await this.db.repo.findMany({
+                    where: { id: { in: [...repoIdToShards.keys()] } },
+                    select: { id: true },
+                });
+                const existingIds = new Set(existingRepos.map(r => r.id));
+                for (const [repoId, shards] of repoIdToShards) {
+                    if (!existingIds.has(repoId)) {
+                        for (const entry of shards) {
+                            const shardPath = `${INDEX_CACHE_DIR}/${entry}`;
+                            logger.info(`Removing orphaned index shard with no DB record: ${shardPath}`);
+                            await rm(shardPath, { force: true });
+                        }
+                    }
+                }
+            }
+        }
+    }
+
     public async dispose() {
         if (this.interval) {
             clearInterval(this.interval);

packages/backend/src/utils.ts

@@ -57,6 +57,14 @@ export const getShardPrefix = (orgId: number, repoId: number) => {
     return `${orgId}_${repoId}`;
 }
 
+export const getRepoIdFromShardFileName = (fileName: string): number | undefined => {
+    const match = fileName.match(/^(\d+)_(\d+)_/);
+    if (!match) {
+        return undefined;
+    }
+    return parseInt(match[2], 10);
+}
+
 export const fetchWithRetry = async <T>(
     fetchFn: () => Promise<T>,
     identifier: string,

packages/shared/src/index.server.ts

@@ -24,6 +24,7 @@ export {
     loadJsonFile,
     getConfigSettings,
     getRepoPath,
+    getRepoIdFromPath,
 } from "./utils.js";
 export * from "./constants.js";
 export {

packages/shared/src/utils.ts

@@ -87,6 +87,11 @@ export const getConfigSettings = async (configPath?: string): Promise<ConfigSett
     }
 }
 
+export const getRepoIdFromPath = (repoPath: string): number | undefined => {
+    const id = parseInt(path.basename(repoPath), 10);
+    return isNaN(id) ? undefined : id;
+}
+
 export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => {
     // If we are dealing with a local repository, then use that as the path.
     // Mark as read-only since we aren't guaranteed to have write access to the local filesystem.