fix(backend): Limit concurrent git operations to prevent resource exhaustion (#590)

When syncing generic-git-host connections with thousands of repositories,
unbounded Promise.all caused resource exhaustion (EAGAIN errors) by spawning
too many concurrent git processes. This resulted in valid repositories being
incorrectly skipped during sync.

- Add p-limit to control concurrent git operations (max 100)
- Follow existing pattern from github.ts for consistency
- Prevents file descriptor and process limit exhaustion
- Uses rolling concurrency to avoid head-of-line blocking

Fixes #590

Author: derek-miller

packages/backend/src/repoCompileUtils.ts

@@ -20,11 +20,17 @@ import assert from 'assert';
 import GitUrlParse from 'git-url-parse';
 import { RepoMetadata } from '@sourcebot/shared';
 import { SINGLE_TENANT_ORG_ID } from './constants.js';
+import pLimit from 'p-limit';
 
 export type RepoData = WithRequired<Prisma.RepoCreateInput, 'connections'>;
 
 const logger = createLogger('repo-compile-utils');
 
+// Limit concurrent git operations to prevent resource exhaustion (EAGAIN errors)
+// when processing thousands of repositories simultaneously
+const MAX_CONCURRENT_GIT_OPERATIONS = 100;
+const gitOperationLimit = pLimit(MAX_CONCURRENT_GIT_OPERATIONS);
+
 type CompileResult = {
     repoData: RepoData[],
     warnings: string[],
@@ -472,7 +478,7 @@ export const compileGenericGitHostConfig_file = async (
     const repos: RepoData[] = [];
     const warnings: string[] = [];
 
-    await Promise.all(repoPaths.map(async (repoPath) => {
+    await Promise.all(repoPaths.map((repoPath) => gitOperationLimit(async () => {
         const isGitRepo = await isPathAValidGitRepoRoot({
             path: repoPath,
         });
@@ -526,7 +532,7 @@ export const compileGenericGitHostConfig_file = async (
         }
 
         repos.push(repo);
-    }));
+    })));
 
     return {
         repoData: repos,