feedback

Author: brendan-kellam

packages/backend/src/repoIndexManager.ts

@@ -1,7 +1,7 @@
 import * as Sentry from '@sentry/node';
 import { PrismaClient, Repo, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
 import { createLogger, Logger } from "@sourcebot/shared";
-import { env, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema } from '@sourcebot/shared';
+import { env, RepoIndexingJobMetadata, repoIndexingJobMetadataSchema, RepoMetadata, repoMetadataSchema, getRepoPath } from '@sourcebot/shared';
 import { existsSync } from 'fs';
 import { readdir, rm } from 'fs/promises';
 import { Job, Queue, ReservedJob, Worker } from "groupmq";
@@ -12,7 +12,7 @@ import { cloneRepository, fetchRepository, getBranches, getCommitHashForRefName,
 import { captureEvent } from './posthog.js';
 import { PromClient } from './promClient.js';
 import { RepoWithConnections, Settings } from "./types.js";
-import { getAuthCredentialsForRepo, getRepoPath, getShardPrefix, groupmqLifecycleExceptionWrapper, measure, setIntervalAsync } from './utils.js';
+import { getAuthCredentialsForRepo, getShardPrefix, groupmqLifecycleExceptionWrapper, measure, setIntervalAsync } from './utils.js';
 import { indexGitRepository } from './zoekt.js';
 
 const LOG_TAG = 'repo-index-manager';

packages/backend/src/utils.ts

@@ -53,25 +53,6 @@ export const arraysEqualShallow = <T>(a?: readonly T[], b?: readonly T[]) => {
     return true;
 }
 
-// @note: this function is duplicated in `packages/web/src/features/fileTree/actions.ts`.
-// @todo: we should move this to a shared package.
-export const getRepoPath = (repo: Repo): { path: string, isReadOnly: boolean } => {
-    // If we are dealing with a local repository, then use that as the path.
-    // Mark as read-only since we aren't guaranteed to have write access to the local filesystem.
-    const cloneUrl = new URL(repo.cloneUrl);
-    if (repo.external_codeHostType === 'genericGitHost' && cloneUrl.protocol === 'file:') {
-        return {
-            path: cloneUrl.pathname,
-            isReadOnly: true,
-        }
-    }
-
-    return {
-        path: path.join(REPOS_CACHE_DIR, repo.id.toString()),
-        isReadOnly: false,
-    }
-}
-
 export const getShardPrefix = (orgId: number, repoId: number) => {
     return `${orgId}_${repoId}`;
 }

packages/backend/src/zoekt.ts

@@ -1,9 +1,9 @@
 import { Repo } from "@sourcebot/db";
-import { createLogger, env } from "@sourcebot/shared";
+import { createLogger, env, getRepoPath } from "@sourcebot/shared";
 import { exec } from "child_process";
 import { INDEX_CACHE_DIR } from "./constants.js";
 import { Settings } from "./types.js";
-import { getRepoPath, getShardPrefix } from "./utils.js";
+import { getShardPrefix } from "./utils.js";
 
 const logger = createLogger('zoekt');
 

packages/mcp/src/client.ts

@@ -20,14 +20,8 @@ export const search = async (request: SearchRequest): Promise<SearchResponse | S
     return searchResponseSchema.parse(result);
 }
 
-export const listRepos = async (params?: { activeAfter?: string, activeBefore?: string }): Promise<ListRepositoriesResponse | ServiceError> => {
+export const listRepos = async (): Promise<ListRepositoriesResponse | ServiceError> => {
     const url = new URL(`${env.SOURCEBOT_HOST}/api/repos`);
-    if (params?.activeAfter) {
-        url.searchParams.append('activeAfter', params.activeAfter);
-    }
-    if (params?.activeBefore) {
-        url.searchParams.append('activeBefore', params.activeBefore);
-    }
 
     const result = await fetch(url.toString(), {
         method: 'GET',

packages/mcp/src/index.ts

@@ -5,7 +5,7 @@ import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
 import escapeStringRegexp from 'escape-string-regexp';
 import { z } from 'zod';
-import { listRepos, search, getFileSource, searchCommits } from './client.js';
+import { getFileSource, listRepos, search, searchCommits } from './client.js';
 import { env, numberSchema } from './env.js';
 import { listReposRequestSchema } from './schemas.js';
 import { TextContent } from './types.js';
@@ -53,14 +53,6 @@ server.tool(
             .string()
             .describe(`The git revision to search in (e.g., 'main', 'HEAD', 'v1.0.0', 'a1b2c3d'). If not provided, defaults to the default branch (usually 'main' or 'master').`)
             .optional(),
-        since: z
-            .string()
-            .describe(`Filter repositories by when they were last indexed by Sourcebot (NOT by commit time). Only searches in repos indexed after this date. Supports ISO 8601 (e.g., '2024-01-01') or relative formats (e.g., '30 days ago', 'last week', 'yesterday').`)
-            .optional(),
-        until: z
-            .string()
-            .describe(`Filter repositories by when they were last indexed by Sourcebot (NOT by commit time). Only searches in repos indexed before this date. Supports ISO 8601 (e.g., '2024-12-31') or relative formats (e.g., 'yesterday').`)
-            .optional(),
         maxTokens: numberSchema
             .describe(`The maximum number of tokens to return (default: ${env.DEFAULT_MINIMUM_TOKENS}). Higher values provide more context but consume more tokens. Values less than ${env.DEFAULT_MINIMUM_TOKENS} will be ignored.`)
             .transform((val) => (val < env.DEFAULT_MINIMUM_TOKENS ? env.DEFAULT_MINIMUM_TOKENS : val))
@@ -74,8 +66,6 @@ server.tool(
         includeCodeSnippets = false,
         caseSensitive = false,
         gitRevision,
-        since,
-        until,
     }) => {
         if (repoIds.length > 0) {
             query += ` ( repo:${repoIds.map(id => escapeStringRegexp(id)).join(' or repo:')} )`;
@@ -85,16 +75,17 @@ server.tool(
             query += ` ( lang:${languages.join(' or lang:')} )`;
         }
 
+        if (gitRevision) {
+            query += ` ( rev:${gitRevision} )`;
+        }
+
         const response = await search({
             query,
             matches: env.DEFAULT_MATCHES,
             contextLines: env.DEFAULT_CONTEXT_LINES,
             isRegexEnabled: true,
             isCaseSensitivityEnabled: caseSensitive,
             source: 'mcp',
-            gitRevision,
-            since,
-            until,
         });
 
         if (isServiceError(response)) {
@@ -182,21 +173,9 @@ server.tool(
 
 server.tool(
     "search_commits",
-    `Searches for commits in a specific repository based on actual commit time (NOT index time).
-
-    **Requirements**: The repository must be cloned on the Sourcebot server disk. Sourcebot automatically clones repositories during indexing, but the cloning process may not be finished when this query is executed. If the repository is not found on the server disk, an error will be returned asking you to try again later.
-
-    **Date Formats**: Supports ISO 8601 (e.g., "2024-01-01") or relative formats (e.g., "30 days ago", "last week", "yesterday").
-
-    **YOU MUST** call 'list_repos' first to obtain the exact repository ID.
-
-    If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.`,
+    `Searches for commits in a specific repository based on actual commit time. If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.`,
     {
-        repoId: z.union([z.number(), z.string()]).describe(`Repository identifier. Can be either:
-        - Numeric database ID (e.g., 123)
-        - Full repository name (e.g., "github.com/owner/repo") as returned by 'list_repos'
-
-        **YOU MUST** call 'list_repos' first to obtain the repository identifier.`),
+        repoId: z.string().describe(`The repository to search commits in. This is the Sourcebot compatible repository ID as returned by 'list_repos'.`),
         query: z.string().describe(`Search query to filter commits by message content (case-insensitive).`).optional(),
         since: z.string().describe(`Show commits more recent than this date. Filters by actual commit time. Supports ISO 8601 (e.g., '2024-01-01') or relative formats (e.g., '30 days ago', 'last week').`).optional(),
         until: z.string().describe(`Show commits older than this date. Filters by actual commit time. Supports ISO 8601 (e.g., '2024-12-31') or relative formats (e.g., 'yesterday').`).optional(),
@@ -205,7 +184,7 @@ server.tool(
     },
     async ({ repoId, query, since, until, author, maxCount }) => {
         const result = await searchCommits({
-            repoId,
+            repository: repoId,
             query,
             since,
             until,
@@ -228,47 +207,14 @@ server.tool(
 
 server.tool(
     "list_repos",
-    `Lists repositories in the organization with optional filtering and pagination.
-
-    **Temporal Filtering**: When using 'activeAfter' or 'activeBefore', only repositories indexed within the specified timeframe are returned. This filters by when Sourcebot last indexed the repository (indexedAt), NOT by git commit dates. For commit-time filtering, use 'search_commits'. When temporal filters are applied, the output includes a 'lastIndexed' field showing when each repository was last indexed.
-
-    **Date Formats**: Supports ISO 8601 (e.g., "2024-01-01") and relative dates (e.g., "30 days ago", "last week", "yesterday").
-
-    If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.`,
-    {
-        query: z
-            .string()
-            .describe("Filter repositories by name (case-insensitive).")
-            .optional(),
-        pageNumber: z
-            .number()
-            .int()
-            .positive()
-            .describe("Page number (1-indexed, default: 1)")
-            .default(1),
-        limit: z
-            .number()
-            .int()
-            .positive()
-            .describe("Number of repositories per page (default: 50)")
-            .default(50),
-        activeAfter: z
-            .string()
-            .describe("Only return repositories indexed after this date (filters by indexedAt). Supports ISO 8601 (e.g., '2024-01-01') or relative formats (e.g., '30 days ago', 'last week').")
-            .optional(),
-        activeBefore: z
-            .string()
-            .describe("Only return repositories indexed before this date (filters by indexedAt). Supports ISO 8601 (e.g., '2024-12-31') or relative formats (e.g., 'yesterday').")
-            .optional(),
-    },
-    async ({ query, pageNumber = 1, limit = 50, activeAfter, activeBefore }: {
+    `Lists repositories in the organization with optional filtering and pagination. If you receive an error that indicates that you're not authenticated, please inform the user to set the SOURCEBOT_API_KEY environment variable.`,
+    listReposRequestSchema.shape,
+    async ({ query, pageNumber = 1, limit = 50 }: {
         query?: string;
         pageNumber?: number;
         limit?: number;
-        activeAfter?: string;
-        activeBefore?: string;
     }) => {
-        const response = await listRepos({ activeAfter, activeBefore });
+        const response = await listRepos();
         if (isServiceError(response)) {
             return {
                 content: [{
@@ -298,17 +244,10 @@ server.tool(
 
         // Format output
         const content: TextContent[] = paginated.map(repo => {
-          const repoUrl = repo.webUrl ?? repo.repoCloneUrl;
-            let output = `id: ${repo.repoName}\nurl: ${repoUrl}`;
-
-            // Include indexedAt when temporal filtering is used
-            if ((activeAfter || activeBefore) && repo.indexedAt) {
-                output += `\nlastIndexed: ${repo.indexedAt.toISOString()}`;
-            }
-
+            const repoUrl = repo.webUrl ?? repo.repoCloneUrl;
             return {
                 type: "text",
-                text: output,
+                text: `id: ${repo.repoName}\nurl: ${repoUrl}`,
             }
         });
 

packages/mcp/src/schemas.ts

@@ -27,9 +27,6 @@ export const searchOptionsSchema = z.object({
     whole: z.boolean().optional(),                    // Whether to return the whole file as part of the response.
     isRegexEnabled: z.boolean().optional(),           // Whether to enable regular expression search.
     isCaseSensitivityEnabled: z.boolean().optional(), // Whether to enable case sensitivity.
-    gitRevision: z.string().optional(),               // Filter by git branch/revision.
-    since: z.string().optional(),                     // Filter repositories by indexed date (start). Filters by when the repo was last indexed by Sourcebot, not by commit time.
-    until: z.string().optional(),                     // Filter repositories by indexed date (end). Filters by when the repo was last indexed by Sourcebot, not by commit time.
 });
 
 export const searchRequestSchema = z.object({
@@ -146,7 +143,6 @@ export const searchResponseSchema = z.object({
     })),
     repositoryInfo: z.array(repositoryInfoSchema),
     isSearchExhaustive: z.boolean(),
-    isBranchFilteringEnabled: z.boolean().optional(), // Whether branch filtering is enabled for this search.
 });
 
 export const repositoryQuerySchema = z.object({
@@ -199,7 +195,7 @@ export const serviceErrorSchema = z.object({
 });
 
 export const searchCommitsRequestSchema = z.object({
-    repoId: z.union([z.number(), z.string()]),
+    repository: z.string(),
     query: z.string().optional(),
     since: z.string().optional(),
     until: z.string().optional(),

packages/shared/src/constants.server.ts

@@ -1,9 +1,12 @@
+import { indexSchema } from "@sourcebot/schemas/v3/index.schema";
+import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
 import { createEnv } from "@t3-oss/env-core";
+import { Ajv } from "ajv";
+import { readFile } from 'fs/promises';
+import stripJsonComments from "strip-json-comments";
 import { z } from "zod";
-import { loadConfig } from "./utils.js";
-import { tenancyModeSchema } from "./types.js";
-import { SourcebotConfig } from "@sourcebot/schemas/v3/index.type";
 import { getTokenFromConfig } from "./crypto.js";
+import { tenancyModeSchema } from "./types.js";
 
 // Booleans are specified as 'true' or 'false' strings.
 const booleanSchema = z.enum(["true", "false"]);
@@ -13,6 +16,10 @@ const booleanSchema = z.enum(["true", "false"]);
 // @see: https://zod.dev/?id=coercion-for-primitives
 const numberSchema = z.coerce.number();
 
+const ajv = new Ajv({
+    validateFormats: false,
+});
+
 export const resolveEnvironmentVariableOverridesFromConfig = async (config: SourcebotConfig): Promise<Record<string, string>> => {
     if (!config.environmentOverrides) {
         return {};
@@ -45,6 +52,66 @@ export const resolveEnvironmentVariableOverridesFromConfig = async (config: Sour
     return resolved;
 }
 
+export const isRemotePath = (path: string) => {
+    return path.startsWith('https://') || path.startsWith('http://');
+}
+
+export const loadConfig = async (configPath?: string): Promise<SourcebotConfig> => {
+    if (!configPath) {
+        throw new Error('CONFIG_PATH is required but not provided');
+    }
+
+    const configContent = await (async () => {
+        if (isRemotePath(configPath)) {
+            const response = await fetch(configPath);
+            if (!response.ok) {
+                throw new Error(`Failed to fetch config file ${configPath}: ${response.statusText}`);
+            }
+            return response.text();
+        } else {
+            // Retry logic for handling race conditions with mounted volumes
+            const maxAttempts = 5;
+            const retryDelayMs = 2000;
+            let lastError: Error | null = null;
+
+            for (let attempt = 1; attempt <= maxAttempts; attempt++) {
+                try {
+                    return await readFile(configPath, {
+                        encoding: 'utf-8',
+                    });
+                } catch (error) {
+                    lastError = error as Error;
+                    
+                    // Only retry on ENOENT errors (file not found)
+                    if ((error as NodeJS.ErrnoException)?.code !== 'ENOENT') {
+                        throw error; // Throw immediately for non-ENOENT errors
+                    }
+                    
+                    // Log warning before retry (except on the last attempt)
+                    if (attempt < maxAttempts) {
+                        console.warn(`Config file not found, retrying in 2s... (Attempt ${attempt}/${maxAttempts})`);
+                        await new Promise(resolve => setTimeout(resolve, retryDelayMs));
+                    }
+                }
+            }
+            
+            // If we've exhausted all retries, throw the last ENOENT error
+            if (lastError) {
+                throw lastError;
+            }
+            
+            throw new Error('Failed to load config after all retry attempts');
+        }
+    })();
+
+    const config = JSON.parse(stripJsonComments(configContent)) as SourcebotConfig;
+    const isValidConfig = ajv.validate(indexSchema, config);
+    if (!isValidConfig) {
+        throw new Error(`Config file '${configPath}' is invalid: ${ajv.errorsText(ajv.errors)}`);
+    }
+    return config;
+}
+
 // Merge process.env with environment variables resolved from config.json
 const runtimeEnv = await (async () => {
     const configPath = process.env.CONFIG_PATH;

packages/shared/src/env.server.ts

@@ -20,19 +20,16 @@ export {
 } from "./types.js";
 export {
     base64Decode,
-    loadConfig,
     loadJsonFile,
-    isRemotePath,
     getConfigSettings,
+    getRepoPath,
 } from "./utils.js";
 export * from "./constants.js";
-export {
-    REPOS_CACHE_DIR,
-    INDEX_CACHE_DIR,
-} from "./constants.server.js";
 export {
     env,
     resolveEnvironmentVariableOverridesFromConfig,
+    loadConfig,
+    isRemotePath,
 } from "./env.server.js";
 export {
     createLogger,