wip: use external url for search as well

Author: brendan-kellam

packages/mcp/src/index.ts

@@ -3,14 +3,13 @@
 // Entry point for the MCP server
 import { McpServer } from '@modelcontextprotocol/sdk/server/mcp.js';
 import { StdioServerTransport } from '@modelcontextprotocol/sdk/server/stdio.js';
+import _dedent from "dedent";
 import escapeStringRegexp from 'escape-string-regexp';
 import { z } from 'zod';
-import { getFileSource, listRepos, search, listCommits } from './client.js';
+import { getFileSource, listCommits, listRepos, search } from './client.js';
 import { env, numberSchema } from './env.js';
 import { fileSourceRequestSchema, listCommitsQueryParamsSchema, listReposQueryParamsSchema } from './schemas.js';
 import { FileSourceRequest, ListCommitsQueryParamsSchema, ListReposQueryParams, TextContent } from './types.js';
-import _dedent from "dedent";
-import { addLineNumbers } from './utils.js';
 
 const dedent = _dedent.withOptions({ alignValues: true });
 
@@ -24,39 +23,41 @@ const server = new McpServer({
 server.tool(
     "search_code",
     dedent`
-    Fetches code that matches the provided regex pattern in \`query\`.
-
-    Results are returned as an array of matching files, with the file's URL, repository, and language.
-
-    If the \`includeCodeSnippets\` property is true, code snippets containing the matches will be included in the response. Only set this to true if the request requires code snippets (e.g., show me examples where library X is used).
-    When referencing a file in your response, **ALWAYS** include the file's external URL as a link. This makes it easier for the user to view the file, even if they don't have it locally checked out.
-    **ONLY USE** the \`filterByRepoIds\` property if the request requires searching a specific repo(s). Otherwise, leave it empty.`,
+    Searches for code that matches the provided search query as a substring by default, or as a regular expression if useRegex is true. Useful for exploring remote repositories by searching for exact symbols, functions, variables, or specific code patterns. To determine if a repository is indexed, use the \`list_repos\` tool. By default, searches are global and will search the default branch of all repositories. Searches can be scoped to specific repositories, languages, and branches. When referencing code outputted by this tool, always include the file's external URL as a link. This makes it easier for the user to view the file, even if they don't have it locally checked out.
+    `,
     {
         query: z
             .string()
-            .describe(`The regex pattern to search for. RULES:
-        1. When a regex special character needs to be escaped, ALWAYS use a single backslash (\) (e.g., 'console\.log')
-        2. **ALWAYS** escape spaces with a single backslash (\) (e.g., 'console\ log')
-        `),
-        filterByRepoIds: z
+            .describe(`The search pattern to match against code contents. Do not escape quotes in your query.`)
+            // Wrap in quotes so the query is treated as a literal phrase (like grep).
+            .transform((val) => `"${val.replace(/"/g, '\\"')}"`),
+        useRegex: z
+            .boolean()
+            .describe(`Whether to use regular expression matching to match the search query against code contents. When false, substring matching is used. (default: false)`)
+            .optional(),
+        filterByRepos: z
             .array(z.string())
-            .describe(`Scope the search to the provided repositories to the Sourcebot compatible repository IDs. **DO NOT** use this property if you want to search all repositories. **YOU MUST** call 'list_repos' first to obtain the exact repository ID.`)
+            .describe(`Scope the search to the provided repositories.`)
             .optional(),
         filterByLanguages: z
             .array(z.string())
-            .describe(`Scope the search to the provided languages. The language MUST be formatted as a GitHub linguist language. Examples: Python, JavaScript, TypeScript, Java, C#, C++, PHP, Go, Rust, Ruby, Swift, Kotlin, Shell, C, Dart, HTML, CSS, PowerShell, SQL, R`)
+            .describe(`Scope the search to the provided languages.`)
+            .optional(),
+        filterByFilepaths: z
+            .array(z.string())
+            .describe(`Scope the search to the provided filepaths. Interpretted as a regex.`)
             .optional(),
         caseSensitive: z
             .boolean()
             .describe(`Whether the search should be case sensitive (default: false).`)
             .optional(),
         includeCodeSnippets: z
             .boolean()
-            .describe(`Whether to include the code snippets in the response (default: false). If false, only the file's URL, repository, and language will be returned. Set to false to get a more concise response.`)
+            .describe(`Whether to include the code snippets in the response. If false, only the file's URL, repository, and language will be returned. (default: false)`)
             .optional(),
-        gitRevision: z
+        ref: z
             .string()
-            .describe(`The git revision to search in (e.g., 'main', 'HEAD', 'v1.0.0', 'a1b2c3d'). If not provided, defaults to the default branch (usually 'main' or 'master').`)
+            .describe(`Commit SHA, branch or tag name to search on. If not provided, defaults to the default branch (usually 'main' or 'master').`)
             .optional(),
         maxTokens: numberSchema
             .describe(`The maximum number of tokens to return (default: ${env.DEFAULT_MINIMUM_TOKENS}). Higher values provide more context but consume more tokens. Values less than ${env.DEFAULT_MINIMUM_TOKENS} will be ignored.`)
@@ -65,30 +66,36 @@ server.tool(
     },
     async ({
         query,
-        filterByRepoIds: repoIds = [],
+        filterByRepos: repos = [],
         filterByLanguages: languages = [],
+        filterByFilepaths: filepaths = [],
         maxTokens = env.DEFAULT_MINIMUM_TOKENS,
         includeCodeSnippets = false,
         caseSensitive = false,
-        gitRevision,
+        ref,
+        useRegex = false,
     }) => {
-        if (repoIds.length > 0) {
-            query += ` ( repo:${repoIds.map(id => escapeStringRegexp(id)).join(' or repo:')} )`;
+        if (repos.length > 0) {
+            query += ` (repo:${repos.map(id => escapeStringRegexp(id)).join(' or repo:')})`;
         }
 
         if (languages.length > 0) {
-            query += ` ( lang:${languages.join(' or lang:')} )`;
+            query += ` (lang:${languages.join(' or lang:')})`;
+        }
+
+        if (filepaths.length > 0) {
+            query += ` (file:${filepaths.map(filepath => escapeStringRegexp(filepath)).join(' or file:')})`;
         }
 
-        if (gitRevision) {
-            query += ` ( rev:${gitRevision} )`;
+        if (ref) {
+            query += ` ( rev:${ref} )`;
         }
 
         const response = await search({
             query,
             matches: env.DEFAULT_MATCHES,
             contextLines: env.DEFAULT_CONTEXT_LINES,
-            isRegexEnabled: true,
+            isRegexEnabled: useRegex,
             isCaseSensitivityEnabled: caseSensitive,
             source: 'mcp',
         });
@@ -111,11 +118,10 @@ server.tool(
                 (acc, chunk) => acc + chunk.matchRanges.length,
                 0,
             );
-            const fileIdentifier = file.webUrl ?? file.fileName.text;
             let text = dedent`
-            file: ${fileIdentifier}
+            file: ${file.webUrl}
             num_matches: ${numMatches}
-            repository: ${file.repository}
+            repo: ${file.repository}
             language: ${file.language}
             `;
 
@@ -219,7 +225,7 @@ server.tool(
         return {
             content: [{
                 type: "text", text: JSON.stringify({
-                    source: addLineNumbers(response.source),
+                    source: response.source,
                     language: response.language,
                     path: response.path,
                     url: response.webUrl,

packages/mcp/src/schemas.ts

@@ -124,7 +124,8 @@ export const searchResponseSchema = z.object({
             // Any matching ranges
             matchRanges: z.array(rangeSchema),
         }),
-        webUrl: z.string().optional(),
+        webUrl: z.string(),
+        externalWebUrl: z.string().optional(),
         repository: z.string(),
         repositoryId: z.number(),
         language: z.string(),

packages/mcp/src/utils.ts

@@ -14,7 +14,3 @@ export class ServiceErrorException extends Error {
         super(JSON.stringify(serviceError));
     }
 }
-
-export const addLineNumbers = (source: string, lineOffset = 1) => {
-    return source.split('\n').map((line, index) => `${index + lineOffset}:${line}`).join('\n');
-}

packages/web/src/app/[domain]/search/components/codePreviewPanel/index.tsx

@@ -42,7 +42,7 @@ export const CodePreviewPanel = ({
                 content: data.source,
                 filepath: previewedFile.fileName.text,
                 matches: previewedFile.chunks,
-                link: previewedFile.webUrl,
+                link: previewedFile.externalWebUrl,
                 language: previewedFile.language,
                 revision: branch ?? "HEAD",
             };

packages/web/src/features/codeNav/types.ts

@@ -25,7 +25,7 @@ export const findRelatedSymbolsResponseSchema = z.object({
         fileName: z.string(),
         repository: z.string(),
         repositoryId: z.number(),
-        webUrl: z.string().optional(),
+        webUrl: z.string(),
         language: z.string(),
         matches: z.array(z.object({
             lineContent: z.string(),

packages/web/src/features/search/fileSourceApi.ts

@@ -7,20 +7,13 @@ import { sew } from "@/actions";
 import { withOptionalAuthV2 } from "@/withAuthV2";
 import { QueryIR } from './ir';
 import escapeStringRegexp from "escape-string-regexp";
-import { SINGLE_TENANT_ORG_DOMAIN } from '@/lib/constants';
-import { getBrowsePath } from '@/app/[domain]/browse/hooks/utils';
-import { headers } from 'next/headers';
-import { getBaseUrl } from '@/lib/utils.server';
 
 // @todo (bkellam) #574 : We should really be using `git show <hash>:<path>` to fetch file contents here.
 // This will allow us to support permalinks to files at a specific revision that may not be indexed
 // by zoekt. We should also refactor this out of the /search folder.
 
 export const getFileSource = async ({ fileName, repository, branch }: FileSourceRequest): Promise<FileSourceResponse | ServiceError> => sew(() =>
     withOptionalAuthV2(async () => {
-        const headersList = await headers();
-        const baseUrl = getBaseUrl(headersList);
-
         const query: QueryIR = {
             and: {
                 children: [
@@ -85,14 +78,8 @@ export const getFileSource = async ({ fileName, repository, branch }: FileSource
             repoDisplayName: repoInfo.displayName,
             repoExternalWebUrl: repoInfo.webUrl,
             branch,
-            webUrl: `${baseUrl}${getBrowsePath({
-                repoName: repository,
-                revisionName: branch,
-                path: fileName,
-                pathType: 'blob',
-                domain: SINGLE_TENANT_ORG_DOMAIN,
-            })}`,
-            externalWebUrl: file.webUrl,
+            webUrl: file.webUrl,
+            externalWebUrl: file.externalWebUrl,
         } satisfies FileSourceResponse;
 
     }));

packages/web/src/features/search/types.ts

@@ -63,7 +63,8 @@ export const searchFileSchema = z.object({
         // Any matching ranges
         matchRanges: z.array(rangeSchema),
     }),
-    webUrl: z.string().optional(),
+    webUrl: z.string(),
+    externalWebUrl: z.string().optional(),
     repository: z.string(),
     repositoryId: z.number(),
     language: z.string(),

packages/web/src/features/search/zoektSearcher.ts

@@ -1,5 +1,5 @@
 import { getCodeHostBrowseFileAtBranchUrl } from "@/lib/utils";
-import { unexpectedError } from "@/lib/serviceError";
+import { ServiceErrorException, unexpectedError } from "@/lib/serviceError";
 import type { ProtoGrpcType } from '@/proto/webserver';
 import { FileMatch__Output as ZoektGrpcFileMatch } from "@/proto/zoekt/webserver/v1/FileMatch";
 import { FlushReason as ZoektGrpcFlushReason } from "@/proto/zoekt/webserver/v1/FlushReason";
@@ -18,6 +18,10 @@ import path from 'path';
 import { isBranchQuery, QueryIR, someInQueryIR } from './ir';
 import { RepositoryInfo, SearchResponse, SearchResultFile, SearchStats, SourceRange, StreamedSearchErrorResponse, StreamedSearchResponse } from "./types";
 import { captureEvent } from "@/lib/posthog";
+import { getBrowsePath } from "@/app/[domain]/browse/hooks/utils";
+import { SINGLE_TENANT_ORG_DOMAIN } from "@/lib/constants";
+import { headers } from "next/headers";
+import { getBaseUrl } from "@/lib/utils.server";
 
 const logger = createLogger("zoekt-searcher");
 
@@ -123,7 +127,7 @@ export const zoektSearch = async (searchRequest: ZoektGrpcSearchRequest, prisma:
     return new Promise((resolve, reject) => {
         client.Search(searchRequest, metadata, (error, response) => {
             if (error || !response) {
-                reject(error || new Error('No response received'));
+                reject(new ServiceErrorException(unexpectedError(error?.details || 'No response received')))
                 return;
             }
 
@@ -379,6 +383,9 @@ const transformZoektSearchResponse = async (response: ZoektGrpcSearchResponse, r
     files: SearchResultFile[],
     repositoryInfo: RepositoryInfo[],
 }> => {
+    const headersList = await headers();
+    const baseUrl = getBaseUrl(headersList);
+
     const files = response.files.map((file) => {
         const fileNameChunks = file.chunk_matches.filter((chunk) => chunk.file_name);
         const repoId = getRepoIdForFile(file);
@@ -413,6 +420,9 @@ const transformZoektSearchResponse = async (response: ZoektGrpcSearchResponse, r
             }
         })
 
+        // If a file has multiple branches, default to the first one.
+        const branchName = file.branches.length > 0 ? file.branches[0] : undefined;
+
         return {
             fileName: {
                 text: fileName,
@@ -421,11 +431,17 @@ const transformZoektSearchResponse = async (response: ZoektGrpcSearchResponse, r
             repository: repo.name,
             repositoryId: repo.id,
             language: file.language,
-            webUrl: getCodeHostBrowseFileAtBranchUrl({
+            webUrl: `${baseUrl}${getBrowsePath({
+                repoName: repo.name,
+                path: fileName,
+                pathType: 'blob',
+                domain: SINGLE_TENANT_ORG_DOMAIN,
+                revisionName: branchName,
+            })}`,
+            externalWebUrl: getCodeHostBrowseFileAtBranchUrl({
                 webUrl: repo.webUrl,
                 codeHostType: repo.external_codeHostType,
-                // If a file has multiple branches, default to the first one.
-                branchName: file.branches?.[0] ?? 'HEAD',
+                branchName: branchName ?? 'HEAD',
                 filePath: fileName,
             }),
             chunks: file.chunk_matches