Skip to content

Commit 4293cbd

Browse files
brendan-kellambrendan-kellam
committed
simplification: remove file truncation
1 parent 4fbb459 commit 4293cbd

File tree

10 files changed

+41
-268
lines changed

10 files changed

+41
-268
lines changed

‎packages/web/src/env.mjs‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -107,8 +107,6 @@ export const env = createEnv({
107107
AWS_REGION: z.string().optional(),
108108

109109
SOURCEBOT_CHAT_MODEL_TEMPERATURE: numberSchema.default(0.3),
110-
SOURCEBOT_CHAT_FILE_MAX_CHARACTERS: numberSchema.default(4000),
111-
112110
SOURCEBOT_CHAT_MAX_STEP_COUNT: numberSchema.default(20),
113111

114112
DEBUG_WRITE_CHAT_MESSAGES_TO_FILE: booleanSchema.default('false'),

‎packages/web/src/features/chat/agent.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@ import { LanguageModel, ModelMessage, StopCondition, streamText } from "ai";
88
import { ANSWER_TAG, FILE_REFERENCE_PREFIX, toolNames } from "./constants";
99
import { createCodeSearchTool, findSymbolDefinitionsTool, findSymbolReferencesTool, readFilesTool } from "./tools";
1010
import { FileSource, Source } from "./types";
11-
import { fileReferenceToString, sourceCodeToModelOutput } from "./utils";
11+
import { addLineNumbers, fileReferenceToString } from "./utils";
1212

1313
const logger = createLogger('chat-agent');
1414

@@ -213,7 +213,7 @@ const createFileSourcesSystemPrompt = async ({ files }: FileSourcesSystemPromptO
213213
The user has mentioned the following files, which are automatically included for analysis.
214214
215215
${files.map(file => `<file path="${file.path}" repository="${file.repo}" language="${file.language}" revision="${file.revision}">
216-
${sourceCodeToModelOutput(file.source).output}
216+
${addLineNumbers(file.source)}
217217
</file>`).join('\n\n')}
218218
`.trim();
219219
}

‎packages/web/src/features/chat/components/chatThread/tools/findSymbolDefinitionsToolComponent.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ export const FindSymbolDefinitionsToolComponent = ({ part }: { part: FindSymbolD
5353
key={file.fileName}
5454
path={file.fileName}
5555
repoName={file.repository}
56-
isTruncated={file.isOutputTruncated}
5756
/>
5857
)
5958
})}

‎packages/web/src/features/chat/components/chatThread/tools/findSymbolReferencesToolComponent.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,6 @@ export const FindSymbolReferencesToolComponent = ({ part }: { part: FindSymbolRe
5353
key={file.fileName}
5454
path={file.fileName}
5555
repoName={file.repository}
56-
isTruncated={file.isOutputTruncated}
5756
/>
5857
)
5958
})}

‎packages/web/src/features/chat/components/chatThread/tools/readFilesToolComponent.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,6 @@ export const ReadFilesToolComponent = ({ part }: { part: ReadFilesToolUIPart })
4848
key={file.path}
4949
path={file.path}
5050
repoName={file.repository}
51-
isTruncated={file.isOutputTruncated}
5251
/>
5352
)
5453
})}

‎packages/web/src/features/chat/components/chatThread/tools/searchCodeToolComponent.tsx‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -58,7 +58,6 @@ export const SearchCodeToolComponent = ({ part }: { part: SearchCodeToolUIPart }
5858
key={file.fileName}
5959
path={file.fileName}
6060
repoName={file.repository}
61-
isTruncated={file.isOutputTruncated}
6261
/>
6362
)
6463
})}

‎packages/web/src/features/chat/components/chatThread/tools/shared.tsx‎

Lines changed: 0 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -13,11 +13,9 @@ import { getBrowsePath } from '@/app/[domain]/browse/hooks/useBrowseNavigation';
1313
export const FileListItem = ({
1414
path,
1515
repoName,
16-
isTruncated,
1716
}: {
1817
path: string,
1918
repoName: string,
20-
isTruncated?: boolean,
2119
}) => {
2220
const domain = useDomain();
2321

@@ -36,11 +34,6 @@ export const FileListItem = ({
3634
>
3735
{path}
3836
</Link>
39-
{isTruncated && (
40-
<span className="text-xs text-muted-foreground ml-2">
41-
(truncated)
42-
</span>
43-
)}
4437
</div>
4538
)
4639
}

‎packages/web/src/features/chat/tools.ts‎

Lines changed: 36 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import { isServiceError } from "@/lib/utils";
66
import { getFileSource } from "../search/fileSourceApi";
77
import { findSearchBasedSymbolDefinitions, findSearchBasedSymbolReferences } from "../codeNav/actions";
88
import { FileSourceResponse } from "../search/types";
9-
import { sourceCodeChunksToModelOutput, sourceCodeToModelOutput } from "./utils";
9+
import { addLineNumbers } from "./utils";
1010
import { toolNames } from "./constants";
1111

1212
export const findSymbolReferencesTool = tool({
@@ -28,21 +28,15 @@ export const findSymbolReferencesTool = tool({
2828
return response;
2929
}
3030

31-
return response.files.map((file) => {
32-
const matches = sourceCodeChunksToModelOutput(file.matches.map(({ lineContent, range }) => ({
33-
source: lineContent,
34-
startLine: range.start.lineNumber,
35-
})))
36-
37-
return {
38-
fileName: file.fileName,
39-
repository: file.repository,
40-
language: file.language,
41-
matches,
42-
revision,
43-
isOutputTruncated: matches.some(({ isTruncated }) => isTruncated),
44-
}
45-
});
31+
return response.files.map((file) => ({
32+
fileName: file.fileName,
33+
repository: file.repository,
34+
language: file.language,
35+
matches: file.matches.map(({ lineContent, range }) => {
36+
return addLineNumbers(lineContent, range.start.lineNumber);
37+
}),
38+
revision,
39+
}));
4640
},
4741
});
4842

@@ -70,21 +64,15 @@ export const findSymbolDefinitionsTool = tool({
7064
return response;
7165
}
7266

73-
return response.files.map((file) => {
74-
const matches = sourceCodeChunksToModelOutput(file.matches.map(({ lineContent, range }) => ({
75-
source: lineContent,
76-
startLine: range.start.lineNumber,
77-
})))
78-
79-
return {
80-
fileName: file.fileName,
81-
repository: file.repository,
82-
language: file.language,
83-
matches,
84-
revision,
85-
isOutputTruncated: matches.some(({ isTruncated }) => isTruncated),
86-
}
87-
});
67+
return response.files.map((file) => ({
68+
fileName: file.fileName,
69+
repository: file.repository,
70+
language: file.language,
71+
matches: file.matches.map(({ lineContent, range }) => {
72+
return addLineNumbers(lineContent, range.start.lineNumber);
73+
}),
74+
revision,
75+
}));
8876
}
8977
});
9078

@@ -115,18 +103,13 @@ export const readFilesTool = tool({
115103
return firstError!;
116104
}
117105

118-
return (responses as FileSourceResponse[]).map((response) => {
119-
const { output: source, isTruncated } = sourceCodeToModelOutput(response.source);
120-
121-
return {
122-
path: response.path,
123-
repository: response.repository,
124-
language: response.language,
125-
source,
126-
revision,
127-
isOutputTruncated: isTruncated,
128-
}
129-
});
106+
return (responses as FileSourceResponse[]).map((response) => ({
107+
path: response.path,
108+
repository: response.repository,
109+
language: response.language,
110+
source: addLineNumbers(response.source),
111+
revision,
112+
}));
130113
}
131114
});
132115

@@ -161,22 +144,16 @@ export const createCodeSearchTool = (selectedRepos: string[]) => tool({
161144
}
162145

163146
return {
164-
files: response.files.map((file) => {
165-
const matches = sourceCodeChunksToModelOutput(file.chunks.map(({ content, contentStart }) => ({
166-
source: content,
167-
startLine: contentStart.lineNumber,
168-
})))
169-
170-
return {
171-
fileName: file.fileName.text,
172-
repository: file.repository,
173-
language: file.language,
174-
matches,
175-
// @todo: make revision configurable.
176-
revision: 'HEAD',
177-
isOutputTruncated: matches.some(({ isTruncated }) => isTruncated),
178-
}
179-
}),
147+
files: response.files.map((file) => ({
148+
fileName: file.fileName.text,
149+
repository: file.repository,
150+
language: file.language,
151+
matches: file.chunks.map(({ content, contentStart }) => {
152+
return addLineNumbers(content, contentStart.lineNumber);
153+
}),
154+
// @todo: make revision configurable.
155+
revision: 'HEAD',
156+
})),
180157
query,
181158
}
182159
},

‎packages/web/src/features/chat/utils.test.ts‎

Lines changed: 1 addition & 146 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
import { expect, test, vi } from 'vitest'
2-
import { fileReferenceToString, getAnswerPartFromAssistantMessage, groupMessageIntoSteps, sourceCodeChunksToModelOutput, sourceCodeToModelOutput } from './utils'
2+
import { fileReferenceToString, getAnswerPartFromAssistantMessage, groupMessageIntoSteps } from './utils'
33
import { FILE_REFERENCE_REGEX, ANSWER_TAG } from './constants';
44
import { SBChatMessage, SBChatMessagePart } from './types';
55

@@ -40,151 +40,6 @@ test('fileReferenceToString matches FILE_REFERENCE_REGEX', () => {
4040
}))).toBe(true);
4141
});
4242

43-
test('sourceCodeToModelOutput adds line numbers correctly', () => {
44-
const source = 'function hello() {\n return "world";\n}';
45-
const result = sourceCodeToModelOutput(source);
46-
47-
expect(result.output).toBe('1:function hello() {\n2: return "world";\n3:}');
48-
expect(result.isTruncated).toBe(false);
49-
});
50-
51-
test('sourceCodeToModelOutput respects lineOffset parameter', () => {
52-
const source = 'const x = 1;\nconst y = 2;';
53-
const result = sourceCodeToModelOutput(source, { lineOffset: 10 });
54-
55-
expect(result.output).toBe('10:const x = 1;\n11:const y = 2;');
56-
expect(result.isTruncated).toBe(false);
57-
});
58-
59-
test('sourceCodeToModelOutput handles empty string', () => {
60-
const source = '';
61-
const result = sourceCodeToModelOutput(source);
62-
63-
expect(result.output).toBe('1:');
64-
expect(result.isTruncated).toBe(false);
65-
});
66-
67-
test('sourceCodeToModelOutput handles single line', () => {
68-
const source = 'console.log("hello");';
69-
const result = sourceCodeToModelOutput(source);
70-
71-
expect(result.output).toBe('1:console.log("hello");');
72-
expect(result.isTruncated).toBe(false);
73-
});
74-
75-
test('sourceCodeToModelOutput truncates when exceeding max characters', () => {
76-
const source = 'a'.repeat(4000);
77-
const { output, isTruncated } = sourceCodeToModelOutput(source);
78-
79-
expect(isTruncated).toBe(true);
80-
expect(output).toEqual(`1:${'a'.repeat(3998)}`); // -2 for "1:" prefix
81-
})
82-
83-
test('sourceCodeToModelOutput truncates when exceeding max character and there is an accumulator', () => {
84-
const accum = 1000;
85-
const source = 'a'.repeat(3000);
86-
const { output, isTruncated } = sourceCodeToModelOutput(source, { charAccum: accum });
87-
88-
// output would normally be 3000 + 2 = 3002 characters.
89-
// 3002 + 1000 = 4002 characters > 4000 limit.
90-
// so truncate to 4000 - 1000 = 3000 characters.
91-
// => 2998 'a's
92-
93-
expect(isTruncated).toBe(true);
94-
expect(output).toEqual(`1:${'a'.repeat(2998)}`); // -2 for "1:" prefix
95-
expect(accum + output.length).toBe(4000);
96-
});
97-
98-
test('sourceCodeChunksToModelOutput truncates chunks when exceeding max characters', () => {
99-
const chunks = [
100-
{
101-
source: 'a'.repeat(998),
102-
startLine: 1
103-
},
104-
{
105-
source: 'b'.repeat(998),
106-
startLine: 2
107-
},
108-
{
109-
source: 'c'.repeat(998),
110-
startLine: 3
111-
},
112-
{
113-
source: 'd'.repeat(998),
114-
startLine: 4
115-
},
116-
// At this point, the accumulator is 4000 characters.
117-
// So all of the next chunk will be truncated.
118-
{
119-
source: 'e'.repeat(998),
120-
startLine: 5
121-
}
122-
]
123-
124-
const output = sourceCodeChunksToModelOutput(chunks);
125-
126-
expect(output[0].output).toEqual(`1:${'a'.repeat(998)}`);
127-
expect(output[0].isTruncated).toBe(false);
128-
129-
expect(output[1].output).toEqual(`2:${'b'.repeat(998)}`);
130-
expect(output[1].isTruncated).toBe(false);
131-
132-
expect(output[2].output).toEqual(`3:${'c'.repeat(998)}`);
133-
expect(output[2].isTruncated).toBe(false);
134-
135-
expect(output[3].output).toEqual(`4:${'d'.repeat(998)}`);
136-
expect(output[3].isTruncated).toBe(false);
137-
138-
expect(output[4].output).toEqual('');
139-
expect(output[4].isTruncated).toBe(true);
140-
});
141-
142-
test('sourceCodeChunksToModelOutput handles chunks with different start lines correctly', () => {
143-
const chunks = [
144-
{
145-
source: 'line1\nline2',
146-
startLine: 42
147-
},
148-
{
149-
source: 'another\nset\nof\nlines',
150-
startLine: 100
151-
}
152-
];
153-
154-
const result = sourceCodeChunksToModelOutput(chunks);
155-
156-
expect(result).toHaveLength(2);
157-
expect(result[0].output).toBe('42:line1\n43:line2');
158-
expect(result[1].output).toBe('100:another\n101:set\n102:of\n103:lines');
159-
});
160-
161-
test('sourceCodeChunksToModelOutput returns empty array for empty input', () => {
162-
const result = sourceCodeChunksToModelOutput([]);
163-
expect(result).toHaveLength(0);
164-
});
165-
166-
test('sourceCodeChunksToModelOutput handles single line chunks', () => {
167-
const chunks = [
168-
{
169-
source: 'const a = 1;',
170-
startLine: 10
171-
},
172-
{
173-
source: 'const b = 2;',
174-
startLine: 15
175-
}
176-
];
177-
178-
const result = sourceCodeChunksToModelOutput(chunks);
179-
180-
expect(result).toHaveLength(2);
181-
expect(result[0].output).toBe('10:const a = 1;');
182-
expect(result[1].output).toBe('15:const b = 2;');
183-
expect(result[0].isTruncated).toBe(false);
184-
expect(result[1].isTruncated).toBe(false);
185-
});
186-
187-
18843
test('groupMessageIntoSteps returns an empty array when there are no parts', () => {
18944
const parts: SBChatMessagePart[] = []
19045

0 commit comments

Comments
 (0)