Skip to content
This repository was archived by the owner on May 15, 2026. It is now read-only.

Commit 43f7ce0

Browse files
hannesrudolphroomote
hannesrudolph
and
roomote
authored
feat: add support for image file @mentions (#10189)
Co-authored-by: Roo Code <roomote@roocode.com>
1 parent e287a82 commit 43f7ce0

9 files changed

Lines changed: 561 additions & 10 deletions

‎src/core/mentions/__tests__/resolveImageMentions.spec.ts‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
import * as path from "path"
2+
3+
import { resolveImageMentions } from "../resolveImageMentions"
4+
5+
vi.mock("../../tools/helpers/imageHelpers", () => ({
6+
isSupportedImageFormat: vi.fn((ext: string) =>
7+
[".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg", ".bmp", ".ico", ".tiff", ".tif", ".avif"].includes(
8+
ext.toLowerCase(),
9+
),
10+
),
11+
readImageAsDataUrlWithBuffer: vi.fn(),
12+
validateImageForProcessing: vi.fn(),
13+
ImageMemoryTracker: vi.fn().mockImplementation(() => ({
14+
getTotalMemoryUsed: vi.fn().mockReturnValue(0),
15+
addMemoryUsage: vi.fn(),
16+
})),
17+
DEFAULT_MAX_IMAGE_FILE_SIZE_MB: 5,
18+
DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB: 20,
19+
}))
20+
21+
import { validateImageForProcessing, readImageAsDataUrlWithBuffer } from "../../tools/helpers/imageHelpers"
22+
23+
const mockReadImageAsDataUrl = vi.mocked(readImageAsDataUrlWithBuffer)
24+
const mockValidateImage = vi.mocked(validateImageForProcessing)
25+
26+
describe("resolveImageMentions", () => {
27+
beforeEach(() => {
28+
vi.clearAllMocks()
29+
// Default: validation passes
30+
mockValidateImage.mockResolvedValue({ isValid: true, sizeInMB: 0.1 })
31+
})
32+
33+
it("should append a data URL when a local png mention is present", async () => {
34+
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
35+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
36+
37+
const result = await resolveImageMentions({
38+
text: "Please look at @/assets/cat.png",
39+
images: [],
40+
cwd: "/workspace",
41+
})
42+
43+
expect(mockValidateImage).toHaveBeenCalled()
44+
expect(mockReadImageAsDataUrl).toHaveBeenCalledWith(path.resolve("/workspace", "assets/cat.png"))
45+
expect(result.text).toBe("Please look at @/assets/cat.png")
46+
expect(result.images).toEqual([dataUrl])
47+
})
48+
49+
it("should support gif images (matching read_file)", async () => {
50+
const dataUrl = `data:image/gif;base64,${Buffer.from("gif-bytes").toString("base64")}`
51+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("gif-bytes") })
52+
53+
const result = await resolveImageMentions({
54+
text: "See @/animation.gif",
55+
images: [],
56+
cwd: "/workspace",
57+
})
58+
59+
expect(result.images).toEqual([dataUrl])
60+
})
61+
62+
it("should support svg images (matching read_file)", async () => {
63+
const dataUrl = `data:image/svg+xml;base64,${Buffer.from("svg-bytes").toString("base64")}`
64+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("svg-bytes") })
65+
66+
const result = await resolveImageMentions({
67+
text: "See @/icon.svg",
68+
images: [],
69+
cwd: "/workspace",
70+
})
71+
72+
expect(result.images).toEqual([dataUrl])
73+
})
74+
75+
it("should ignore non-image mentions", async () => {
76+
const result = await resolveImageMentions({
77+
text: "See @/src/index.ts",
78+
images: [],
79+
cwd: "/workspace",
80+
})
81+
82+
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
83+
expect(result.images).toEqual([])
84+
})
85+
86+
it("should skip unreadable files (fail-soft)", async () => {
87+
mockReadImageAsDataUrl.mockRejectedValue(new Error("ENOENT"))
88+
89+
const result = await resolveImageMentions({
90+
text: "See @/missing.webp",
91+
images: [],
92+
cwd: "/workspace",
93+
})
94+
95+
expect(result.images).toEqual([])
96+
})
97+
98+
it("should respect rooIgnoreController", async () => {
99+
const dataUrl = `data:image/jpeg;base64,${Buffer.from("jpg-bytes").toString("base64")}`
100+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("jpg-bytes") })
101+
const rooIgnoreController = {
102+
validateAccess: vi.fn().mockReturnValue(false),
103+
}
104+
105+
const result = await resolveImageMentions({
106+
text: "See @/secret.jpg",
107+
images: [],
108+
cwd: "/workspace",
109+
rooIgnoreController,
110+
})
111+
112+
expect(rooIgnoreController.validateAccess).toHaveBeenCalledWith("secret.jpg")
113+
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
114+
expect(result.images).toEqual([])
115+
})
116+
117+
it("should dedupe when mention repeats", async () => {
118+
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
119+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
120+
121+
const result = await resolveImageMentions({
122+
text: "@/a.png and again @/a.png",
123+
images: [],
124+
cwd: "/workspace",
125+
})
126+
127+
expect(result.images).toHaveLength(1)
128+
})
129+
130+
it("should skip images when supportsImages is false", async () => {
131+
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
132+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
133+
134+
const result = await resolveImageMentions({
135+
text: "See @/cat.png",
136+
images: [],
137+
cwd: "/workspace",
138+
supportsImages: false,
139+
})
140+
141+
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
142+
expect(result.images).toEqual([])
143+
})
144+
145+
it("should skip images that exceed size limits", async () => {
146+
mockValidateImage.mockResolvedValue({
147+
isValid: false,
148+
reason: "size_limit",
149+
notice: "Image too large",
150+
})
151+
152+
const result = await resolveImageMentions({
153+
text: "See @/huge.png",
154+
images: [],
155+
cwd: "/workspace",
156+
})
157+
158+
expect(mockValidateImage).toHaveBeenCalled()
159+
expect(mockReadImageAsDataUrl).not.toHaveBeenCalled()
160+
expect(result.images).toEqual([])
161+
})
162+
163+
it("should skip images that would exceed memory limit", async () => {
164+
mockValidateImage.mockResolvedValue({
165+
isValid: false,
166+
reason: "memory_limit",
167+
notice: "Would exceed memory limit",
168+
})
169+
170+
const result = await resolveImageMentions({
171+
text: "See @/large.png",
172+
images: [],
173+
cwd: "/workspace",
174+
})
175+
176+
expect(result.images).toEqual([])
177+
})
178+
179+
it("should pass custom size limits to validation", async () => {
180+
const dataUrl = `data:image/png;base64,${Buffer.from("png-bytes").toString("base64")}`
181+
mockReadImageAsDataUrl.mockResolvedValue({ dataUrl, buffer: Buffer.from("png-bytes") })
182+
183+
await resolveImageMentions({
184+
text: "See @/cat.png",
185+
images: [],
186+
cwd: "/workspace",
187+
maxImageFileSize: 10,
188+
maxTotalImageSize: 50,
189+
})
190+
191+
expect(mockValidateImage).toHaveBeenCalledWith(expect.any(String), true, 10, 50, 0)
192+
})
193+
})

‎src/core/mentions/index.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -284,7 +284,13 @@ async function getFileOrFolderContent(
284284
const stats = await fs.stat(absPath)
285285

286286
if (stats.isFile()) {
287-
if (rooIgnoreController && !rooIgnoreController.validateAccess(absPath)) {
287+
// Avoid trying to include image binary content as text context.
288+
// Image mentions are handled separately via image attachment flow.
289+
const isBinary = await isBinaryFile(absPath).catch(() => false)
290+
if (isBinary) {
291+
return `(Binary file ${mentionPath} omitted)`
292+
}
293+
if (rooIgnoreController && !rooIgnoreController.validateAccess(unescapedPath)) {
288294
return `(File ${mentionPath} is ignored by .rooignore)`
289295
}
290296
try {

‎src/core/mentions/resolveImageMentions.ts‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
import * as path from "path"
2+
3+
import { mentionRegexGlobal, unescapeSpaces } from "../../shared/context-mentions"
4+
import {
5+
isSupportedImageFormat,
6+
readImageAsDataUrlWithBuffer,
7+
validateImageForProcessing,
8+
ImageMemoryTracker,
9+
DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
10+
DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
11+
} from "../tools/helpers/imageHelpers"
12+
13+
const MAX_IMAGES_PER_MESSAGE = 20
14+
15+
export interface ResolveImageMentionsOptions {
16+
text: string
17+
images?: string[]
18+
cwd: string
19+
rooIgnoreController?: { validateAccess: (filePath: string) => boolean }
20+
/** Whether the current model supports images. Defaults to true. */
21+
supportsImages?: boolean
22+
/** Maximum size per image file in MB. Defaults to 5MB. */
23+
maxImageFileSize?: number
24+
/** Maximum total size of all images in MB. Defaults to 20MB. */
25+
maxTotalImageSize?: number
26+
}
27+
28+
export interface ResolveImageMentionsResult {
29+
text: string
30+
images: string[]
31+
}
32+
33+
function isPathWithinCwd(absPath: string, cwd: string): boolean {
34+
const rel = path.relative(cwd, absPath)
35+
return rel !== "" && !rel.startsWith("..") && !path.isAbsolute(rel)
36+
}
37+
38+
function dedupePreserveOrder(values: string[]): string[] {
39+
const seen = new Set<string>()
40+
const result: string[] = []
41+
for (const v of values) {
42+
if (seen.has(v)) continue
43+
seen.add(v)
44+
result.push(v)
45+
}
46+
return result
47+
}
48+
49+
/**
50+
* Resolves local image file mentions like `@/path/to/image.png` found in `text` into `data:image/...;base64,...`
51+
* and appends them to the outgoing `images` array.
52+
*
53+
* Behavior matches the read_file tool:
54+
* - Supports the same image formats: png, jpg, jpeg, gif, webp, svg, bmp, ico, tiff, avif
55+
* - Respects per-file size limits (default 5MB)
56+
* - Respects total memory limits (default 20MB)
57+
* - Skips images if model doesn't support them
58+
* - Respects `.rooignore` via `rooIgnoreController.validateAccess` when provided
59+
*/
60+
export async function resolveImageMentions({
61+
text,
62+
images,
63+
cwd,
64+
rooIgnoreController,
65+
supportsImages = true,
66+
maxImageFileSize = DEFAULT_MAX_IMAGE_FILE_SIZE_MB,
67+
maxTotalImageSize = DEFAULT_MAX_TOTAL_IMAGE_SIZE_MB,
68+
}: ResolveImageMentionsOptions): Promise<ResolveImageMentionsResult> {
69+
const existingImages = Array.isArray(images) ? images : []
70+
if (existingImages.length >= MAX_IMAGES_PER_MESSAGE) {
71+
return { text, images: existingImages.slice(0, MAX_IMAGES_PER_MESSAGE) }
72+
}
73+
74+
// If model doesn't support images, skip image processing entirely
75+
if (!supportsImages) {
76+
return { text, images: existingImages }
77+
}
78+
79+
const mentions = Array.from(text.matchAll(mentionRegexGlobal))
80+
.map((m) => m[1])
81+
.filter(Boolean)
82+
if (mentions.length === 0) {
83+
return { text, images: existingImages }
84+
}
85+
86+
const imageMentions = mentions.filter((mention) => {
87+
if (!mention.startsWith("/")) return false
88+
const relPath = unescapeSpaces(mention.slice(1))
89+
const ext = path.extname(relPath).toLowerCase()
90+
return isSupportedImageFormat(ext)
91+
})
92+
93+
if (imageMentions.length === 0) {
94+
return { text, images: existingImages }
95+
}
96+
97+
const imageMemoryTracker = new ImageMemoryTracker()
98+
const newImages: string[] = []
99+
100+
for (const mention of imageMentions) {
101+
if (existingImages.length + newImages.length >= MAX_IMAGES_PER_MESSAGE) {
102+
break
103+
}
104+
105+
const relPath = unescapeSpaces(mention.slice(1))
106+
const absPath = path.resolve(cwd, relPath)
107+
if (!isPathWithinCwd(absPath, cwd)) {
108+
continue
109+
}
110+
111+
if (rooIgnoreController && !rooIgnoreController.validateAccess(relPath)) {
112+
continue
113+
}
114+
115+
// Validate image size limits (matches read_file behavior)
116+
try {
117+
const validationResult = await validateImageForProcessing(
118+
absPath,
119+
supportsImages,
120+
maxImageFileSize,
121+
maxTotalImageSize,
122+
imageMemoryTracker.getTotalMemoryUsed(),
123+
)
124+
125+
if (!validationResult.isValid) {
126+
// Skip this image due to size/memory limits, but continue processing others
127+
continue
128+
}
129+
130+
const { dataUrl } = await readImageAsDataUrlWithBuffer(absPath)
131+
newImages.push(dataUrl)
132+
133+
// Track memory usage
134+
if (validationResult.sizeInMB) {
135+
imageMemoryTracker.addMemoryUsage(validationResult.sizeInMB)
136+
}
137+
} catch {
138+
// Fail-soft: skip unreadable/missing files.
139+
continue
140+
}
141+
}
142+
143+
const merged = dedupePreserveOrder([...existingImages, ...newImages]).slice(0, MAX_IMAGES_PER_MESSAGE)
144+
return { text, images: merged }
145+
}

‎src/core/webview/__tests__/ClineProvider.spec.ts‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3051,7 +3051,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
30513051
expect(mockCline.overwriteClineMessages).toHaveBeenCalledWith([mockMessages[0]])
30523052
expect(mockCline.overwriteApiConversationHistory).toHaveBeenCalledWith([{ ts: 1000 }])
30533053
// Verify submitUserMessage was called with the edited content
3054-
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with preserved images", undefined)
3054+
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with preserved images", [])
30553055
})
30563056

30573057
test("handles editing messages with file attachments", async () => {
@@ -3104,7 +3104,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
31043104
})
31053105

31063106
expect(mockCline.overwriteClineMessages).toHaveBeenCalled()
3107-
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with file attachment", undefined)
3107+
expect(mockCline.submitUserMessage).toHaveBeenCalledWith("Edited message with file attachment", [])
31083108
})
31093109
})
31103110

@@ -3635,7 +3635,7 @@ describe("ClineProvider - Comprehensive Edit/Delete Edge Cases", () => {
36353635
await messageHandler({ type: "editMessageConfirm", messageTs: 2000, text: largeEditedContent })
36363636

36373637
expect(mockCline.overwriteClineMessages).toHaveBeenCalled()
3638-
expect(mockCline.submitUserMessage).toHaveBeenCalledWith(largeEditedContent, undefined)
3638+
expect(mockCline.submitUserMessage).toHaveBeenCalledWith(largeEditedContent, [])
36393639
})
36403640

36413641
test("handles deleting messages with large payloads", async () => {

0 commit comments

Comments
 (0)