feat(vscode-lm): add image support for VS Code LM API provider

- Upgrade @types/vscode to ^1.106.0 to support LanguageModelDataPart
- Modify vscode-lm-format to convert image blocks to LanguageModelDataPart.image()
- Update VsCodeLmHandler.getModel() to detect image-capable models based on family
- Update useSelectedModel to use model supportsImages capability
- Add tests for image conversion functionality

Closes #11064

Author: roomote

.changeset/vscode-lm-image-support.md

@@ -0,0 +1,13 @@
+---
+"roo-cline": minor
+---
+
+Add image support for VS Code LM API provider
+
+- Upgraded @types/vscode to ^1.106.0 to support LanguageModelDataPart
+- Modified vscode-lm-format to convert image blocks to LanguageModelDataPart.image()
+- Updated VsCodeLmHandler.getModel() to detect image-capable models based on model family
+- Updated useSelectedModel to use model's supportsImages capability instead of always false
+- Added tests for image conversion functionality
+
+This enables users of the VS Code LM API provider to upload images when using vision-capable models like GPT-4o, Claude 3.5 Sonnet, and Gemini models through GitHub Copilot.

pnpm-lock.yaml

@@ -529,14 +529,18 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
 
 			const modelId = this.client.id || modelParts.join(SELECTOR_SEPARATOR)
 
+			// Check if the model supports images based on known model families
+			// VS Code Language Model API 1.106+ supports image inputs via LanguageModelDataPart
+			const supportsImages = checkModelSupportsImages(this.client.family, this.client.id)
+
 			// Build model info with conservative defaults for missing values
 			const modelInfo: ModelInfo = {
 				maxTokens: -1, // Unlimited tokens by default
 				contextWindow:
 					typeof this.client.maxInputTokens === "number"
 						? Math.max(0, this.client.maxInputTokens)
 						: openAiModelInfoSaneDefaults.contextWindow,
-				supportsImages: false, // VSCode Language Model API currently doesn't support image inputs
+				supportsImages,
 				supportsPromptCache: true,
 				inputPrice: 0,
 				outputPrice: 0,
@@ -586,6 +590,45 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
 	}
 }
 
+/**
+ * Model families known to support image inputs via VS Code Language Model API.
+ * These models support the LanguageModelDataPart.image() API introduced in VS Code 1.106+.
+ */
+const IMAGE_CAPABLE_MODEL_FAMILIES = [
+	// OpenAI models with vision capabilities
+	"gpt-4o",
+	"gpt-4o-mini",
+	"gpt-4.1",
+	"gpt-5",
+	"gpt-5-mini",
+	// Anthropic Claude models with vision
+	"claude-3.5-sonnet",
+	"claude-3-5-sonnet",
+	"claude-sonnet-4",
+	"claude-4-sonnet",
+	// Google Gemini models with vision
+	"gemini-2.0-flash",
+	"gemini-2.5-pro",
+	"gemini-pro-vision",
+]
+
+/**
+ * Checks if a model supports image inputs based on its family or ID.
+ * @param family The model family (e.g., "gpt-4o", "claude-3.5-sonnet")
+ * @param id The model ID
+ * @returns true if the model supports image inputs
+ */
+function checkModelSupportsImages(family: string, id: string): boolean {
+	// Check if the family matches any known image-capable model
+	const familyLower = family.toLowerCase()
+	const idLower = id.toLowerCase()
+
+	return IMAGE_CAPABLE_MODEL_FAMILIES.some(
+		(capableFamily) =>
+			familyLower.includes(capableFamily.toLowerCase()) || idLower.includes(capableFamily.toLowerCase()),
+	)
+}
+
 // Static blacklist of VS Code Language Model IDs that should be excluded from the model list e.g. because they will never work
 const VSCODE_LM_STATIC_BLACKLIST: string[] = ["claude-3.7-sonnet", "claude-3.7-sonnet-thought"]
 

src/api/providers/vscode-lm.ts

@@ -26,7 +26,13 @@ interface MockLanguageModelToolCallPart {
 interface MockLanguageModelToolResultPart {
 	type: "tool_result"
 	callId: string
-	content: MockLanguageModelTextPart[]
+	content: (MockLanguageModelTextPart | MockLanguageModelDataPart)[]
+}
+
+interface MockLanguageModelDataPart {
+	type: "data"
+	data: Uint8Array
+	mimeType: string
 }
 
 // Mock vscode namespace
@@ -54,10 +60,32 @@ vitest.mock("vscode", () => {
 		type = "tool_result"
 		constructor(
 			public callId: string,
-			public content: MockLanguageModelTextPart[],
+			public content: (MockLanguageModelTextPart | MockLanguageModelDataPart)[],
 		) {}
 	}
 
+	class MockLanguageModelDataPart {
+		type = "data"
+		constructor(
+			public data: Uint8Array,
+			public mimeType: string,
+		) {}
+
+		static image(data: Uint8Array, mime: string) {
+			return new MockLanguageModelDataPart(data, mime)
+		}
+
+		static json(value: any, mime?: string) {
+			const bytes = new TextEncoder().encode(JSON.stringify(value))
+			return new MockLanguageModelDataPart(bytes, mime || "application/json")
+		}
+
+		static text(value: string, mime?: string) {
+			const bytes = new TextEncoder().encode(value)
+			return new MockLanguageModelDataPart(bytes, mime || "text/plain")
+		}
+	}
+
 	return {
 		LanguageModelChatMessage: {
 			Assistant: vitest.fn((content) => ({
@@ -75,6 +103,7 @@ vitest.mock("vscode", () => {
 		LanguageModelTextPart: MockLanguageModelTextPart,
 		LanguageModelToolCallPart: MockLanguageModelToolCallPart,
 		LanguageModelToolResultPart: MockLanguageModelToolResultPart,
+		LanguageModelDataPart: MockLanguageModelDataPart,
 	}
 })
 
@@ -150,7 +179,7 @@ describe("convertToVsCodeLmMessages", () => {
 		expect(toolCall.type).toBe("tool_call")
 	})
 
-	it("should handle image blocks with appropriate placeholders", () => {
+	it("should convert image blocks to LanguageModelDataPart", () => {
 		const messages: Anthropic.Messages.MessageParam[] = [
 			{
 				role: "user",
@@ -161,7 +190,7 @@ describe("convertToVsCodeLmMessages", () => {
 						source: {
 							type: "base64",
 							media_type: "image/png",
-							data: "base64data",
+							data: "dGVzdA==", // "test" in base64
 						},
 					},
 				],
@@ -171,8 +200,61 @@ describe("convertToVsCodeLmMessages", () => {
 		const result = convertToVsCodeLmMessages(messages)
 
 		expect(result).toHaveLength(1)
-		const imagePlaceholder = result[0].content[1] as MockLanguageModelTextPart
-		expect(imagePlaceholder.value).toContain("[Image (base64): image/png not supported by VSCode LM API]")
+		expect(result[0].content).toHaveLength(2)
+
+		// First part should be text
+		const textPart = result[0].content[0] as MockLanguageModelTextPart
+		expect(textPart.type).toBe("text")
+		expect(textPart.value).toBe("Look at this:")
+
+		// Second part should be a LanguageModelDataPart for the image
+		const imagePart = result[0].content[1] as unknown as MockLanguageModelDataPart
+		expect(imagePart.type).toBe("data")
+		expect(imagePart.mimeType).toBe("image/png")
+		expect(imagePart.data).toBeInstanceOf(Uint8Array)
+	})
+
+	it("should handle images in tool results", () => {
+		const messages: Anthropic.Messages.MessageParam[] = [
+			{
+				role: "user",
+				content: [
+					{
+						type: "tool_result",
+						tool_use_id: "tool-1",
+						content: [
+							{ type: "text", text: "Screenshot result:" },
+							{
+								type: "image",
+								source: {
+									type: "base64",
+									media_type: "image/jpeg",
+									data: "dGVzdA==",
+								},
+							},
+						],
+					},
+				],
+			},
+		]
+
+		const result = convertToVsCodeLmMessages(messages)
+
+		expect(result).toHaveLength(1)
+		expect(result[0].content).toHaveLength(1)
+
+		const toolResult = result[0].content[0] as MockLanguageModelToolResultPart
+		expect(toolResult.type).toBe("tool_result")
+		expect(toolResult.content).toHaveLength(2)
+
+		// First item in tool result should be text
+		const textPart = toolResult.content[0] as MockLanguageModelTextPart
+		expect(textPart.type).toBe("text")
+
+		// Second item should be image data
+		const imagePart = toolResult.content[1] as MockLanguageModelDataPart
+		expect(imagePart.type).toBe("data")
+		expect(imagePart.mimeType).toBe("image/jpeg")
 	})
 })
 

src/api/transform/__tests__/vscode-lm-format.spec.ts

@@ -28,6 +28,39 @@ function asObjectSafe(value: any): object {
 	}
 }
 
+/**
+ * Converts an Anthropic image block to a VS Code LanguageModelDataPart.
+ * Uses the new LanguageModelDataPart.image() API available in VS Code 1.106+.
+ * @param imageBlock The Anthropic image block param
+ * @returns A LanguageModelDataPart for the image
+ */
+function convertImageToDataPart(imageBlock: Anthropic.ImageBlockParam): vscode.LanguageModelDataPart {
+	const source = imageBlock.source
+	const mediaType = source.media_type || "image/png"
+
+	if (source.type === "base64") {
+		// Convert base64 string to Uint8Array
+		const binaryString = atob(source.data)
+		const bytes = new Uint8Array(binaryString.length)
+		for (let i = 0; i < binaryString.length; i++) {
+			bytes[i] = binaryString.charCodeAt(i)
+		}
+		return vscode.LanguageModelDataPart.image(bytes, mediaType)
+	} else if (source.type === "url") {
+		// For URL-based images, we create a placeholder since LanguageModelDataPart.image
+		// expects binary data. The URL would need to be fetched first.
+		// This is a limitation - URL images should be fetched and converted to base64 upstream.
+		console.warn(
+			"Roo Code <Language Model API>: URL-based images require fetching the image data first. Using placeholder.",
+		)
+		return new vscode.LanguageModelDataPart(new Uint8Array(0), mediaType)
+	}
+
+	// Fallback for unknown source types
+	console.warn(`Roo Code <Language Model API>: Unknown image source type: ${(source as any).type}`)
+	return new vscode.LanguageModelDataPart(new Uint8Array(0), mediaType)
+}
+
 export function convertToVsCodeLmMessages(
 	anthropicMessages: Anthropic.Messages.MessageParam[],
 ): vscode.LanguageModelChatMessage[] {
@@ -66,28 +99,24 @@ export function convertToVsCodeLmMessages(
 				const contentParts = [
 					// Convert tool messages to ToolResultParts
 					...toolMessages.map((toolMessage) => {
-						// Process tool result content into TextParts
-						const toolContentParts: vscode.LanguageModelTextPart[] =
+						// Process tool result content into TextParts or DataParts
+						const toolContentParts: (vscode.LanguageModelTextPart | vscode.LanguageModelDataPart)[] =
 							typeof toolMessage.content === "string"
 								? [new vscode.LanguageModelTextPart(toolMessage.content)]
 								: (toolMessage.content?.map((part) => {
 										if (part.type === "image") {
-											return new vscode.LanguageModelTextPart(
-												`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
-											)
+											return convertImageToDataPart(part)
 										}
 										return new vscode.LanguageModelTextPart(part.text)
 									}) ?? [new vscode.LanguageModelTextPart("")])
 
 						return new vscode.LanguageModelToolResultPart(toolMessage.tool_use_id, toolContentParts)
 					}),
 
-					// Convert non-tool messages to TextParts after tool messages
+					// Convert non-tool messages to TextParts or DataParts after tool messages
 					...nonToolMessages.map((part) => {
 						if (part.type === "image") {
-							return new vscode.LanguageModelTextPart(
-								`[Image (${part.source?.type || "Unknown source-type"}): ${part.source?.media_type || "unknown media-type"} not supported by VSCode LM API]`,
-							)
+							return convertImageToDataPart(part)
 						}
 						return new vscode.LanguageModelTextPart(part.text)
 					}),

src/api/transform/vscode-lm-format.ts

@@ -554,7 +554,7 @@
 		"@types/string-similarity": "^4.0.2",
 		"@types/tmp": "^0.2.6",
 		"@types/turndown": "^5.0.5",
-		"@types/vscode": "^1.84.0",
+		"@types/vscode": "^1.106.0",
 		"@vscode/test-electron": "^2.5.2",
 		"@vscode/vsce": "3.3.2",
 		"ai": "^6.0.0",

src/package.json

@@ -312,7 +312,8 @@ function getSelectedModel({
 				: vscodeLlmDefaultModelId
 			const modelFamily = apiConfiguration?.vsCodeLmModelSelector?.family ?? vscodeLlmDefaultModelId
 			const info = vscodeLlmModels[modelFamily as keyof typeof vscodeLlmModels]
-			return { id, info: { ...openAiModelInfoSaneDefaults, ...info, supportsImages: false } } // VSCode LM API currently doesn't support images.
+			// VS Code LM API 1.106+ supports images via LanguageModelDataPart - use model's supportsImages capability
+			return { id, info: { ...openAiModelInfoSaneDefaults, ...info } }
 		}
 		case "cerebras": {
 			const id = apiConfiguration.apiModelId ?? defaultModelId