Skip to content
This repository was archived by the owner on May 15, 2026. It is now read-only.

Commit 7ec9664

Browse files
committed
feat(vscode-lm): re-apply image support with types version fix
Re-applies the image support feature from PR #11065 that was reverted, with a fix for the @types/vscode version issue. The fix keeps @types/vscode at ^1.84.0 in package.json (matching engines.vscode) while the pnpm-lock.yaml resolves to version 1.108.1 which provides the LanguageModelDataPart API. This approach: - Maintains compatibility with VS Code 1.84.0+ - Provides TypeScript types for the new LanguageModelDataPart API - Resolves the vsix build failure from the original PR Changes: - Add convertImageToDataPart() for image-to-LanguageModelDataPart conversion - Add checkModelSupportsImages() for model image capability detection - Update useSelectedModel to use models actual supportsImages capability - Add comprehensive tests for image support Relates to #11064, #11065
1 parent 40b2bdc commit 7ec9664

6 files changed

Lines changed: 335 additions & 20 deletions

File tree

pnpm-lock.yaml

Lines changed: 7 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

src/api/providers/__tests__/vscode-lm.spec.ts

Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import type { Mock } from "vitest"
2+
import { checkModelSupportsImages, IMAGE_CAPABLE_MODEL_PREFIXES } from "../vscode-lm"
23

34
// Mocks must come first, before imports
45
vi.mock("vscode", () => {
@@ -537,3 +538,92 @@ describe("VsCodeLmHandler", () => {
537538
})
538539
})
539540
})
541+
542+
describe("checkModelSupportsImages", () => {
543+
describe("OpenAI GPT models", () => {
544+
it("should return true for all gpt-* models (GitHub Copilot)", () => {
545+
// All GPT models in GitHub Copilot support images
546+
expect(checkModelSupportsImages("gpt", "gpt-4o")).toBe(true)
547+
expect(checkModelSupportsImages("gpt", "gpt-4.1")).toBe(true)
548+
expect(checkModelSupportsImages("gpt", "gpt-5")).toBe(true)
549+
expect(checkModelSupportsImages("gpt", "gpt-5.1")).toBe(true)
550+
expect(checkModelSupportsImages("gpt", "gpt-5.2")).toBe(true)
551+
expect(checkModelSupportsImages("gpt-mini", "gpt-5-mini")).toBe(true)
552+
expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex")).toBe(true)
553+
expect(checkModelSupportsImages("gpt-codex", "gpt-5.2-codex")).toBe(true)
554+
expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex-max")).toBe(true)
555+
expect(checkModelSupportsImages("gpt-codex", "gpt-5.1-codex-mini")).toBe(true)
556+
})
557+
558+
it("should return true for o1 and o3 reasoning models", () => {
559+
expect(checkModelSupportsImages("o1", "o1-preview")).toBe(true)
560+
expect(checkModelSupportsImages("o1", "o1-mini")).toBe(true)
561+
expect(checkModelSupportsImages("o3", "o3")).toBe(true)
562+
})
563+
})
564+
565+
describe("Anthropic Claude models", () => {
566+
it("should return true for all claude-* models (GitHub Copilot)", () => {
567+
// All Claude models in GitHub Copilot support images
568+
expect(checkModelSupportsImages("claude-haiku", "claude-haiku-4.5")).toBe(true)
569+
expect(checkModelSupportsImages("claude-opus", "claude-opus-4.5")).toBe(true)
570+
expect(checkModelSupportsImages("claude-sonnet", "claude-sonnet-4")).toBe(true)
571+
expect(checkModelSupportsImages("claude-sonnet", "claude-sonnet-4.5")).toBe(true)
572+
})
573+
})
574+
575+
describe("Google Gemini models", () => {
576+
it("should return true for all gemini-* models (GitHub Copilot)", () => {
577+
// All Gemini models in GitHub Copilot support images
578+
expect(checkModelSupportsImages("gemini-pro", "gemini-2.5-pro")).toBe(true)
579+
expect(checkModelSupportsImages("gemini-flash", "gemini-3-flash-preview")).toBe(true)
580+
expect(checkModelSupportsImages("gemini-pro", "gemini-3-pro-preview")).toBe(true)
581+
})
582+
})
583+
584+
describe("non-vision models", () => {
585+
it("should return false for grok models (text-only in GitHub Copilot)", () => {
586+
// Grok is the only model family in GitHub Copilot that doesn't support images
587+
expect(checkModelSupportsImages("grok", "grok-code-fast-1")).toBe(false)
588+
})
589+
590+
it("should return false for models with non-matching prefixes", () => {
591+
// Models that don't start with gpt, claude, gemini, o1, or o3
592+
expect(checkModelSupportsImages("mistral", "mistral-large")).toBe(false)
593+
expect(checkModelSupportsImages("llama", "llama-3-70b")).toBe(false)
594+
expect(checkModelSupportsImages("unknown", "some-random-model")).toBe(false)
595+
})
596+
})
597+
598+
describe("case insensitivity", () => {
599+
it("should match regardless of case", () => {
600+
expect(checkModelSupportsImages("GPT", "GPT-4O")).toBe(true)
601+
expect(checkModelSupportsImages("CLAUDE", "CLAUDE-SONNET-4")).toBe(true)
602+
expect(checkModelSupportsImages("GEMINI", "GEMINI-2.5-PRO")).toBe(true)
603+
})
604+
})
605+
606+
describe("prefix matching", () => {
607+
it("should only match IDs that start with known prefixes", () => {
608+
// ID must START with the prefix, not just contain it
609+
expect(checkModelSupportsImages("custom", "gpt-4o")).toBe(true) // ID starts with gpt
610+
expect(checkModelSupportsImages("custom", "my-gpt-model")).toBe(false) // gpt not at start
611+
expect(checkModelSupportsImages("custom", "not-claude-model")).toBe(false) // claude not at start
612+
})
613+
})
614+
})
615+
616+
describe("IMAGE_CAPABLE_MODEL_PREFIXES", () => {
617+
it("should export the model prefixes array", () => {
618+
expect(Array.isArray(IMAGE_CAPABLE_MODEL_PREFIXES)).toBe(true)
619+
expect(IMAGE_CAPABLE_MODEL_PREFIXES.length).toBeGreaterThan(0)
620+
})
621+
622+
it("should include key model prefixes", () => {
623+
expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("gpt")
624+
expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("claude")
625+
expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("gemini")
626+
expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("o1")
627+
expect(IMAGE_CAPABLE_MODEL_PREFIXES).toContain("o3")
628+
})
629+
})

src/api/providers/vscode-lm.ts

Lines changed: 42 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -529,14 +529,18 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
529529

530530
const modelId = this.client.id || modelParts.join(SELECTOR_SEPARATOR)
531531

532+
// Check if the model supports images based on known model families
533+
// VS Code Language Model API 1.106+ supports image inputs via LanguageModelDataPart
534+
const supportsImages = checkModelSupportsImages(this.client.family, this.client.id)
535+
532536
// Build model info with conservative defaults for missing values
533537
const modelInfo: ModelInfo = {
534538
maxTokens: -1, // Unlimited tokens by default
535539
contextWindow:
536540
typeof this.client.maxInputTokens === "number"
537541
? Math.max(0, this.client.maxInputTokens)
538542
: openAiModelInfoSaneDefaults.contextWindow,
539-
supportsImages: false, // VSCode Language Model API currently doesn't support image inputs
543+
supportsImages,
540544
supportsPromptCache: true,
541545
inputPrice: 0,
542546
outputPrice: 0,
@@ -586,8 +590,43 @@ export class VsCodeLmHandler extends BaseProvider implements SingleCompletionHan
586590
}
587591
}
588592

589-
// Static blacklist of VS Code Language Model IDs that should be excluded from the model list e.g. because they will never work
590-
const VSCODE_LM_STATIC_BLACKLIST: string[] = ["claude-3.7-sonnet", "claude-3.7-sonnet-thought"]
593+
/**
594+
* Model ID prefixes that support image inputs via VS Code Language Model API.
595+
* These models support the LanguageModelDataPart.image() API introduced in VS Code 1.106+.
596+
*
597+
* All GitHub Copilot models with these prefixes support images.
598+
* Only grok-* models don't support images (text only).
599+
*
600+
* Source: https://models.dev/api.json (github-copilot provider models)
601+
*/
602+
export const IMAGE_CAPABLE_MODEL_PREFIXES = [
603+
"gpt", // All GPT models (gpt-4o, gpt-4.1, gpt-5, gpt-5.1, gpt-5.2, gpt-5-mini, gpt-5.1-codex, etc.)
604+
"claude", // All Claude models (claude-haiku-4.5, claude-opus-4.5, claude-sonnet-4, claude-sonnet-4.5)
605+
"gemini", // All Gemini models (gemini-2.5-pro, gemini-3-flash-preview, gemini-3-pro-preview)
606+
"o1", // OpenAI o1 reasoning models
607+
"o3", // OpenAI o3 reasoning models
608+
]
609+
610+
/**
611+
* Checks if a model supports image inputs based on its model ID.
612+
* Uses prefix matching against known image-capable model families.
613+
*
614+
* @param _family The model family (unused, kept for API compatibility)
615+
* @param id The model ID
616+
* @returns true if the model supports image inputs
617+
*/
618+
export function checkModelSupportsImages(_family: string, id: string): boolean {
619+
const idLower = id.toLowerCase()
620+
return IMAGE_CAPABLE_MODEL_PREFIXES.some((prefix) => idLower.startsWith(prefix))
621+
}
622+
623+
// Static blacklist of VS Code Language Model IDs that should be excluded from the model list
624+
// e.g. because they don't support native tool calling or will never work
625+
const VSCODE_LM_STATIC_BLACKLIST: string[] = [
626+
"claude-3.7-sonnet",
627+
"claude-3.7-sonnet-thought",
628+
"claude-opus-41", // Does not support native tool calling
629+
]
591630

592631
export async function getVsCodeLmModels() {
593632
try {

0 commit comments

Comments
 (0)