Skip to content

Commit ff6893a

Browse files
SteveSandersonMSCopilot
SteveSandersonMS
and
Copilot
committed
Add modelCapabilities override to Node SDK
Co-Authored-By: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent 200bfef commit ff6893a

File tree

8 files changed

+475
-2
lines changed

8 files changed

+475
-2
lines changed

‎nodejs/src/client.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -712,6 +712,7 @@ export class CopilotClient {
712712
availableTools: config.availableTools,
713713
excludedTools: config.excludedTools,
714714
provider: config.provider,
715+
modelCapabilities: config.modelCapabilities,
715716
requestPermission: true,
716717
requestUserInput: !!config.onUserInputRequest,
717718
requestElicitation: !!config.onElicitationRequest,

‎nodejs/src/generated/rpc.ts‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,23 @@ export interface ModelsListResult {
7373
* Maximum total context window size in tokens
7474
*/
7575
max_context_window_tokens: number;
76+
/**
77+
* Vision-specific limits
78+
*/
79+
vision?: {
80+
/**
81+
* MIME types the model accepts
82+
*/
83+
supported_media_types: string[];
84+
/**
85+
* Maximum number of images per prompt
86+
*/
87+
max_prompt_images: number;
88+
/**
89+
* Maximum image size in bytes
90+
*/
91+
max_prompt_image_size: number;
92+
};
7693
};
7794
};
7895
/**
@@ -396,6 +413,43 @@ export interface SessionModelSwitchToParams {
396413
* Reasoning effort level to use for the model
397414
*/
398415
reasoningEffort?: string;
416+
/**
417+
* Override individual model capabilities resolved by the runtime
418+
*/
419+
modelCapabilities?: {
420+
/**
421+
* Feature flags indicating what the model supports
422+
*/
423+
supports?: {
424+
vision?: boolean;
425+
reasoningEffort?: boolean;
426+
};
427+
/**
428+
* Token limits for prompts, outputs, and context window
429+
*/
430+
limits?: {
431+
max_prompt_tokens?: number;
432+
max_output_tokens?: number;
433+
/**
434+
* Maximum total context window size in tokens
435+
*/
436+
max_context_window_tokens?: number;
437+
vision?: {
438+
/**
439+
* MIME types the model accepts
440+
*/
441+
supported_media_types?: string[];
442+
/**
443+
* Maximum number of images per prompt
444+
*/
445+
max_prompt_images?: number;
446+
/**
447+
* Maximum image size in bytes
448+
*/
449+
max_prompt_image_size?: number;
450+
};
451+
};
452+
};
399453
}
400454

401455
export interface SessionModeGetResult {

‎nodejs/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ export type {
3636
MessageOptions,
3737
ModelBilling,
3838
ModelCapabilities,
39+
ModelCapabilitiesOverride,
3940
ModelInfo,
4041
ModelPolicy,
4142
PermissionHandler,

‎nodejs/src/session.ts‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ import type {
2424
PermissionRequest,
2525
PermissionRequestResult,
2626
ReasoningEffort,
27+
ModelCapabilitiesOverride,
2728
SectionTransformFn,
2829
SessionCapabilities,
2930
SessionEvent,
@@ -1029,7 +1030,10 @@ export class CopilotSession {
10291030
* await session.setModel("claude-sonnet-4.6", { reasoningEffort: "high" });
10301031
* ```
10311032
*/
1032-
async setModel(model: string, options?: { reasoningEffort?: ReasoningEffort }): Promise<void> {
1033+
async setModel(
1034+
model: string,
1035+
options?: { reasoningEffort?: ReasoningEffort; modelCapabilities?: ModelCapabilitiesOverride },
1036+
): Promise<void> {
10331037
await this.rpc.model.switchTo({ modelId: model, ...options });
10341038
}
10351039

‎nodejs/src/types.ts‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1065,6 +1065,9 @@ export interface SessionConfig {
10651065
*/
10661066
reasoningEffort?: ReasoningEffort;
10671067

1068+
/** Per-property overrides for model capabilities, deep-merged over runtime defaults. */
1069+
modelCapabilities?: ModelCapabilitiesOverride;
1070+
10681071
/**
10691072
* Override the default configuration directory location.
10701073
* When specified, the session will use this directory for storing config and state.
@@ -1465,6 +1468,13 @@ export interface ModelCapabilities {
14651468
};
14661469
}
14671470

1471+
/** Deep-partial override for model capabilities — every property optional. */
1472+
export type ModelCapabilitiesOverride = {
1473+
[K in keyof ModelCapabilities]?: {
1474+
[P in keyof ModelCapabilities[K]]?: ModelCapabilities[K][P];
1475+
};
1476+
};
1477+
14681478
/**
14691479
* Model policy state
14701480
*/

‎nodejs/test/e2e/session_config.test.ts‎

Lines changed: 75 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ import { approveAll } from "../../src/index.js";
55
import { createSdkTestContext } from "./harness/sdkTestContext.js";
66

77
describe("Session Configuration", async () => {
8-
const { copilotClient: client, workDir } = await createSdkTestContext();
8+
const { copilotClient: client, workDir, openAiEndpoint } = await createSdkTestContext();
99

1010
it("should use workingDirectory for tool execution", async () => {
1111
const subDir = join(workDir, "subproject");
@@ -75,4 +75,78 @@ describe("Session Configuration", async () => {
7575
// Just verify send doesn't throw — attachment support varies by runtime
7676
await session.disconnect();
7777
});
78+
79+
const PNG_1X1 = Buffer.from(
80+
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==",
81+
"base64",
82+
);
83+
const VIEW_IMAGE_PROMPT = "Use the view tool to look at the file test.png and describe what you see";
84+
85+
function hasImageUrlContent(messages: Array<{ role: string; content: unknown }>): boolean {
86+
return messages.some(
87+
(m) =>
88+
m.role === "user" &&
89+
Array.isArray(m.content) &&
90+
m.content.some((p: { type: string }) => p.type === "image_url"),
91+
);
92+
}
93+
94+
it("vision disabled then enabled via setModel", async () => {
95+
await writeFile(join(workDir, "test.png"), PNG_1X1);
96+
97+
const session = await client.createSession({
98+
onPermissionRequest: approveAll,
99+
modelCapabilities: { supports: { vision: false } },
100+
});
101+
102+
// Turn 1: vision off — no image_url expected
103+
await session.sendAndWait({ prompt: VIEW_IMAGE_PROMPT });
104+
const trafficAfterT1 = await openAiEndpoint.getExchanges();
105+
const t1Messages = trafficAfterT1.flatMap((e) => e.request.messages ?? []);
106+
expect(hasImageUrlContent(t1Messages)).toBe(false);
107+
108+
// Switch vision on (re-specify same model with updated capabilities)
109+
await session.setModel("claude-sonnet-4.5", {
110+
modelCapabilities: { supports: { vision: true } },
111+
});
112+
113+
// Turn 2: vision on — image_url expected
114+
await session.sendAndWait({ prompt: VIEW_IMAGE_PROMPT });
115+
const trafficAfterT2 = await openAiEndpoint.getExchanges();
116+
// Only check exchanges added after turn 1
117+
const newExchanges = trafficAfterT2.slice(trafficAfterT1.length);
118+
const t2Messages = newExchanges.flatMap((e) => e.request.messages ?? []);
119+
expect(hasImageUrlContent(t2Messages)).toBe(true);
120+
121+
await session.disconnect();
122+
});
123+
124+
it("vision enabled then disabled via setModel", async () => {
125+
await writeFile(join(workDir, "test.png"), PNG_1X1);
126+
127+
const session = await client.createSession({
128+
onPermissionRequest: approveAll,
129+
modelCapabilities: { supports: { vision: true } },
130+
});
131+
132+
// Turn 1: vision on — image_url expected
133+
await session.sendAndWait({ prompt: VIEW_IMAGE_PROMPT });
134+
const trafficAfterT1 = await openAiEndpoint.getExchanges();
135+
const t1Messages = trafficAfterT1.flatMap((e) => e.request.messages ?? []);
136+
expect(hasImageUrlContent(t1Messages)).toBe(true);
137+
138+
// Switch vision off
139+
await session.setModel("claude-sonnet-4.5", {
140+
modelCapabilities: { supports: { vision: false } },
141+
});
142+
143+
// Turn 2: vision off — no image_url expected in new exchanges
144+
await session.sendAndWait({ prompt: VIEW_IMAGE_PROMPT });
145+
const trafficAfterT2 = await openAiEndpoint.getExchanges();
146+
const newExchanges = trafficAfterT2.slice(trafficAfterT1.length);
147+
const t2Messages = newExchanges.flatMap((e) => e.request.messages ?? []);
148+
expect(hasImageUrlContent(t2Messages)).toBe(false);
149+
150+
await session.disconnect();
151+
});
78152
});

‎test/snapshots/session_config/vision_disabled_then_enabled_via_setmodel.yaml‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
models:
2+
- claude-sonnet-4.5
3+
conversations:
4+
- messages:
5+
- role: system
6+
content: ${system}
7+
- role: user
8+
content: Use the view tool to look at the file test.png and describe what you see
9+
- role: assistant
10+
content: I'll view the test.png file for you.
11+
- role: assistant
12+
tool_calls:
13+
- id: toolcall_0
14+
type: function
15+
function:
16+
name: report_intent
17+
arguments: '{"intent":"Viewing image file"}'
18+
- role: assistant
19+
tool_calls:
20+
- id: toolcall_1
21+
type: function
22+
function:
23+
name: view
24+
arguments: '{"path":"${workdir}/test.png"}'
25+
- messages:
26+
- role: system
27+
content: ${system}
28+
- role: user
29+
content: Use the view tool to look at the file test.png and describe what you see
30+
- role: assistant
31+
content: I'll view the test.png file for you.
32+
tool_calls:
33+
- id: toolcall_0
34+
type: function
35+
function:
36+
name: report_intent
37+
arguments: '{"intent":"Viewing image file"}'
38+
- id: toolcall_1
39+
type: function
40+
function:
41+
name: view
42+
arguments: '{"path":"${workdir}/test.png"}'
43+
- role: tool
44+
tool_call_id: toolcall_0
45+
content: Intent logged
46+
- role: tool
47+
tool_call_id: toolcall_1
48+
content: Viewed image file successfully.
49+
- role: assistant
50+
content: The image shows the **GitHub Copilot logo** - a stylized hexagonal icon in shades of blue and purple with a
51+
distinctive geometric design. The logo features a three-dimensional hexagon shape with gradient colors
52+
transitioning from deep blue to lighter blue/purple tones, creating a modern, tech-forward aesthetic. This is
53+
GitHub's AI pair programming assistant branding.
54+
- role: user
55+
content: Use the view tool to look at the file test.png and describe what you see
56+
- role: assistant
57+
tool_calls:
58+
- id: toolcall_2
59+
type: function
60+
function:
61+
name: report_intent
62+
arguments: '{"intent":"Viewing image file"}'
63+
- role: assistant
64+
tool_calls:
65+
- id: toolcall_3
66+
type: function
67+
function:
68+
name: view
69+
arguments: '{"path":"${workdir}/test.png"}'
70+
- messages:
71+
- role: system
72+
content: ${system}
73+
- role: user
74+
content: Use the view tool to look at the file test.png and describe what you see
75+
- role: assistant
76+
content: I'll view the test.png file for you.
77+
tool_calls:
78+
- id: toolcall_0
79+
type: function
80+
function:
81+
name: report_intent
82+
arguments: '{"intent":"Viewing image file"}'
83+
- id: toolcall_1
84+
type: function
85+
function:
86+
name: view
87+
arguments: '{"path":"${workdir}/test.png"}'
88+
- role: tool
89+
tool_call_id: toolcall_0
90+
content: Intent logged
91+
- role: tool
92+
tool_call_id: toolcall_1
93+
content: Viewed image file successfully.
94+
- role: assistant
95+
content: The image shows the **GitHub Copilot logo** - a stylized hexagonal icon in shades of blue and purple with a
96+
distinctive geometric design. The logo features a three-dimensional hexagon shape with gradient colors
97+
transitioning from deep blue to lighter blue/purple tones, creating a modern, tech-forward aesthetic. This is
98+
GitHub's AI pair programming assistant branding.
99+
- role: user
100+
content: Use the view tool to look at the file test.png and describe what you see
101+
- role: assistant
102+
tool_calls:
103+
- id: toolcall_2
104+
type: function
105+
function:
106+
name: report_intent
107+
arguments: '{"intent":"Viewing image file"}'
108+
- id: toolcall_3
109+
type: function
110+
function:
111+
name: view
112+
arguments: '{"path":"${workdir}/test.png"}'
113+
- role: tool
114+
tool_call_id: toolcall_2
115+
content: Intent logged
116+
- role: tool
117+
tool_call_id: toolcall_3
118+
content: Viewed image file successfully.
119+
- role: user
120+
- role: assistant
121+
content: >-
122+
Looking at the image from test.png, I can see what appears to be a **very light, nearly blank image** with a
123+
pale yellowish or off-white color. The image is quite minimal - it looks like either:
124+
125+
126+
- A blank/empty canvas with a subtle cream or pale yellow tint
127+
128+
- A very light gradient or texture that's barely visible
129+
130+
- Possibly a placeholder or background image
131+
132+
133+
There's no distinct text, shapes, logos, or other clear visual elements visible in this image. It's
134+
essentially a solid light color fill, possibly with very subtle variations in tone.

0 commit comments

Comments
 (0)