diff --git a/docs/configuration/providers/ollama.md b/docs/configuration/providers/ollama.md index ffdbb0c5..d193405f 100644 --- a/docs/configuration/providers/ollama.md +++ b/docs/configuration/providers/ollama.md @@ -34,6 +34,15 @@ By default, Ollama uses a 2048 token context window which is too small for agent OLLAMA_NUM_CTX=32768 ollama serve ``` +Nanocoder also forwards its resolved context limit to Ollama as `options.num_ctx` on each request. That means these settings flow through automatically: + +- `/context-max` or `--context-max` +- provider-level `contextWindow` +- per-model `contextWindows` +- subagent `contextWindow` + +Use the Ollama server default (`OLLAMA_NUM_CTX`) as your baseline ceiling, then tighten individual Nanocoder sessions or models with the settings above when you want smaller/faster contexts. + Or set it permanently in your environment. ### Signs of an Insufficient Context Limit diff --git a/docs/features/subagents.md b/docs/features/subagents.md index 61503a75..960a28da 100644 --- a/docs/features/subagents.md +++ b/docs/features/subagents.md @@ -125,7 +125,7 @@ Always use your tools — never guess. The `provider` must match a provider name configured in your `agents.config.json`. -If you set `contextWindow`, Nanocoder creates that subagent with its own context limit override. This is useful when a lightweight research agent should run with a smaller local-model context than your main coding agent. +If you set `contextWindow`, Nanocoder creates that subagent with its own context limit override. This is useful when a lightweight research agent should run with a smaller local-model context than your main coding agent. For Ollama providers, this override is also forwarded to the API request as `options.num_ctx`, so the smaller limit affects the actual model runtime too. ## Priority and Overrides diff --git a/source/ai-sdk-client/chat/chat-handler.ts b/source/ai-sdk-client/chat/chat-handler.ts index 8ff13d4e..474aef2b 100644 --- a/source/ai-sdk-client/chat/chat-handler.ts +++ b/source/ai-sdk-client/chat/chat-handler.ts @@ -159,8 +159,9 @@ export async function handleChat( // OpenRouter provider routing / reasoning / transforms / fallback // models). buildProviderOptions returns undefined when nothing // applies, so the SDK call site doesn't see an empty object. - const providerOptions = buildProviderOptions( + const providerOptions = await buildProviderOptions( providerConfig, + currentModel, systemContent, modeOverrides?.modelParameters, ); diff --git a/source/ai-sdk-client/chat/provider-options.spec.ts b/source/ai-sdk-client/chat/provider-options.spec.ts index 383eed26..151dd551 100644 --- a/source/ai-sdk-client/chat/provider-options.spec.ts +++ b/source/ai-sdk-client/chat/provider-options.spec.ts @@ -24,25 +24,40 @@ function withOpenRouter( return makeProvider({openrouter, ...overrides}); } -test('returns undefined when no provider-specific options apply', t => { - const result = buildProviderOptions( - makeProvider({name: 'Ollama', type: 'ollama'}), +test('returns Ollama num_ctx provider options when a context limit is resolved', async t => { + const result = await buildProviderOptions( + makeProvider({ + name: 'Ollama', + config: {baseURL: 'http://localhost:11434/v1', apiKey: 'test-key'}, + contextWindow: 32768, + }), + 'llama3.1', 'system prompt', undefined, ); - t.is(result, undefined); + t.deepEqual(result, { + Ollama: { + options: { + num_ctx: 32768, + }, + }, + }); }); -test('returns undefined for OpenRouter with no openrouter config and no reasoning shortcut', t => { - const result = buildProviderOptions(makeProvider(), 'system prompt', { - temperature: 0.7, - }); +test('returns undefined for OpenRouter with no openrouter config and no reasoning shortcut', async t => { + const result = await buildProviderOptions( + makeProvider(), + 'x-ai/grok-4', + 'system prompt', + {temperature: 0.7}, + ); t.is(result, undefined); }); -test('chatgpt-codex always returns providerOptions.openai with defaults', t => { - const result = buildProviderOptions( +test('chatgpt-codex always returns providerOptions.openai with defaults', async t => { + const result = await buildProviderOptions( makeProvider({name: 'codex', sdkProvider: 'chatgpt-codex'}), + 'gpt-5', 'hello system', undefined, ); @@ -56,9 +71,10 @@ test('chatgpt-codex always returns providerOptions.openai with defaults', t => { }); }); -test('chatgpt-codex honours overridden reasoningEffort and reasoningSummary', t => { - const result = buildProviderOptions( +test('chatgpt-codex honours overridden reasoningEffort and reasoningSummary', async t => { + const result = await buildProviderOptions( makeProvider({name: 'codex', sdkProvider: 'chatgpt-codex'}), + 'gpt-5', 'hello', {reasoningEffort: 'high', reasoningSummary: 'detailed'}, ); @@ -72,7 +88,7 @@ test('chatgpt-codex honours overridden reasoningEffort and reasoningSummary', t }); }); -test('OpenRouter forwards provider routing block', t => { +test('OpenRouter forwards provider routing block', async t => { const provider = withOpenRouter({ provider: { order: ['Anthropic', 'OpenAI'], @@ -80,7 +96,7 @@ test('OpenRouter forwards provider routing block', t => { sort: 'price', }, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { provider: { @@ -92,11 +108,11 @@ test('OpenRouter forwards provider routing block', t => { }); }); -test('OpenRouter accepts object-form sort for cross-model partitioning', t => { +test('OpenRouter accepts object-form sort for cross-model partitioning', async t => { const provider = withOpenRouter({ provider: {sort: {by: 'price', partition: 'model'}}, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { provider: {sort: {by: 'price', partition: 'model'}}, @@ -104,12 +120,12 @@ test('OpenRouter accepts object-form sort for cross-model partitioning', t => { }); }); -test('OpenRouter forwards plugins and fallback models list', t => { +test('OpenRouter forwards plugins and fallback models list', async t => { const provider = withOpenRouter({ plugins: [{id: 'context-compression', engine: 'middle-out'}], models: ['anthropic/claude-3.5-sonnet', 'openai/gpt-4o'], }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { plugins: [{id: 'context-compression', engine: 'middle-out'}], @@ -118,31 +134,31 @@ test('OpenRouter forwards plugins and fallback models list', t => { }); }); -test('OpenRouter forwards service_tier=flex', t => { +test('OpenRouter forwards service_tier=flex', async t => { const provider = withOpenRouter({service_tier: 'flex'}); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, {openrouter: {service_tier: 'flex'}}); }); -test('OpenRouter forwards service_tier=priority', t => { +test('OpenRouter forwards service_tier=priority', async t => { const provider = withOpenRouter({service_tier: 'priority'}); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, {openrouter: {service_tier: 'priority'}}); }); -test('OpenRouter forwards route and user', t => { +test('OpenRouter forwards route and user', async t => { const provider = withOpenRouter({route: 'fallback', user: 'user-123'}); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: {route: 'fallback', user: 'user-123'}, }); }); -test('OpenRouter forwards extended reasoning block with xhigh and exclude', t => { +test('OpenRouter forwards extended reasoning block with xhigh and exclude', async t => { const provider = withOpenRouter({ reasoning: {effort: 'xhigh', max_tokens: 12000, exclude: false}, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { reasoning: {effort: 'xhigh', max_tokens: 12000, exclude: false}, @@ -150,25 +166,25 @@ test('OpenRouter forwards extended reasoning block with xhigh and exclude', t => }); }); -test('OpenRouter maps tune-level reasoningEffort to reasoning.effort', t => { - const result = buildProviderOptions(makeProvider(), '', { +test('OpenRouter maps tune-level reasoningEffort to reasoning.effort', async t => { + const result = await buildProviderOptions(makeProvider(), 'x-ai/grok-4', '', { reasoningEffort: 'high', }); t.deepEqual(result, {openrouter: {reasoning: {effort: 'high'}}}); }); -test('OpenRouter passes minimal reasoningEffort straight through', t => { - const result = buildProviderOptions(makeProvider(), '', { +test('OpenRouter passes minimal reasoningEffort straight through', async t => { + const result = await buildProviderOptions(makeProvider(), 'x-ai/grok-4', '', { reasoningEffort: 'minimal', }); t.deepEqual(result, {openrouter: {reasoning: {effort: 'minimal'}}}); }); -test('OpenRouter merges tune shortcut effort with provider-config reasoning block', t => { +test('OpenRouter merges tune shortcut effort with provider-config reasoning block', async t => { const provider = withOpenRouter({ reasoning: {max_tokens: 8000, exclude: true}, }); - const result = buildProviderOptions(provider, '', { + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', { reasoningEffort: 'medium', }); t.deepEqual(result, { @@ -178,15 +194,15 @@ test('OpenRouter merges tune shortcut effort with provider-config reasoning bloc }); }); -test('provider-config reasoning.effort wins over tune shortcut', t => { +test('provider-config reasoning.effort wins over tune shortcut', async t => { const provider = withOpenRouter({reasoning: {effort: 'high'}}); - const result = buildProviderOptions(provider, '', { + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', { reasoningEffort: 'low', }); t.deepEqual(result, {openrouter: {reasoning: {effort: 'high'}}}); }); -test('OpenRouter routing extras (zdr, max_price, latency thresholds) flow through', t => { +test('OpenRouter routing extras (zdr, max_price, latency thresholds) flow through', async t => { const provider = withOpenRouter({ provider: { zdr: true, @@ -196,7 +212,7 @@ test('OpenRouter routing extras (zdr, max_price, latency thresholds) flow throug preferred_max_latency: 2000, }, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { provider: { @@ -210,23 +226,23 @@ test('OpenRouter routing extras (zdr, max_price, latency thresholds) flow throug }); }); -test('OpenRouter detection is case-insensitive on provider name', t => { +test('OpenRouter detection is case-insensitive on provider name', async t => { const provider = withOpenRouter( {service_tier: 'flex'}, {name: 'openrouter'}, ); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, {openrouter: {service_tier: 'flex'}}); }); -test('OpenRouter combines provider, reasoning, plugins, models, service_tier', t => { +test('OpenRouter combines provider, reasoning, plugins, models, service_tier', async t => { const provider = withOpenRouter({ provider: {sort: 'throughput'}, plugins: [{id: 'web'}], models: ['openai/gpt-4o'], service_tier: 'flex', }); - const result = buildProviderOptions(provider, '', {reasoningEffort: 'high'}); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', {reasoningEffort: 'high'}); t.deepEqual(result, { openrouter: { provider: {sort: 'throughput'}, @@ -238,18 +254,18 @@ test('OpenRouter combines provider, reasoning, plugins, models, service_tier', t }); }); -test('OpenRouter ignores chatgpt-codex-only fields without dropping requests', t => { - const result = buildProviderOptions(makeProvider(), '', { +test('OpenRouter ignores chatgpt-codex-only fields without dropping requests', async t => { + const result = await buildProviderOptions(makeProvider(), 'x-ai/grok-4', '', { reasoningSummary: 'detailed', }); t.is(result, undefined); }); -test('OpenRouter forwards extraBody pass-through fields', t => { +test('OpenRouter forwards extraBody pass-through fields', async t => { const provider = withOpenRouter({ extraBody: {usage: {include: true}, debug: {echo_upstream_body: true}}, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: { usage: {include: true}, @@ -258,21 +274,21 @@ test('OpenRouter forwards extraBody pass-through fields', t => { }); }); -test('OpenRouter extraBody is overridden by typed fields on key collision', t => { +test('OpenRouter extraBody is overridden by typed fields on key collision', async t => { const provider = withOpenRouter({ service_tier: 'flex', extraBody: {service_tier: 'priority', custom: 'kept'}, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, { openrouter: {service_tier: 'flex', custom: 'kept'}, }); }); -test('OpenRouter extraBody alone is enough to emit providerOptions', t => { +test('OpenRouter extraBody alone is enough to emit providerOptions', async t => { const provider = withOpenRouter({ extraBody: {experimental_flag: true}, }); - const result = buildProviderOptions(provider, '', undefined); + const result = await buildProviderOptions(provider, 'x-ai/grok-4', '', undefined); t.deepEqual(result, {openrouter: {experimental_flag: true}}); }); diff --git a/source/ai-sdk-client/chat/provider-options.ts b/source/ai-sdk-client/chat/provider-options.ts index bae87a41..08984ac3 100644 --- a/source/ai-sdk-client/chat/provider-options.ts +++ b/source/ai-sdk-client/chat/provider-options.ts @@ -1,3 +1,4 @@ +import {getModelContextLimit} from '@/models/models-dev-client.js'; import type {AIProviderConfig, ModelParameters} from '@/types/index'; import {isOpenRouterProvider} from '../providers/openrouter.js'; @@ -33,11 +34,12 @@ export type ProviderOptions = Record>; * Returns `undefined` when no provider-specific options apply, so the SDK * call site can spread it without producing an empty `providerOptions: {}`. */ -export function buildProviderOptions( +export async function buildProviderOptions( providerConfig: AIProviderConfig, + currentModel: string, systemContent: string, modelParameters: ModelParameters | undefined, -): ProviderOptions | undefined { +): Promise { if (providerConfig.sdkProvider === 'chatgpt-codex') { return { openai: { @@ -86,9 +88,32 @@ export function buildProviderOptions( return {openrouter: payload}; } + if (isOllamaProvider(providerConfig)) { + const contextLimit = await getModelContextLimit(currentModel, { + providerConfig, + }); + + if (contextLimit && contextLimit > 0) { + return { + [providerConfig.name]: { + options: { + num_ctx: contextLimit, + }, + }, + }; + } + } + return undefined; } +function isOllamaProvider(providerConfig: AIProviderConfig): boolean { + const providerName = providerConfig.name.toLowerCase(); + const baseURL = providerConfig.config.baseURL?.toLowerCase() ?? ''; + + return providerName === 'ollama' || baseURL.includes('11434'); +} + /** * Resolve OpenRouter's `reasoning` body field by merging the top-level * `reasoningEffort` shortcut (from ModelParameters / `/tune`) with the more