diff --git a/packages/types/src/provider-settings.ts b/packages/types/src/provider-settings.ts index 443cbafcf16..22c5cb81ffb 100644 --- a/packages/types/src/provider-settings.ts +++ b/packages/types/src/provider-settings.ts @@ -374,6 +374,7 @@ const litellmSchema = baseProviderSettingsSchema.extend({ litellmApiKey: z.string().optional(), litellmModelId: z.string().optional(), litellmUsePromptCache: z.boolean().optional(), + litellmUseAzureBedrock: z.boolean().optional(), }) const cerebrasSchema = apiModelIdProviderModelSchema.extend({ diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index a95118469e2..7d7c945c83b 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -40,6 +40,17 @@ vi.mock("../fetchers/modelCache", () => ({ "claude-3-opus": { ...litellmDefaultModelInfo, maxTokens: 8192 }, "llama-3": { ...litellmDefaultModelInfo, maxTokens: 8192 }, "gpt-4-turbo": { ...litellmDefaultModelInfo, maxTokens: 8192 }, + "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0": { + ...litellmDefaultModelInfo, + maxTokens: 8192, + supportsNativeTools: true, + }, + "anthropic.claude-sonnet-4-20250514-v1:0": { + ...litellmDefaultModelInfo, + maxTokens: 8192, + supportsNativeTools: true, + }, + "amazon.titan-text-express-v1": { ...litellmDefaultModelInfo, maxTokens: 8192, supportsNativeTools: true }, }) }), getModelsFromCache: vi.fn().mockReturnValue(undefined), @@ -388,4 +399,294 @@ describe("LiteLLMHandler", () => { expect(createCall.max_completion_tokens).toBeUndefined() }) }) + + describe("Bedrock model handling", () => { + it("should exclude parallel_tool_calls when litellmUseAzureBedrock is explicitly true", async () => { + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "gpt-4", // Non-Bedrock model ID + litellmUseAzureBedrock: true, // Explicitly set to Bedrock + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + parallelToolCalls: true, + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls is NOT included when litellmUseAzureBedrock is true + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBeUndefined() + expect(createCall.tools).toBeDefined() + }) + + it("should include parallel_tool_calls when litellmUseAzureBedrock is explicitly false", async () => { + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", // Bedrock model ID + litellmUseAzureBedrock: false, // Explicitly set to NOT Bedrock + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + parallelToolCalls: true, + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls IS included when litellmUseAzureBedrock is false + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBe(true) + expect(createCall.tools).toBeDefined() + }) + + it("should auto-detect and exclude parallel_tool_calls for Bedrock models when litellmUseAzureBedrock is not set", async () => { + const bedrockModels = ["bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", "amazon.titan-text-express-v1"] + + for (const modelId of bedrockModels) { + vi.clearAllMocks() + + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: modelId, + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + parallelToolCalls: true, + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls is NOT included for Bedrock models + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBeUndefined() + expect(createCall.tools).toBeDefined() // Tools should still be present + } + }) + + it("should auto-detect and include parallel_tool_calls for non-Bedrock models when litellmUseAzureBedrock is not set", async () => { + const nonBedrockModels = [ + "gpt-4", + "claude-3-opus", + "gpt-4-turbo", + "anthropic.claude-sonnet-4-20250514-v1:0", + ] + + for (const modelId of nonBedrockModels) { + vi.clearAllMocks() + + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: modelId, + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + parallelToolCalls: true, + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls IS included for non-Bedrock models + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBe(true) + expect(createCall.tools).toBeDefined() + } + }) + + it("should default parallel_tool_calls to false for non-Bedrock models when not specified", async () => { + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "gpt-4", + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + // parallelToolCalls not specified + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls defaults to false + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBe(false) + }) + }) }) diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index 9acbc35d074..6bf79c29d64 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -38,6 +38,30 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa return /\bgpt-?5(?!\d)/i.test(modelId) } + /** + * Check if the model is routed through AWS Bedrock + * Bedrock doesn't support the parallel_tool_calls parameter + * + * If the user has explicitly set litellmUseAzureBedrock, use that setting. + * Otherwise, fall back to auto-detection based on model ID patterns. + * Note: We exclude 'anthropic.' prefix as it can match direct Anthropic API access through LiteLLM + */ + private isBedrockModel(modelId: string): boolean { + // User-specified option takes precedence + if (this.options.litellmUseAzureBedrock !== undefined) { + return this.options.litellmUseAzureBedrock + } + + // Fall back to auto-detection + const lowerModel = modelId.toLowerCase() + return ( + lowerModel.includes("bedrock") || + lowerModel.includes("amazon.") || + // Match AWS Bedrock model ID patterns (excluding anthropic to avoid false positives) + /^(amazon|ai21|cohere|meta|mistral)\./.test(lowerModel) + ) + } + override async *createMessage( systemPrompt: string, messages: Anthropic.Messages.MessageParam[], @@ -133,7 +157,9 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa }, ...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }), ...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }), - ...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }), + // Bedrock doesn't support parallel_tool_calls parameter, so exclude it for Bedrock models + ...(useNativeTools && + !this.isBedrockModel(modelId) && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }), } // GPT-5 models require max_completion_tokens instead of the deprecated max_tokens parameter diff --git a/webview-ui/src/components/settings/providers/LiteLLM.tsx b/webview-ui/src/components/settings/providers/LiteLLM.tsx index 0b89b671ce9..e08b3c5cb1e 100644 --- a/webview-ui/src/components/settings/providers/LiteLLM.tsx +++ b/webview-ui/src/components/settings/providers/LiteLLM.tsx @@ -156,6 +156,20 @@ export const LiteLLM = ({ simplifySettings={simplifySettings} /> + {/* Bedrock backend option */} +
+ { + setApiConfigurationField("litellmUseAzureBedrock", e.target.checked) + }}> + {t("settings:providers.litellmUseAzureBedrock")} + +
+ {t("settings:providers.litellmUseAzureBedrockDescription")} +
+
+ {/* Show prompt caching option if the selected model supports it */} {(() => { const selectedModelId = apiConfiguration.litellmModelId || litellmDefaultModelId diff --git a/webview-ui/src/i18n/locales/en/settings.json b/webview-ui/src/i18n/locales/en/settings.json index 2396bded682..66ae4201ad8 100644 --- a/webview-ui/src/i18n/locales/en/settings.json +++ b/webview-ui/src/i18n/locales/en/settings.json @@ -370,6 +370,8 @@ "getXaiApiKey": "Get xAI API Key", "litellmApiKey": "LiteLLM API Key", "litellmBaseUrl": "LiteLLM Base URL", + "litellmUseAzureBedrock": "Backend is AWS Bedrock", + "litellmUseAzureBedrockDescription": "Enable this if your LiteLLM proxy routes to AWS Bedrock models. This ensures Bedrock-incompatible parameters are excluded from requests.", "awsCredentials": "AWS Credentials", "awsProfile": "AWS Profile", "awsApiKey": "Amazon Bedrock API Key",