diff --git a/src/api/providers/__tests__/lite-llm.spec.ts b/src/api/providers/__tests__/lite-llm.spec.ts index a95118469e2..516ae5ea335 100644 --- a/src/api/providers/__tests__/lite-llm.spec.ts +++ b/src/api/providers/__tests__/lite-llm.spec.ts @@ -40,6 +40,11 @@ vi.mock("../fetchers/modelCache", () => ({ "claude-3-opus": { ...litellmDefaultModelInfo, maxTokens: 8192 }, "llama-3": { ...litellmDefaultModelInfo, maxTokens: 8192 }, "gpt-4-turbo": { ...litellmDefaultModelInfo, maxTokens: 8192 }, + "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0": { + ...litellmDefaultModelInfo, + maxTokens: 8192, + supportsNativeTools: true, + }, }) }), getModelsFromCache: vi.fn().mockReturnValue(undefined), @@ -388,4 +393,124 @@ describe("LiteLLMHandler", () => { expect(createCall.max_completion_tokens).toBeUndefined() }) }) + + describe("parallel_tool_calls handling", () => { + it("should omit parallel_tool_calls parameter for all models when using native tools", async () => { + const testModels = [ + "gpt-4", + "claude-3-opus", + "bedrock/anthropic.claude-3-5-sonnet-20241022-v2:0", + "gpt-4-turbo", + ] + + for (const modelId of testModels) { + vi.clearAllMocks() + + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: modelId, + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + parallelToolCalls: true, + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls is NOT included for any model + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBeUndefined() + expect(createCall.tools).toBeDefined() // Tools should still be present + } + }) + + it("should omit parallel_tool_calls even when parallelToolCalls is not specified in metadata", async () => { + const options: ApiHandlerOptions = { + ...mockOptions, + litellmModelId: "gpt-4", + } + handler = new LiteLLMHandler(options) + + const systemPrompt = "You are a helpful assistant" + const messages: Anthropic.Messages.MessageParam[] = [{ role: "user", content: "Test" }] + + // Mock the stream response + const mockStream = { + async *[Symbol.asyncIterator]() { + yield { + choices: [{ delta: { content: "Response" } }], + usage: { + prompt_tokens: 10, + completion_tokens: 5, + }, + } + }, + } + + mockCreate.mockReturnValue({ + withResponse: vi.fn().mockResolvedValue({ data: mockStream }), + }) + + const metadata = { + taskId: "test-task", + tools: [ + { + type: "function" as const, + function: { + name: "test_tool", + description: "A test tool", + parameters: { type: "object", properties: {} }, + }, + }, + ], + toolProtocol: "native" as const, + // parallelToolCalls not specified + } + + const generator = handler.createMessage(systemPrompt, messages, metadata) + for await (const chunk of generator) { + // Consume the generator + } + + // Verify that parallel_tool_calls is still not included + const createCall = mockCreate.mock.calls[0][0] + expect(createCall.parallel_tool_calls).toBeUndefined() + }) + }) }) diff --git a/src/api/providers/lite-llm.ts b/src/api/providers/lite-llm.ts index 9acbc35d074..bbdef2d124d 100644 --- a/src/api/providers/lite-llm.ts +++ b/src/api/providers/lite-llm.ts @@ -133,7 +133,8 @@ export class LiteLLMHandler extends RouterProvider implements SingleCompletionHa }, ...(useNativeTools && { tools: this.convertToolsForOpenAI(metadata.tools) }), ...(useNativeTools && metadata.tool_choice && { tool_choice: metadata.tool_choice }), - ...(useNativeTools && { parallel_tool_calls: metadata?.parallelToolCalls ?? false }), + // Omit parallel_tool_calls parameter - not supported by all LiteLLM backends (e.g. Bedrock) + // We handle parallel tool call restrictions via system prompts instead } // GPT-5 models require max_completion_tokens instead of the deprecated max_tokens parameter