diff --git a/.changeset/fix-bedrock-thinking-generate-object.md b/.changeset/fix-bedrock-thinking-generate-object.md new file mode 100644 index 00000000000..04143c2fd7b --- /dev/null +++ b/.changeset/fix-bedrock-thinking-generate-object.md @@ -0,0 +1,5 @@ +--- +"@effect/ai-amazon-bedrock": patch +--- + +`generateObject` no longer fails when extended thinking is configured via `withConfigOverride`. Anthropic's API rejects requests that set `thinking` in `additionalModelRequestFields` alongside a forced `toolChoice` — which `generateObject` always does. The fix strips `thinking` from `additionalModelRequestFields` in the `json` response format path before the request is sent. diff --git a/packages/ai/amazon-bedrock/src/AmazonBedrockLanguageModel.ts b/packages/ai/amazon-bedrock/src/AmazonBedrockLanguageModel.ts index dbcadd08643..729c05a76fd 100644 --- a/packages/ai/amazon-bedrock/src/AmazonBedrockLanguageModel.ts +++ b/packages/ai/amazon-bedrock/src/AmazonBedrockLanguageModel.ts @@ -206,6 +206,26 @@ export const make = Effect.fnUntraced(function*(options: { const { messages, system } = yield* prepareMessages(providerOptions) const { additionalTools, betas, toolConfig } = yield* prepareTools(providerOptions, config) const responseFormat = providerOptions.responseFormat + + // Anthropic rejects requests that combine extended thinking with forced tool use. + // generateObject always forces toolChoice, so strip "thinking" in the json path. + // Return an explicit object (even if empty) when fields are present so the spread + // overrides the thinking config already placed in the request by ...config above. + const requestAdditionalFields: Record | undefined = responseFormat.type === "json" + ? (Predicate.isNotUndefined(config.additionalModelRequestFields) || + Predicate.isNotUndefined(additionalTools)) + ? (() => { + const { thinking: _thinking, ...rest } = { + ...config.additionalModelRequestFields, + ...additionalTools + } + return rest + })() + : undefined + : Predicate.isNotUndefined(additionalTools) + ? { ...config.additionalModelRequestFields, ...additionalTools } + : undefined + const request: typeof ConverseRequest.Encoded = { ...config, system, @@ -231,13 +251,8 @@ export const make = Effect.fnUntraced(function*(options: { ? { toolConfig } : {}), // Handle additional model request fields - ...(Predicate.isNotUndefined(additionalTools) - ? { - additionalModelRequestFields: { - ...config.additionalModelRequestFields, - ...additionalTools - } - } + ...(Predicate.isNotUndefined(requestAdditionalFields) + ? { additionalModelRequestFields: requestAdditionalFields } : {}) } return { betas, request } diff --git a/packages/ai/amazon-bedrock/test/AmazonBedrockLanguageModel.test.ts b/packages/ai/amazon-bedrock/test/AmazonBedrockLanguageModel.test.ts new file mode 100644 index 00000000000..f6026113efa --- /dev/null +++ b/packages/ai/amazon-bedrock/test/AmazonBedrockLanguageModel.test.ts @@ -0,0 +1,175 @@ +import * as LanguageModel from "@effect/ai/LanguageModel" +import { assert, describe, it } from "@effect/vitest" +import * as Effect from "effect/Effect" +import * as Layer from "effect/Layer" +import * as Schema from "effect/Schema" +import { AmazonBedrockClient } from "../src/AmazonBedrockClient.js" +import * as AmazonBedrockLanguageModel from "../src/AmazonBedrockLanguageModel.js" +import { ConverseResponse } from "../src/AmazonBedrockSchema.js" +import type { ConverseRequest } from "../src/AmazonBedrockSchema.js" + +// --------------------------------------------------------------------------- +// Schema decoders +// --------------------------------------------------------------------------- + +const decodeConverseResponse = Schema.decodeUnknownSync(ConverseResponse) + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +const TestModel = "us.anthropic.claude-3-5-sonnet-20241022-v2:0" as const +const TestObjectName = "MockResult" +const MockResultSchema = Schema.Struct({ field: Schema.String }) + +const textResponse = () => + decodeConverseResponse({ + output: { + message: { + role: "assistant", + content: [{ text: "hello" }] + } + }, + metrics: { latencyMs: 100 }, + stopReason: "end_turn", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } + }) + +const toolUseResponse = (name: string) => + decodeConverseResponse({ + output: { + message: { + role: "assistant", + content: [{ + toolUse: { + toolUseId: "test-id", + name, + input: { field: "value" } + } + }] + } + }, + metrics: { latencyMs: 100 }, + stopReason: "tool_use", + usage: { inputTokens: 10, outputTokens: 5, totalTokens: 15 } + }) + +const makeCapturingLayer = ( + captured: Array, + response: ConverseResponse +) => + AmazonBedrockLanguageModel.layer({ model: TestModel }).pipe( + Layer.provide( + Layer.succeed(AmazonBedrockClient, { + client: null as any, + streamRequest: null as any, + converse: (opts) => + Effect.sync(() => { + captured.push(opts.payload) + return response + }), + converseStream: null as any + }) + ) + ) + +// --------------------------------------------------------------------------- +// makeRequest — request construction tests +// --------------------------------------------------------------------------- + +describe("AmazonBedrockLanguageModel", () => { + describe("makeRequest / additionalModelRequestFields", () => { + it.effect("strips thinking when using generateObject (forced toolChoice.tool)", () => + Effect.gen(function*() { + const captured: Array = [] + + yield* LanguageModel.generateObject({ + prompt: [], + schema: MockResultSchema, + objectName: TestObjectName + }).pipe( + AmazonBedrockLanguageModel.withConfigOverride({ + additionalModelRequestFields: { + thinking: { type: "enabled", budget_tokens: 5000 } + } + }), + Effect.provide(makeCapturingLayer(captured, toolUseResponse(TestObjectName))) + ) + + assert.strictEqual(captured.length, 1) + const req = captured[0] + + // Must force tool use for generateObject + assert.deepStrictEqual(req.toolConfig?.toolChoice, { tool: { name: TestObjectName } }) + + // Anthropic rejects thinking + forced tool use — must be stripped + assert.isUndefined(req.additionalModelRequestFields?.["thinking"]) + })) + + it.effect("preserves thinking when using generateText (no forced toolChoice)", () => + Effect.gen(function*() { + const captured: Array = [] + + yield* LanguageModel.generateText({ prompt: [] }).pipe( + AmazonBedrockLanguageModel.withConfigOverride({ + additionalModelRequestFields: { + thinking: { type: "enabled", budget_tokens: 5000 } + } + }), + Effect.provide(makeCapturingLayer(captured, textResponse())) + ) + + assert.strictEqual(captured.length, 1) + const req = captured[0] + + // thinking must flow through for non-forced-tool-use requests + assert.deepStrictEqual(req.additionalModelRequestFields?.["thinking"], { + type: "enabled", + budget_tokens: 5000 + }) + })) + + it.effect("does not set additionalModelRequestFields when none configured and using generateObject", () => + Effect.gen(function*() { + const captured: Array = [] + + yield* LanguageModel.generateObject({ + prompt: [], + schema: MockResultSchema, + objectName: TestObjectName + }).pipe( + Effect.provide(makeCapturingLayer(captured, toolUseResponse(TestObjectName))) + ) + + assert.strictEqual(captured.length, 1) + // No additionalModelRequestFields should be injected when none was configured + assert.isUndefined(captured[0].additionalModelRequestFields) + })) + + it.effect("preserves non-thinking fields in additionalModelRequestFields with generateObject", () => + Effect.gen(function*() { + const captured: Array = [] + + yield* LanguageModel.generateObject({ + prompt: [], + schema: MockResultSchema, + objectName: TestObjectName + }).pipe( + AmazonBedrockLanguageModel.withConfigOverride({ + additionalModelRequestFields: { + thinking: { type: "enabled", budget_tokens: 5000 }, + someOtherField: "preserved" + } + }), + Effect.provide(makeCapturingLayer(captured, toolUseResponse(TestObjectName))) + ) + + assert.strictEqual(captured.length, 1) + const req = captured[0] + + // thinking stripped, other fields survive + assert.isUndefined(req.additionalModelRequestFields?.["thinking"]) + assert.strictEqual(req.additionalModelRequestFields?.["someOtherField"], "preserved") + })) + }) +})