Skip to content

Commit cb407cc

Browse files
committed
fix: preserve Anthropic thinking blocks and signatures in LiteLLM round-trip
When using Claude models through LiteLLM, extended thinking blocks (with signatures) were lost after the first turn because: 1. _extract_reasoning_value() only read reasoning_content (flattened string without signatures), ignoring thinking_blocks 2. _content_to_message_param() set reasoning_content on the outgoing message, which LiteLLM's anthropic_messages_pt() template silently drops This fix: - Adds _is_anthropic_provider() helper to detect anthropic/bedrock/ vertex_ai providers - Updates _extract_reasoning_value() to prefer thinking_blocks (with per-block signatures) over reasoning_content - Updates _convert_reasoning_value_to_parts() to handle ChatCompletionThinkingBlock dicts, preserving thought_signature - Updates _content_to_message_param() to embed thinking blocks directly in the message content list for Anthropic providers, bypassing the broken reasoning_content path Fixes #4801
1 parent d62558c commit cb407cc

2 files changed

Lines changed: 131 additions & 60 deletions

File tree

src/google/adk/models/lite_llm.py

Lines changed: 69 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -233,6 +233,16 @@ def _get_provider_from_model(model: str) -> str:
233233
return ""
234234

235235

236+
# Providers that route to Anthropic's API and require thinking blocks
237+
# embedded directly in the message content list.
238+
_ANTHROPIC_PROVIDERS = frozenset({"anthropic", "bedrock", "vertex_ai"})
239+
240+
241+
def _is_anthropic_provider(provider: str) -> bool:
242+
"""Returns True if the provider routes to an Anthropic model endpoint."""
243+
return provider.lower() in _ANTHROPIC_PROVIDERS if provider else False
244+
245+
236246
# Default MIME type when none can be inferred
237247
_DEFAULT_MIME_TYPE = "application/octet-stream"
238248

@@ -399,26 +409,34 @@ def _is_thinking_blocks_format(reasoning_value: Any) -> bool:
399409
def _convert_reasoning_value_to_parts(reasoning_value: Any) -> List[types.Part]:
400410
"""Converts provider reasoning payloads into Gemini thought parts.
401411
402-
Handles Anthropic thinking_blocks (list of dicts with type/thinking/signature)
403-
by preserving the signature on each part's thought_signature field. This is
404-
required for Anthropic to maintain thinking across tool call boundaries.
412+
Handles two formats:
413+
- Anthropic thinking_blocks with 'thinking' and optional 'signature' fields.
414+
- A plain string or nested structure (OpenAI/Azure/Ollama) via
415+
_iter_reasoning_texts.
405416
"""
406-
if _is_thinking_blocks_format(reasoning_value):
417+
if isinstance(reasoning_value, list):
407418
parts: List[types.Part] = []
408419
for block in reasoning_value:
409-
if not isinstance(block, dict):
410-
continue
411-
block_type = block.get("type", "")
412-
if block_type == "redacted":
413-
continue
414-
thinking_text = block.get("thinking", "")
415-
signature = block.get("signature", "")
416-
if not thinking_text:
417-
continue
418-
part = types.Part(text=thinking_text, thought=True)
419-
if signature:
420-
part.thought_signature = signature.encode("utf-8")
421-
parts.append(part)
420+
if isinstance(block, dict):
421+
block_type = block.get("type", "")
422+
if block_type == "redacted":
423+
continue
424+
if block_type == "thinking":
425+
thinking_text = block.get("thinking", "")
426+
if thinking_text:
427+
part = types.Part(text=thinking_text, thought=True)
428+
signature = block.get("signature")
429+
if signature:
430+
decoded_signature = _decode_thought_signature(signature)
431+
part.thought_signature = (
432+
decoded_signature or str(signature).encode("utf-8")
433+
)
434+
parts.append(part)
435+
continue
436+
# Fall back to text extraction for non-thinking-block items.
437+
for text in _iter_reasoning_texts(block):
438+
if text:
439+
parts.append(types.Part(text=text, thought=True))
422440
return parts
423441
return [
424442
types.Part(text=text, thought=True)
@@ -430,16 +448,16 @@ def _convert_reasoning_value_to_parts(reasoning_value: Any) -> List[types.Part]:
430448
def _extract_reasoning_value(message: Message | Delta | None) -> Any:
431449
"""Fetches the reasoning payload from a LiteLLM message.
432450
433-
Checks for 'thinking_blocks' (Anthropic structured format with signatures),
434-
'reasoning_content' (LiteLLM standard, used by Azure/Foundry, Ollama via
435-
LiteLLM) and 'reasoning' (used by LM Studio, vLLM).
436-
Prioritizes 'thinking_blocks' when present (Anthropic models), then
437-
'reasoning_content', then 'reasoning'.
451+
Checks for 'thinking_blocks' (Anthropic thinking with signatures),
452+
'reasoning_content' (LiteLLM standard, used by Azure/Foundry,
453+
Ollama via LiteLLM), and 'reasoning' (used by LM Studio, vLLM).
454+
Prioritizes 'thinking_blocks' when the key is present, as they contain
455+
the signature required for Anthropic's extended thinking API.
438456
"""
439457
if message is None:
440458
return None
441-
# Anthropic models return thinking_blocks with type/thinking/signature fields.
442-
# This must be preserved to maintain thinking across tool call boundaries.
459+
# Prefer thinking_blocks (Anthropic) — they carry per-block signatures
460+
# needed for multi-turn conversations with extended thinking.
443461
thinking_blocks = message.get("thinking_blocks")
444462
if thinking_blocks is not None:
445463
return thinking_blocks
@@ -912,6 +930,33 @@ async def _content_to_message_param(
912930
):
913931
reasoning_texts.append(_decode_inline_text_data(part.inline_data.data))
914932

933+
# Anthropic/Bedrock providers require thinking blocks to be embedded
934+
# directly in the message content list. LiteLLM's prompt template for
935+
# Anthropic drops the top-level reasoning_content field, so thinking
936+
# blocks disappear from multi-turn histories and the model stops
937+
# producing them after the first turn. Signatures are required by the
938+
# Anthropic API for thinking blocks in multi-turn conversations.
939+
if reasoning_parts and _is_anthropic_provider(provider):
940+
content_list = []
941+
for part in reasoning_parts:
942+
if part.text:
943+
block = {"type": "thinking", "thinking": part.text}
944+
if part.thought_signature:
945+
sig = part.thought_signature
946+
if isinstance(sig, bytes):
947+
sig = base64.b64encode(sig).decode("utf-8")
948+
block["signature"] = sig
949+
content_list.append(block)
950+
if isinstance(final_content, list):
951+
content_list.extend(final_content)
952+
elif final_content:
953+
content_list.append({"type": "text", "text": final_content})
954+
return ChatCompletionAssistantMessage(
955+
role=role,
956+
content=content_list or None,
957+
tool_calls=tool_calls or None,
958+
)
959+
915960
reasoning_content = _NEW_LINE.join(text for text in reasoning_texts if text)
916961
return ChatCompletionAssistantMessage(
917962
role=role,

tests/unittests/models/test_litellm.py

Lines changed: 62 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
from google.adk.models.lite_llm import _get_completion_inputs
3939
from google.adk.models.lite_llm import _get_content
4040
from google.adk.models.lite_llm import _get_provider_from_model
41+
from google.adk.models.lite_llm import _is_anthropic_provider
4142
from google.adk.models.lite_llm import _is_anthropic_model
4243
from google.adk.models.lite_llm import _message_to_generate_content_response
4344
from google.adk.models.lite_llm import _MISSING_TOOL_RESULT_MESSAGE
@@ -4689,6 +4690,17 @@ def test_handles_litellm_logger_names(logger_name):
46894690
# ── Anthropic thinking_blocks tests ─────────────────────────────
46904691

46914692

4693+
def test_is_anthropic_provider():
4694+
"""Verify _is_anthropic_provider matches known Claude provider prefixes."""
4695+
assert _is_anthropic_provider("anthropic")
4696+
assert _is_anthropic_provider("bedrock")
4697+
assert _is_anthropic_provider("vertex_ai")
4698+
assert _is_anthropic_provider("ANTHROPIC") # case-insensitive
4699+
assert not _is_anthropic_provider("openai")
4700+
assert not _is_anthropic_provider("")
4701+
assert not _is_anthropic_provider(None)
4702+
4703+
46924704
@pytest.mark.parametrize(
46934705
"model_string,expected",
46944706
[
@@ -4723,9 +4735,10 @@ def test_is_anthropic_model(model_string, expected):
47234735

47244736

47254737
def test_extract_reasoning_value_prefers_thinking_blocks():
4726-
"""thinking_blocks takes precedence over reasoning_content."""
4738+
"""thinking_blocks (Anthropic format with signatures) take priority."""
47274739
thinking_blocks = [
4728-
{"type": "thinking", "thinking": "deep thought", "signature": "sig123"},
4740+
{"type": "thinking", "thinking": "step 1", "signature": "c2lnX2E="},
4741+
{"type": "thinking", "thinking": "step 2", "signature": "c2lnX2I="},
47294742
]
47304743
message = {
47314744
"role": "assistant",
@@ -4748,25 +4761,36 @@ def test_extract_reasoning_value_falls_back_without_thinking_blocks():
47484761
assert result == "flat reasoning"
47494762

47504763

4751-
def test_convert_reasoning_value_to_parts_thinking_blocks_preserves_signature():
4752-
"""thinking_blocks format produces parts with thought_signature."""
4764+
def test_convert_reasoning_value_to_parts_preserves_base64_signature():
4765+
"""Base64 signatures are decoded to raw bytes on thought parts."""
47534766
thinking_blocks = [
4754-
{"type": "thinking", "thinking": "step 1", "signature": "sig_abc"},
4755-
{"type": "thinking", "thinking": "step 2", "signature": "sig_def"},
4767+
{"type": "thinking", "thinking": "step 1", "signature": "c2lnX2E="},
4768+
{"type": "thinking", "thinking": "step 2", "signature": "c2lnX2I="},
47564769
]
47574770
parts = _convert_reasoning_value_to_parts(thinking_blocks)
47584771
assert len(parts) == 2
47594772
assert parts[0].text == "step 1"
47604773
assert parts[0].thought is True
4761-
assert parts[0].thought_signature == b"sig_abc"
4774+
assert parts[0].thought_signature == b"sig_a"
47624775
assert parts[1].text == "step 2"
4763-
assert parts[1].thought_signature == b"sig_def"
4776+
assert parts[1].thought_signature == b"sig_b"
4777+
4778+
4779+
def test_convert_reasoning_value_to_parts_raw_signature_falls_back_to_utf8():
4780+
"""Non-base64 signatures are preserved as utf-8 bytes."""
4781+
thinking_blocks = [
4782+
{"type": "thinking", "thinking": "step 1", "signature": "sig_raw"},
4783+
]
4784+
parts = _convert_reasoning_value_to_parts(thinking_blocks)
4785+
assert len(parts) == 1
4786+
assert parts[0].text == "step 1"
4787+
assert parts[0].thought_signature == b"sig_raw"
47644788

47654789

47664790
def test_convert_reasoning_value_to_parts_skips_redacted_blocks():
47674791
"""Redacted thinking blocks are excluded from parts."""
47684792
thinking_blocks = [
4769-
{"type": "thinking", "thinking": "visible", "signature": "sig1"},
4793+
{"type": "thinking", "thinking": "visible", "signature": "c2lnMQ=="},
47704794
{"type": "redacted", "data": "hidden"},
47714795
]
47724796
parts = _convert_reasoning_value_to_parts(thinking_blocks)
@@ -4777,8 +4801,8 @@ def test_convert_reasoning_value_to_parts_skips_redacted_blocks():
47774801
def test_convert_reasoning_value_to_parts_skips_empty_thinking():
47784802
"""Blocks with empty thinking text are excluded."""
47794803
thinking_blocks = [
4780-
{"type": "thinking", "thinking": "", "signature": "sig1"},
4781-
{"type": "thinking", "thinking": "real thought", "signature": "sig2"},
4804+
{"type": "thinking", "thinking": "", "signature": "c2lnMQ=="},
4805+
{"type": "thinking", "thinking": "real thought", "signature": "c2lnMg=="},
47824806
]
47834807
parts = _convert_reasoning_value_to_parts(thinking_blocks)
47844808
assert len(parts) == 1
@@ -4812,13 +4836,14 @@ async def test_content_to_message_param_anthropic_outputs_thinking_blocks():
48124836
content, model="anthropic/claude-4-sonnet"
48134837
)
48144838
assert result["role"] == "assistant"
4815-
assert "thinking_blocks" in result
4839+
assert result["thinking_blocks"] == [
4840+
{
4841+
"type": "thinking",
4842+
"thinking": "deep thought",
4843+
"signature": "sig_round_trip",
4844+
}
4845+
]
48164846
assert result.get("reasoning_content") is None
4817-
blocks = result["thinking_blocks"]
4818-
assert len(blocks) == 1
4819-
assert blocks[0]["type"] == "thinking"
4820-
assert blocks[0]["thinking"] == "deep thought"
4821-
assert blocks[0]["signature"] == "sig_round_trip"
48224847
assert result["content"] == "Hello!"
48234848

48244849

@@ -4839,43 +4864,45 @@ async def test_content_to_message_param_non_anthropic_uses_reasoning_content():
48394864

48404865

48414866
@pytest.mark.asyncio
4842-
async def test_anthropic_thinking_blocks_round_trip():
4843-
"""End-to-end: thinking_blocks in response → Part → thinking_blocks out."""
4844-
# Simulate LiteLLM response with thinking_blocks
4867+
async def test_anthropic_provider_thinking_blocks_round_trip():
4868+
"""End-to-end: thinking_blocks in response stay intact for Anthropic provider."""
48454869
response_message = {
48464870
"role": "assistant",
48474871
"content": "Final answer",
48484872
"thinking_blocks": [
48494873
{
48504874
"type": "thinking",
48514875
"thinking": "Let me reason...",
4852-
"signature": "abc123signature",
4876+
"signature": "c2lnX2E=",
48534877
},
48544878
],
48554879
}
48564880

4857-
# Step 1: Extract reasoning value
48584881
reasoning_value = _extract_reasoning_value(response_message)
48594882
assert isinstance(reasoning_value, list)
48604883

4861-
# Step 2: Convert to parts (preserves signature)
48624884
parts = _convert_reasoning_value_to_parts(reasoning_value)
48634885
assert len(parts) == 1
4864-
assert parts[0].thought_signature == b"abc123signature"
4886+
assert parts[0].thought_signature == b"sig_a"
48654887

4866-
# Step 3: Build Content for history
4867-
all_parts = parts + [types.Part(text="Final answer")]
4888+
all_parts = parts + [
4889+
types.Part(text="Final answer"),
4890+
types.Part.from_function_call(name="add", args={"a": 1, "b": 2}),
4891+
]
48684892
content = types.Content(role="model", parts=all_parts)
48694893

4870-
# Step 4: Convert back to message param for Anthropic
4871-
result = await _content_to_message_param(
4872-
content, model="anthropic/claude-4-sonnet"
4873-
)
4874-
blocks = result["thinking_blocks"]
4875-
assert len(blocks) == 1
4876-
assert blocks[0]["type"] == "thinking"
4877-
assert blocks[0]["thinking"] == "Let me reason..."
4878-
assert blocks[0]["signature"] == "abc123signature"
4894+
msg = await _content_to_message_param(content, provider="anthropic")
4895+
assert isinstance(msg["content"], list)
4896+
assert msg["content"][0] == {
4897+
"type": "thinking",
4898+
"thinking": "Let me reason...",
4899+
"signature": "c2lnX2E=",
4900+
}
4901+
assert msg["content"][1] == {"type": "text", "text": "Final answer"}
4902+
assert msg["tool_calls"] is not None
4903+
assert len(msg["tool_calls"]) == 1
4904+
assert msg["tool_calls"][0]["function"]["name"] == "add"
4905+
assert msg.get("reasoning_content") is None
48794906

48804907

48814908
@pytest.mark.asyncio
@@ -4891,6 +4918,5 @@ async def test_content_to_message_param_anthropic_no_signature_falls_back():
48914918
result = await _content_to_message_param(
48924919
content, model="anthropic/claude-4-sonnet"
48934920
)
4894-
# Falls back to reasoning_content when no signatures present
48954921
assert result.get("reasoning_content") == "thinking without sig"
48964922
assert "thinking_blocks" not in result

0 commit comments

Comments
 (0)