[UPDATE] Update

[ghstack-poisoned]

Author: mergennachin

extension/llm/server/python/chat_template.py

@@ -14,6 +14,7 @@
 so it must be a deliberate choice rather than a silent default.
 """
 
+import json
 import logging
 from typing import Any, Optional
 
@@ -25,6 +26,29 @@
 _DEFAULT_SPECIAL_TOKENS = ["<|im_end|>", "<|endoftext|>", "<|eot_id|>", "<|end|>"]
 
 
+def _decode_tool_call_arguments(messages: list[dict[str, Any]]) -> None:
+    """In-place: parse each tool call's ``function.arguments`` from a JSON string
+    into an object.
+
+    OpenAI sends assistant tool-call arguments as a JSON-encoded string, but HF
+    chat templates expect a mapping (e.g. Qwen renders ``arguments|items`` into
+    ``<parameter=…>`` tags). Without this, a multi-turn tool conversation makes
+    the template raise "Can only get item pairs from a mapping". Left as-is if
+    the value isn't valid JSON, so a template that wants the raw string still works.
+    """
+    for m in messages:
+        for tc in m.get("tool_calls") or []:
+            fn = tc.get("function")
+            if not isinstance(fn, dict):
+                continue
+            args = fn.get("arguments")
+            if isinstance(args, str):
+                try:
+                    fn["arguments"] = json.loads(args)
+                except (ValueError, TypeError):
+                    pass
+
+
 class ChatTemplate:
     def __init__(
         self,
@@ -69,8 +93,10 @@ def render(
     ) -> str:
         kwargs = {**self._defaults, **(template_kwargs or {})}
         if self._hf is not None:
+            dumped = [m.model_dump(exclude_none=True) for m in messages]
+            _decode_tool_call_arguments(dumped)
             return self._hf.apply_chat_template(
-                [m.model_dump(exclude_none=True) for m in messages],
+                dumped,
                 tools=tools,
                 add_generation_prompt=True,
                 tokenize=False,

extension/llm/server/python/tests/test_template.py

@@ -126,8 +126,10 @@ def test_each_role_labeled():
 
 # Tool round-trip: a turn-2 request (assistant tool_call + tool result) must
 # serialize into the shape any HF chat template consumes — the multi-turn loop
-# breaks at turn 2 otherwise.
-def test_tool_call_roundtrip_messages_passthrough():
+# breaks at turn 2 otherwise. OpenAI sends tool-call arguments as a JSON string;
+# HF templates expect a mapping (Qwen renders `arguments|items`), so the server
+# decodes it before templating.
+def test_tool_call_arguments_decoded_for_template():
     t, fake = _template_with_fake()
     t.render(
         [
@@ -152,6 +154,30 @@ def test_tool_call_roundtrip_messages_passthrough():
     msgs = fake.seen_messages
     asst = next(m for m in msgs if m["role"] == "assistant")
     assert asst["tool_calls"][0]["function"]["name"] == "get_weather"
-    assert asst["tool_calls"][0]["function"]["arguments"] == '{"city": "Paris"}'
+    # Decoded from the JSON string into a mapping the template can iterate.
+    assert asst["tool_calls"][0]["function"]["arguments"] == {"city": "Paris"}
     tool = next(m for m in msgs if m["role"] == "tool")
     assert tool["tool_call_id"] == "c1" and "temp_c" in tool["content"]
+
+
+def test_tool_call_non_json_arguments_left_as_string():
+    # A non-JSON arguments value must not crash; it passes through unchanged.
+    t, fake = _template_with_fake()
+    t.render(
+        [
+            ChatMessage(
+                role="assistant",
+                content=None,
+                tool_calls=[
+                    ToolCall(
+                        index=0,
+                        id="c1",
+                        type="function",
+                        function=FunctionCall(name="f", arguments="not json"),
+                    )
+                ],
+            )
+        ]
+    )
+    asst = next(m for m in fake.seen_messages if m["role"] == "assistant")
+    assert asst["tool_calls"][0]["function"]["arguments"] == "not json"