Skip to content

Commit 8c4dbcc

Browse files
alcoftTAOalcoftTAO
committed
Implemented Qwen3.5 (NOT TESTED).
1 parent 6e52096 commit 8c4dbcc

1 file changed

Lines changed: 225 additions & 0 deletions

File tree

‎llama_cpp/llama_chat_format.py‎

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4421,6 +4421,231 @@ def __call__(self, **kwargs):
44214421
# Use parent implementation
44224422
return super().__call__(**kwargs)
44234423

4424+
class Qwen35ChatHandler(Llava15ChatHandler):
4425+
CHAT_FORMAT = (
4426+
"{%- set image_count = namespace(value=0) -%}"
4427+
"{%- set video_count = namespace(value=0) -%}"
4428+
"{%- macro render_content(content, do_vision_count, is_system_content=false) -%}"
4429+
" {%- if content is string -%}"
4430+
" {{- content -}}"
4431+
" {%- elif content is iterable and content is not mapping -%}"
4432+
" {%- for item in content -%}"
4433+
" {%- if 'image' in item or 'image_url' in item -%}"
4434+
" {%- if is_system_content -%}"
4435+
" {{- raise_exception('System message cannot contain images.') -}}"
4436+
" {%- endif -%}"
4437+
" {%- if do_vision_count -%}"
4438+
" {%- set image_count.value = image_count.value + 1 -%}"
4439+
" {%- endif -%}"
4440+
" {%- if add_vision_id -%}"
4441+
" {{- 'Picture ' ~ image_count.value ~ ': ' -}}"
4442+
" {%- endif -%}"
4443+
" {{- '<|vision_start|>' -}}"
4444+
" {%- if 'image' in item -%}"
4445+
" {%- if item.image is string -%}"
4446+
" {{- item.image -}}"
4447+
" {%- else -%}"
4448+
" {{- item.image.url -}}"
4449+
" {%- endif -%}"
4450+
" {%- elif 'image_url' in item -%}"
4451+
" {%- if item.image_url is string -%}"
4452+
" {{- item.image_url -}}"
4453+
" {%- else -%}"
4454+
" {{- item.image_url.url -}}"
4455+
" {%- endif -%}"
4456+
" {%- endif -%}"
4457+
" {{- '<|vision_end|>' -}}"
4458+
" {%- elif 'video' in item -%}"
4459+
" {%- if is_system_content -%}"
4460+
" {{- raise_exception('System message cannot contain videos.') -}}"
4461+
" {%- endif -%}"
4462+
" {%- if do_vision_count -%}"
4463+
" {%- set video_count.value = video_count.value + 1 -%}"
4464+
" {%- endif -%}"
4465+
" {%- if add_vision_id -%}"
4466+
" {{- 'Video ' ~ video_count.value ~ ': ' -}}"
4467+
" {%- endif -%}"
4468+
" {{- '<|vision_start|>' -}}"
4469+
" {{- item.video -}}"
4470+
" {{- '<|vision_end|>' -}}"
4471+
" {%- elif 'text' in item -%}"
4472+
" {{- item.text -}}"
4473+
" {%- else -%}"
4474+
" {{- raise_exception('Unexpected item type in content.') -}}"
4475+
" {%- endif -%}"
4476+
" {%- endfor -%}"
4477+
" {%- elif content is none or content is undefined -%}"
4478+
" {{- '' -}}"
4479+
" {%- else -%}"
4480+
" {{- raise_exception('Unexpected content type.') -}}"
4481+
" {%- endif -%}"
4482+
"{%- endmacro -%}"
4483+
"{%- if not messages -%}"
4484+
" {{- raise_exception('No messages provided.') -}}"
4485+
"{%- endif -%}"
4486+
"{%- if tools and tools is iterable and tools is not mapping -%}"
4487+
" {{- '<|im_start|>system\n' -}}"
4488+
" {{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' -}}"
4489+
" {%- for tools in tools -%}"
4490+
" {{- '\n' -}}"
4491+
" {{- tool | tojson -}}"
4492+
" {%- endfor -%}"
4493+
" {{- '\n</tools>' -}}"
4494+
" {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' -}}"
4495+
" {%- if messages[0].role == 'system' -%}"
4496+
" {%- set content = render_content(messages[0].content, false, true) | trim -%}"
4497+
" {%- if content -%}"
4498+
" {{- '\n\n' + content -}}"
4499+
" {%- endif -%}"
4500+
" {%- endif -%}"
4501+
" {{- '<|im_end|>\n' -}}"
4502+
"{%- elif messages[0].role == 'system' -%}"
4503+
" {%- set content = render_content(messages[0].content, false, true) -%}"
4504+
" {{- '<|im_start|>system\n' + content + '<|im_end|>\n' -}}"
4505+
"{%- endif -%}"
4506+
"{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1) -%}"
4507+
"{%- for message in messages[::-1] -%}"
4508+
" {%- set index = messages | length - 1 - loop.index0 -%}"
4509+
" {%- if ns.multi_step_tool and message.role == 'user' -%}"
4510+
" {%- set content = render_content(message.content, false) | trim -%}"
4511+
" {%- if not (content.startswith('<tool_response>') and content.endswith('</tool_response>')) -%}"
4512+
" {%- set ns.multi_step_tool = false -%}"
4513+
" {%- set ns.last_query_index = index -%}"
4514+
" {%- endif -%}"
4515+
" {%- endif -%}"
4516+
"{%- endfor -%}"
4517+
"{%- if ns.multi_step_tool -%}"
4518+
" {{- raise_exception('No user query found in messages.') -}}"
4519+
"{%- endif -%}"
4520+
"{%- for message in messages -%}"
4521+
" {%- set content = render_content(message.content, true) | trim -%}"
4522+
" {%- if message.role == 'system' -%}"
4523+
" {%- if not loop.first -%}"
4524+
" {{- raise_exception('System message must be at the beginning.') -}}"
4525+
" {%- endif -%}"
4526+
" {%- elif message.role == 'user' -%}"
4527+
" {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' -}}"
4528+
" {%- elif message.role == 'assistant' -%}"
4529+
" {%- set reasoning_content = '' -%}"
4530+
" {%- if message.reasoning_content is string -%}"
4531+
" {%- set reasoning_content = message.reasoning_content -%}"
4532+
" {%- elif '</think>' in content -%}"
4533+
" {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') -%}"
4534+
" {%- set content = content.split('</think>')[-1].lstrip('\n') -%}"
4535+
" {%- endif -%}"
4536+
" {%- set reasoning_content = reasoning_content | trim -%}"
4537+
" {%- if loop.index0 > ns.last_query_index -%}"
4538+
" {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content -}}"
4539+
" {%- else -%}"
4540+
" {{- '<|im_start|>' + message.role + '\n' + content -}}"
4541+
" {%- endif -%}"
4542+
" {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping -%}"
4543+
" {%- for tool_call in message.tool_call -%}"
4544+
" {%- if tool_call.function is defined -%}"
4545+
" {%- set tool_call = tool_call.function -%}"
4546+
" {%- endif -%}"
4547+
" {%- if loop.first -%}"
4548+
" {%- if content | trim -%}"
4549+
" {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4550+
" {%- else -%}"
4551+
" {{- '<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4552+
" {%- endif -%}"
4553+
" {%- else -%}"
4554+
" {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4555+
" {%- endif -%}"
4556+
" {%- if tool_call.arguments is defined -%}"
4557+
" {%- for (args_name, args_value) in tool_calls.arguments | items -%}"
4558+
" {{- '<parameter=' + args.name + '>\n' -}}"
4559+
" {%- set args_value = args_value | tojson | safe if args_value is mapping or args_value is sequence and args_value is not string else args_value | string -%}"
4560+
" {{- args_value -}}"
4561+
" {{- '\n</parameter>' -}}"
4562+
" {%- endfor -%}"
4563+
" {%- endif -%}"
4564+
" {{- '</function>\n</tool_call>' -}}"
4565+
" {%- endfor -%}"
4566+
" {%- endif -%}"
4567+
" {{- '<|im_end|>\n' -}}"
4568+
" {%- elif message.role == 'tool' -%}"
4569+
" {%- if loop.previtem and loop.previtem.role != 'tool' -%}"
4570+
" {{- '<|im_start|>user' -}}"
4571+
" {%- endif -%}"
4572+
" {{- '\n<tool_response>\n' -}}"
4573+
" {{- content -}}"
4574+
" {{- '\n</tool_response>' -}}"
4575+
" {%- if not loop.last and loop.nextitem.role != 'tool' -%}"
4576+
" {{- '<|im_end|>\n' -}}"
4577+
" {%- elif loop.last -%}"
4578+
" {{- '<|im_end|>\n' -}}"
4579+
" {%- endif -%}"
4580+
" {%- else -%}"
4581+
" {{- raise_exception('Unexpected message role.') -}}"
4582+
" {%- endif -%}"
4583+
"{%- endfor -%}"
4584+
"{%- if add_generation_prompt -%}"
4585+
" {{- '<|im_start|>assistant\n' -}}"
4586+
" {%- if enable_thinking is defined and enable_thinking is false -%}"
4587+
" {{- '<think>\n\n</think>\n\n' -}}"
4588+
" {%- else -%}"
4589+
" {{- '<think>\n' -}}"
4590+
" {%- endif -%}"
4591+
"{%- endif -%}"
4592+
)
4593+
4594+
def __init__(
4595+
self,
4596+
reasoning: bool = True,
4597+
force_reasoning: bool = False,
4598+
add_vision_id: bool = True,
4599+
**kwargs,
4600+
):
4601+
"""
4602+
Parameters:
4603+
- reasoning (bool):
4604+
- True (default): Enables reasoning for better results.
4605+
- False: Disables reasoning for faster results.
4606+
- force_reasoning (bool):
4607+
- True: Force the reasoning in the model by adding <think> to the chat template.
4608+
- False (default): Don't force the reasoning.
4609+
- add_vision_id (bool):
4610+
- True (default): Count all the images. Recommended for multi-image.
4611+
- False: Doesn't count the images. Can save tokens with single-image.
4612+
"""
4613+
self.reasoning = reasoning
4614+
self.force_reasoning = force_reasoning
4615+
self.add_vision_id = add_vision_id
4616+
4617+
super().__init__(image_min_tokens=self.image_min_tokens, **kwargs)
4618+
4619+
def __call__(self, **kwargs):
4620+
self.extra_template_arguments["enable_thinking"] = self.reasoning
4621+
self.extra_template_arguments["force_reasoning"] = self.force_reasoning
4622+
self.extra_template_arguments["add_vision_id"] = self.add_vision_id
4623+
4624+
llama = kwargs['llama']
4625+
4626+
# Clear state for multiple runs
4627+
llama.reset()
4628+
llama._ctx.memory_clear(True)
4629+
llama.n_tokens = 0
4630+
4631+
if hasattr(llama, 'input_ids'):
4632+
llama.input_ids.fill(0)
4633+
4634+
# Clear any handler state
4635+
if hasattr(self, '_last_image_embed'):
4636+
self._last_image_embed = None
4637+
self._last_image_hash = None
4638+
4639+
if self.verbose:
4640+
messages = kwargs.get('messages', [])
4641+
try:
4642+
image_count = len(self.get_image_urls(messages))
4643+
print(f"Qwen35ChatHandler(reasoning={self.reasoning}) - Cleared state, processing {image_count} images", file=sys.stderr)
4644+
except Exception:
4645+
print(f"Qwen35ChatHandler(reasoning={self.reasoning}) - Cleared state", file=sys.stderr)
4646+
4647+
# Use parent implementation
4648+
return super().__call__(**kwargs)
44244649

44254650
@register_chat_completion_handler("chatml-function-calling")
44264651
def chatml_function_calling(

0 commit comments

Comments
 (0)