Skip to content

Commit af3d8bd

Browse files
alcoftTAOalcoftTAO
committed
updated template
1 parent 781790f commit af3d8bd

1 file changed

Lines changed: 219 additions & 0 deletions

File tree

‎llama_cpp/llama_chat_format.py‎

Lines changed: 219 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4513,6 +4513,225 @@ def __call__(self, **kwargs):
45134513
# Use parent implementation
45144514
return super().__call__(**kwargs)
45154515

4516+
class Qwen35ChatHandler(Llava15ChatHandler):
4517+
CHAT_FORMAT = (
4518+
"{%- set image_count = namespace(value=0) -%}"
4519+
"{%- set video_count = namespace(value=0) -%}"
4520+
"{%- macro render_content(content, do_vision_count, is_system_content=false) -%}"
4521+
" {%- if content is string -%}"
4522+
" {{- content -}}"
4523+
" {%- elif content is iterable and content is not mapping -%}"
4524+
" {%- for item in content -%}"
4525+
" {%- if 'image' in item or 'image_url' in item -%}"
4526+
" {%- if is_system_content -%}"
4527+
" {{- raise_exception('System message cannot contain images.') -}}"
4528+
" {%- endif -%}"
4529+
" {%- if do_vision_count -%}"
4530+
" {%- set image_count.value = image_count.value + 1 -%}"
4531+
" {%- endif -%}"
4532+
" {%- if add_vision_id -%}"
4533+
" {{- 'Picture ' ~ image_count.value ~ ': ' -}}"
4534+
" {%- endif -%}"
4535+
" {{- '<|vision_start|>' -}}"
4536+
" {%- if 'image' in item -%}"
4537+
" {%- if item.image is string -%}"
4538+
" {{- item.image -}}"
4539+
" {%- else -%}"
4540+
" {{- item.image.url -}}"
4541+
" {%- endif -%}"
4542+
" {%- elif 'image_url' in item -%}"
4543+
" {%- if item.image_url is string -%}"
4544+
" {{- item.image_url -}}"
4545+
" {%- else -%}"
4546+
" {{- item.image_url.url -}}"
4547+
" {%- endif -%}"
4548+
" {%- endif -%}"
4549+
" {{- '<|vision_end|>' -}}"
4550+
" {%- elif 'video' in item -%}"
4551+
" {%- if is_system_content -%}"
4552+
" {{- raise_exception('System message cannot contain videos.') -}}"
4553+
" {%- endif -%}"
4554+
" {%- if do_vision_count -%}"
4555+
" {%- set video_count.value = video_count.value + 1 -%}"
4556+
" {%- endif -%}"
4557+
" {%- if add_vision_id -%}"
4558+
" {{- 'Video ' ~ video_count.value ~ ': ' -}}"
4559+
" {%- endif -%}"
4560+
" {{- '<|vision_start|>' -}}"
4561+
" {{- item.video -}}"
4562+
" {{- '<|vision_end|>' -}}"
4563+
" {%- elif 'text' in item -%}"
4564+
" {{- item.text -}}"
4565+
" {%- else -%}"
4566+
" {{- raise_exception('Unexpected item type in content.') -}}"
4567+
" {%- endif -%}"
4568+
" {%- endfor -%}"
4569+
" {%- elif content is none or content is undefined -%}"
4570+
" {{- '' -}}"
4571+
" {%- else -%}"
4572+
" {{- raise_exception('Unexpected content type.') -}}"
4573+
" {%- endif -%}"
4574+
"{%- endmacro -%}"
4575+
"{%- if not messages -%}"
4576+
" {{- raise_exception('No messages provided.') -}}"
4577+
"{%- endif -%}"
4578+
"{%- if tools and tools is iterable and tools is not mapping -%}"
4579+
" {{- '<|im_start|>system\n' -}}"
4580+
" {{- '# Tools\n\nYou have access to the following functions:\n\n<tools>' -}}"
4581+
" {%- for tool in tools -%}"
4582+
" {{- '\n' -}}"
4583+
" {{- tool | tojson -}}"
4584+
" {%- endfor -%}"
4585+
" {{- '\n</tools>' -}}"
4586+
" {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' -}}"
4587+
" {%- if messages[0].role == 'system' -%}"
4588+
" {%- set content = render_content(messages[0].content, false, true) | trim -%}"
4589+
" {%- if content -%}"
4590+
" {{- '\n\n' + content -}}"
4591+
" {%- endif -%}"
4592+
" {%- endif -%}"
4593+
" {{- '<|im_end|>\n' -}}"
4594+
"{%- elif messages[0].role == 'system' -%}"
4595+
" {%- set content = render_content(messages[0].content, false, true) -%}"
4596+
" {{- '<|im_start|>system\n' + content + '<|im_end|>\n' -}}"
4597+
"{%- endif -%}"
4598+
"{%- set ns = namespace(multi_step_tool=true, last_query_index=messages | length - 1) -%}"
4599+
"{%- for message in messages[::-1] -%}"
4600+
" {%- set index = messages | length - 1 - loop.index0 -%}"
4601+
" {%- if ns.multi_step_tool and message.role == 'user' -%}"
4602+
" {%- set content = render_content(message.content, false) | trim -%}"
4603+
" {%- if not (content.startswith('<tool_response>') and content.endswith('</tool_response>')) -%}"
4604+
" {%- set ns.multi_step_tool = false -%}"
4605+
" {%- set ns.last_query_index = index -%}"
4606+
" {%- endif -%}"
4607+
" {%- endif -%}"
4608+
"{%- endfor -%}"
4609+
"{%- if ns.multi_step_tool -%}"
4610+
" {{- raise_exception('No user query found in messages.') -}}"
4611+
"{%- endif -%}"
4612+
"{%- for message in messages -%}"
4613+
" {%- set content = render_content(message.content, true) | trim -%}"
4614+
" {%- if message.role == 'system' -%}"
4615+
" {%- if not loop.first -%}"
4616+
" {{- raise_exception('System message must be at the beginning.') -}}"
4617+
" {%- endif -%}"
4618+
" {%- elif message.role == 'user' -%}"
4619+
" {{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>\n' -}}"
4620+
" {%- elif message.role == 'assistant' -%}"
4621+
" {%- set reasoning_content = '' -%}"
4622+
" {%- if message.reasoning_content is string -%}"
4623+
" {%- set reasoning_content = message.reasoning_content -%}"
4624+
" {%- elif '</think>' in content -%}"
4625+
" {%- set reasoning_content = content.split('</think>')[0].rstrip('\n').split('<think>')[-1].lstrip('\n') -%}"
4626+
" {%- set content = content.split('</think>')[-1].lstrip('\n') -%}"
4627+
" {%- endif -%}"
4628+
" {%- set reasoning_content = reasoning_content | trim -%}"
4629+
" {%- if loop.index0 > ns.last_query_index -%}"
4630+
" {{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content -}}"
4631+
" {%- else -%}"
4632+
" {{- '<|im_start|>' + message.role + '\n' + content -}}"
4633+
" {%- endif -%}"
4634+
" {%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping -%}"
4635+
" {%- for tool_call in message.tool_call -%}"
4636+
" {%- if tool_call.function is defined -%}"
4637+
" {%- set tool_call = tool_call.function -%}"
4638+
" {%- endif -%}"
4639+
" {%- if loop.first -%}"
4640+
" {%- if content | trim -%}"
4641+
" {{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4642+
" {%- else -%}"
4643+
" {{- '<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4644+
" {%- endif -%}"
4645+
" {%- else -%}"
4646+
" {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' -}}"
4647+
" {%- endif -%}"
4648+
" {%- if tool_call.arguments is defined -%}"
4649+
" {%- for (args_name, args_value) in tool_calls.arguments | items -%}"
4650+
" {{- '<parameter=' + args.name + '>\n' -}}"
4651+
" {%- set args_value = args_value | tojson | safe if args_value is mapping or args_value is sequence and args_value is not string else args_value | string -%}"
4652+
" {{- args_value -}}"
4653+
" {{- '\n</parameter>' -}}"
4654+
" {%- endfor -%}"
4655+
" {%- endif -%}"
4656+
" {{- '</function>\n</tool_call>' -}}"
4657+
" {%- endfor -%}"
4658+
" {%- endif -%}"
4659+
" {{- '<|im_end|>\n' -}}"
4660+
" {%- elif message.role == 'tool' -%}"
4661+
" {%- if loop.previtem and loop.previtem.role != 'tool' -%}"
4662+
" {{- '<|im_start|>user' -}}"
4663+
" {%- endif -%}"
4664+
" {{- '\n<tool_response>\n' -}}"
4665+
" {{- content -}}"
4666+
" {{- '\n</tool_response>' -}}"
4667+
" {%- if not loop.last and loop.nextitem.role != 'tool' -%}"
4668+
" {{- '<|im_end|>\n' -}}"
4669+
" {%- elif loop.last -%}"
4670+
" {{- '<|im_end|>\n' -}}"
4671+
" {%- endif -%}"
4672+
" {%- else -%}"
4673+
" {{- raise_exception('Unexpected message role.') -}}"
4674+
" {%- endif -%}"
4675+
"{%- endfor -%}"
4676+
"{%- if add_generation_prompt -%}"
4677+
" {{- '<|im_start|>assistant\n' -}}"
4678+
" {%- if enable_thinking is false -%}"
4679+
" {{- '<think>\n\n</think>\n\n' -}}"
4680+
" {%- else -%}"
4681+
" {{- '<think>\n' -}}"
4682+
" {%- endif -%}"
4683+
"{%- endif -%}"
4684+
)
4685+
4686+
def __init__(
4687+
self,
4688+
reasoning: bool = True,
4689+
add_vision_id: bool = True,
4690+
**kwargs,
4691+
):
4692+
"""
4693+
Parameters:
4694+
- reasoning (bool):
4695+
- True (default): Enables reasoning for better results.
4696+
- False: Disables reasoning for faster results.
4697+
- add_vision_id (bool):
4698+
- True (default): Count all the images. Recommended for multi-image.
4699+
- False: Doesn't count the images. Can save tokens with single-image.
4700+
"""
4701+
self.reasoning = reasoning
4702+
self.add_vision_id = add_vision_id
4703+
4704+
super().__init__(**kwargs)
4705+
4706+
def __call__(self, **kwargs):
4707+
self.extra_template_arguments["enable_thinking"] = self.reasoning
4708+
self.extra_template_arguments["add_vision_id"] = self.add_vision_id
4709+
4710+
llama = kwargs['llama']
4711+
4712+
# Clear state for multiple runs
4713+
llama.reset()
4714+
llama._ctx.memory_clear(True)
4715+
llama.n_tokens = 0
4716+
4717+
if hasattr(llama, 'input_ids'):
4718+
llama.input_ids.fill(0)
4719+
4720+
# Clear any handler state
4721+
if hasattr(self, '_last_image_embed'):
4722+
self._last_image_embed = None
4723+
self._last_image_hash = None
4724+
4725+
if self.verbose:
4726+
messages = kwargs.get('messages', [])
4727+
try:
4728+
image_count = len(self.get_image_urls(messages))
4729+
print(f"Qwen35ChatHandler(reasoning={self.reasoning}) - Cleared state, processing {image_count} images", file=sys.stderr)
4730+
except Exception:
4731+
print(f"Qwen35ChatHandler(reasoning={self.reasoning}) - Cleared state", file=sys.stderr)
4732+
4733+
# Use parent implementation
4734+
return super().__call__(**kwargs)
45164735

45174736
@register_chat_completion_handler("chatml-function-calling")
45184737
def chatml_function_calling(

0 commit comments

Comments
 (0)