Use apply_chat_template to calculate tokens (#309)

dixyes · web-flow · commit 6d671d261087 · 2024-06-13T19:05:46.000+08:00
diff --git a/chatglm_cpp/openai_api.py b/chatglm_cpp/openai_api.py
@@ -209,7 +209,7 @@ def tool_call(**kwargs):
         temperature=body.temperature,
     )
     logging.info(f'prompt: "{messages[-1].content}", sync response: "{output.content}"')
-    prompt_tokens = len(pipeline.tokenizer.encode_messages(messages, max_context_length))
+    prompt_tokens = len(pipeline.tokenizer.apply_chat_template(messages, max_context_length))
     completion_tokens = len(pipeline.tokenizer.encode(output.content, body.max_tokens))
 
     finish_reason = "stop"

Original file line number	Diff line number	Diff line change
`@@ -209,7 +209,7 @@ def tool_call(**kwargs):`
`209`	`209`	`temperature=body.temperature,`
`210`	`210`	`)`
`211`	`211`	`logging.info(f'prompt: "{messages[-1].content}", sync response: "{output.content}"')`
`212`		`- prompt_tokens = len(pipeline.tokenizer.encode_messages(messages, max_context_length))`
	`212`	`+ prompt_tokens = len(pipeline.tokenizer.apply_chat_template(messages, max_context_length))`
`213`	`213`	`completion_tokens = len(pipeline.tokenizer.encode(output.content, body.max_tokens))`
`214`	`214`
`215`	`215`	`finish_reason = "stop"`