Skip to content

Commit a201a43

Browse files
[None][perf] offload chat template rendering into async (NVIDIA#15284)
Signed-off-by: yechank <161688079+yechank-nvidia@users.noreply.github.com>
1 parent 6c43b3e commit a201a43

5 files changed

Lines changed: 196 additions & 18 deletions

File tree

tensorrt_llm/inputs/utils.py

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,37 @@ def apply_chat_template(
707707
return result
708708

709709

710+
async def async_apply_chat_template(
711+
*,
712+
model_type: str,
713+
tokenizer: Union[TransformersTokenizer, TokenizerBase],
714+
processor: ProcessorMixin,
715+
conversation: list[ConversationMessage],
716+
add_generation_prompt: bool,
717+
mm_placeholder_counts: list[dict[str, int]],
718+
tools: Optional[list[dict[str, Any]]] = None,
719+
documents: Optional[list[dict[str, str]]] = None,
720+
chat_template: Optional[str] = None,
721+
chat_template_kwargs: Optional[dict[str, Any]] = None,
722+
enable_tokenize: bool = False,
723+
) -> (str | List[str]):
724+
"""Apply chat template without blocking the event loop."""
725+
return await asyncio.to_thread(
726+
apply_chat_template,
727+
model_type=model_type,
728+
tokenizer=tokenizer,
729+
processor=processor,
730+
conversation=conversation,
731+
add_generation_prompt=add_generation_prompt,
732+
mm_placeholder_counts=mm_placeholder_counts,
733+
tools=tools,
734+
documents=documents,
735+
chat_template=chat_template,
736+
chat_template_kwargs=chat_template_kwargs,
737+
enable_tokenize=enable_tokenize,
738+
)
739+
740+
710741
def default_multimodal_input_loader(
711742
*,
712743
tokenizer: Optional[Union[TransformersTokenizer, TokenizerBase]],

tensorrt_llm/serve/openai_server.py

Lines changed: 12 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,8 @@
4141
from tensorrt_llm.inputs.media_io import BaseMediaIO
4242
from tensorrt_llm.inputs.multimodal import MultimodalServerConfig
4343
from tensorrt_llm.inputs.registry import BaseMultimodalInputProcessor
44-
from tensorrt_llm.inputs.utils import ConversationMessage, apply_chat_template
44+
from tensorrt_llm.inputs.utils import (ConversationMessage,
45+
async_apply_chat_template)
4546
from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams
4647
from tensorrt_llm.llmapi import MultimodalEncoder, SchedulingParams, tracing
4748
from tensorrt_llm.llmapi.disagg_utils import (DisaggClusterConfig,
@@ -1547,7 +1548,7 @@ async def chat_stream_generator(
15471548
if request.prompt_token_ids is not None:
15481549
prompt = request.prompt_token_ids
15491550
else:
1550-
prompt: str = apply_chat_template(
1551+
prompt_task = async_apply_chat_template(
15511552
model_type=resolve_top_level_model_type(self.model_config),
15521553
tokenizer=self.tokenizer,
15531554
processor=self.processor,
@@ -1559,9 +1560,12 @@ async def chat_stream_generator(
15591560
chat_template=request.chat_template or self.chat_template,
15601561
chat_template_kwargs=request.chat_template_kwargs or {},
15611562
)
1563+
prompt, (mm_data, mm_embeddings) = await asyncio.gather(
1564+
prompt_task, mm_coroutines)
15621565
prompt = prompt_inputs(prompt)
15631566

1564-
mm_data, mm_embeddings = await mm_coroutines
1567+
if request.prompt_token_ids is not None:
1568+
mm_data, mm_embeddings = await mm_coroutines
15651569
if mm_data:
15661570
prompt["multi_modal_data"] = mm_data
15671571
if mm_embeddings:
@@ -1723,7 +1727,7 @@ async def create_mm_embedding_response(promise: RequestOutput):
17231727
if request.prompt_token_ids is not None:
17241728
prompt = request.prompt_token_ids
17251729
else:
1726-
prompt: str = apply_chat_template(
1730+
prompt_task = async_apply_chat_template(
17271731
model_type=resolve_top_level_model_type(self.model_config),
17281732
tokenizer=self.tokenizer,
17291733
processor=self.processor,
@@ -1735,9 +1739,12 @@ async def create_mm_embedding_response(promise: RequestOutput):
17351739
chat_template=request.chat_template,
17361740
chat_template_kwargs=request.chat_template_kwargs or {},
17371741
)
1742+
prompt, (mm_data, mm_embeddings) = await asyncio.gather(
1743+
prompt_task, mm_coroutines)
17381744
prompt = prompt_inputs(prompt)
17391745

1740-
mm_data, mm_embeddings = await mm_coroutines
1746+
if request.prompt_token_ids is not None:
1747+
mm_data, mm_embeddings = await mm_coroutines
17411748
if mm_embeddings:
17421749
raise ValueError("Cannot use multimodal embeddings as input")
17431750
if mm_data is not None:

tensorrt_llm/serve/resource_governor.py

Lines changed: 17 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
LLM/Proxy/Worker chain.
2020
"""
2121

22+
import asyncio
2223
import traceback
2324
from http import HTTPStatus
2425
from typing import Callable, List, Optional
@@ -27,9 +28,12 @@
2728
from starlette.responses import JSONResponse, Response
2829

2930
from tensorrt_llm.executor.request import TruncateKVCacheRequest
30-
from tensorrt_llm.inputs.utils import ConversationMessage, apply_chat_template
31+
from tensorrt_llm.inputs.utils import ConversationMessage, async_apply_chat_template
3132
from tensorrt_llm.logger import logger
32-
from tensorrt_llm.serve.chat_utils import parse_chat_messages_coroutines
33+
from tensorrt_llm.serve.chat_utils import (
34+
parse_chat_messages_coroutines,
35+
resolve_top_level_model_type,
36+
)
3337
from tensorrt_llm.serve.openai_protocol import (
3438
KVCacheTruncateRequest,
3539
ensure_request_chat_template_allowed,
@@ -91,7 +95,7 @@ def _put_or_unavailable(self, request: TruncateKVCacheRequest) -> Optional[Respo
9195
queue.put(request)
9296
return None
9397

94-
def _convert_messages(
98+
async def _convert_messages(
9599
self,
96100
messages,
97101
tool_dicts,
@@ -102,20 +106,24 @@ def _convert_messages(
102106
) -> List[int]:
103107
"""Convert chat messages to token IDs via chat template + tokenization."""
104108
conversation: List[ConversationMessage] = []
105-
conversation, _, __ = parse_chat_messages_coroutines(messages, self.model_config, None)
106-
return apply_chat_template(
107-
model_type=self.model_config.model_type,
109+
conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines(
110+
messages, self.model_config, None
111+
)
112+
token_task = async_apply_chat_template(
113+
model_type=resolve_top_level_model_type(self.model_config),
108114
tokenizer=self.tokenizer,
109115
processor=self.processor,
110116
conversation=conversation,
111117
add_generation_prompt=add_generation_prompt,
112-
mm_placeholder_counts=[],
118+
mm_placeholder_counts=mm_placeholder_counts,
113119
tools=tool_dicts,
114120
documents=documents,
115121
chat_template=chat_template,
116122
chat_template_kwargs=chat_template_kwargs or {},
117123
enable_tokenize=True,
118124
)
125+
token_ids, _ = await asyncio.gather(token_task, mm_coroutines)
126+
return token_ids
119127

120128
async def _truncate_kv_cache(self, request: KVCacheTruncateRequest) -> Response:
121129
try:
@@ -126,7 +134,7 @@ async def _truncate_kv_cache(self, request: KVCacheTruncateRequest) -> Response:
126134
chat_template_kwargs = request.chat_template_kwargs or {}
127135

128136
messages_to_retain = (
129-
self._convert_messages(
137+
await self._convert_messages(
130138
request.messages_to_retain,
131139
tool_dicts,
132140
request.add_generation_prompt,
@@ -139,7 +147,7 @@ async def _truncate_kv_cache(self, request: KVCacheTruncateRequest) -> Response:
139147
)
140148

141149
messages = (
142-
self._convert_messages(
150+
await self._convert_messages(
143151
request.messages,
144152
tool_dicts,
145153
request.add_generation_prompt,

tensorrt_llm/serve/responses_utils.py

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@
4242

4343
from tensorrt_llm.bindings import steady_clock_now
4444
from tensorrt_llm.executor import GenerationResult
45-
from tensorrt_llm.inputs.utils import apply_chat_template
45+
from tensorrt_llm.inputs.utils import async_apply_chat_template
4646
from tensorrt_llm.llmapi import SamplingParams
4747
from tensorrt_llm.llmapi.llm import RequestOutput
4848
from tensorrt_llm.llmapi.reasoning_parser import (BaseReasoningParser,
@@ -822,13 +822,11 @@ async def _create_input_tokens(
822822

823823
conversation, mm_coroutines, mm_placeholder_counts = parse_chat_messages_coroutines(
824824
messages, model_config)
825-
mm_data = await mm_coroutines
826-
827825
tools_dict = [
828826
tool.model_dump()
829827
for tool in _get_chat_completion_function_tools(request.tools)
830828
]
831-
token_ids = apply_chat_template(
829+
token_task = async_apply_chat_template(
832830
model_type=resolve_top_level_model_type(model_config),
833831
tokenizer=tokenizer,
834832
processor=processor,
@@ -838,6 +836,9 @@ async def _create_input_tokens(
838836
mm_placeholder_counts=mm_placeholder_counts,
839837
enable_tokenize=True,
840838
)
839+
token_ids, (mm_data,
840+
_mm_embeddings) = await asyncio.gather(token_task,
841+
mm_coroutines)
841842

842843
return token_ids, mm_data
843844

tests/unittest/inputs/test_chat_template_dispatch.py

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
# Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
33
"""Tests for content-format-driven chat template dispatch and placeholder handling."""
44

5+
import threading
6+
57
import pytest
68

79
from tensorrt_llm.inputs.content_format import ContentFormat
@@ -15,6 +17,7 @@
1517
_build_openai_content,
1618
_resolve_content_format,
1719
add_multimodal_placeholders,
20+
async_apply_chat_template,
1821
interleave_mm_placeholders,
1922
)
2023

@@ -324,3 +327,131 @@ def test_excess_existing_placeholders_preserved(self):
324327
)
325328
assert result == text
326329
assert result.count("<image>") == 3
330+
331+
332+
class TestAsyncApplyChatTemplate:
333+
@pytest.mark.asyncio
334+
async def test_runs_in_worker_thread(self):
335+
event_loop_thread_id = threading.current_thread().ident
336+
337+
class TrackingTokenizer:
338+
def __init__(self):
339+
self.worker_thread_id = None
340+
341+
def apply_chat_template(self, **_):
342+
self.worker_thread_id = threading.current_thread().ident
343+
return "rendered"
344+
345+
tokenizer = TrackingTokenizer()
346+
347+
result = await async_apply_chat_template(
348+
model_type="test_string_model",
349+
tokenizer=tokenizer,
350+
processor=None,
351+
conversation=[ConversationMessage(role="user", content="hello", media=[])],
352+
add_generation_prompt=True,
353+
mm_placeholder_counts=[{}],
354+
chat_template="{{ messages }}",
355+
)
356+
357+
assert result == "rendered"
358+
assert tokenizer.worker_thread_id is not None
359+
assert tokenizer.worker_thread_id != event_loop_thread_id
360+
361+
362+
class TestServingChatTemplateGather:
363+
"""Cover the asyncio.gather integration in the serving chat-template paths."""
364+
365+
@pytest.mark.asyncio
366+
async def test_resource_governor_convert_messages(self, monkeypatch):
367+
from unittest.mock import Mock
368+
369+
import tensorrt_llm.serve.resource_governor as rg
370+
371+
governor = object.__new__(rg.ResourceGovernor)
372+
governor.model_config = Mock()
373+
governor.tokenizer = Mock()
374+
governor.processor = None
375+
376+
async def fake_mm_coroutine():
377+
# parse_chat_messages_coroutines' coroutine yields
378+
# (mm_data, mm_embeddings).
379+
return ({"image": ["data"]}, None)
380+
381+
monkeypatch.setattr(
382+
rg,
383+
"parse_chat_messages_coroutines",
384+
lambda messages, model_config, _: ([], fake_mm_coroutine(), [{}]),
385+
)
386+
# Must resolve the top-level model type, matching the serving call
387+
# sites (not the raw model_config.model_type).
388+
monkeypatch.setattr(rg, "resolve_top_level_model_type", lambda cfg: "resolved-model-type")
389+
390+
captured = {}
391+
392+
async def fake_async_apply(**kwargs):
393+
captured.update(kwargs)
394+
return [1, 2, 3]
395+
396+
monkeypatch.setattr(rg, "async_apply_chat_template", fake_async_apply)
397+
398+
token_ids = await governor._convert_messages(
399+
messages=[{"role": "user", "content": "hi"}],
400+
tool_dicts=None,
401+
add_generation_prompt=True,
402+
documents=None,
403+
chat_template=None,
404+
chat_template_kwargs=None,
405+
)
406+
407+
# Returns only token_ids, not the (mm_data, mm_embeddings) tuple.
408+
assert token_ids == [1, 2, 3]
409+
# Uses the top-level resolver and forwards the real placeholder counts.
410+
assert captured["model_type"] == "resolved-model-type"
411+
assert captured["mm_placeholder_counts"] == [{}]
412+
413+
@pytest.mark.asyncio
414+
async def test_responses_create_input_tokens_unpacks_mm_tuple(self, monkeypatch):
415+
"""_create_input_tokens must return mm_data, not the whole gather tuple."""
416+
from unittest.mock import Mock
417+
418+
import tensorrt_llm.serve.responses_utils as ru
419+
420+
async def fake_create_input_messages(request, prev_msgs):
421+
return [{"role": "user", "content": "hi"}]
422+
423+
async def fake_mm_coroutine():
424+
return ({"image": ["data"]}, {"image": ["embed"]})
425+
426+
monkeypatch.setattr(ru, "_create_input_messages", fake_create_input_messages)
427+
monkeypatch.setattr(
428+
ru,
429+
"parse_chat_messages_coroutines",
430+
lambda messages, model_config: ([], fake_mm_coroutine(), [{}]),
431+
)
432+
monkeypatch.setattr(ru, "resolve_top_level_model_type", lambda cfg: "resolved-model-type")
433+
monkeypatch.setattr(ru, "_get_chat_completion_function_tools", lambda tools: [])
434+
435+
async def fake_async_apply(**kwargs):
436+
return [1, 2, 3]
437+
438+
monkeypatch.setattr(ru, "async_apply_chat_template", fake_async_apply)
439+
440+
request = Mock()
441+
request.tools = None
442+
request.store = False
443+
444+
token_ids, mm_data = await ru._create_input_tokens(
445+
request=request,
446+
prev_response=None,
447+
prev_msgs=None,
448+
conversation_store=None,
449+
enable_store=False,
450+
tokenizer=Mock(),
451+
model_config=Mock(),
452+
processor=None,
453+
)
454+
455+
assert token_ids == [1, 2, 3]
456+
# mm_data is the data dict, not the (mm_data, mm_embeddings) tuple.
457+
assert mm_data == {"image": ["data"]}

0 commit comments

Comments
 (0)