Skip to content

Commit 36f9ee5

Browse files
committed
[TRTLLM-12721][fix] Preserve legacy disagg timeout cleanup
Signed-off-by: Chien-Chun Hung <2679986+chienchunhung@users.noreply.github.com>
1 parent 5f8881c commit 36f9ee5

1 file changed

Lines changed: 37 additions & 16 deletions

File tree

tensorrt_llm/_torch/pyexecutor/py_executor.py

Lines changed: 37 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -4415,18 +4415,27 @@ def _check_disagg_ctx_cache_transfer_status(self, atLeastNum: int = 0):
44154415

44164416
for request_id in list(requests_in_transfer.keys()):
44174417
request = requests_in_transfer[request_id]
4418-
if (is_disagg_inflight_cancel_enabled()
4419-
and request.py_kv_transfer_timed_out
4420-
and request_id not in completed_req_ids):
4421-
if request_id not in self._disagg_timed_out_ctx_cancelled_ids:
4422-
is_cancelled = self.kv_cache_transceiver.cancel_request(
4423-
request)
4424-
if is_cancelled:
4425-
self._disagg_timed_out_ctx_cancelled_ids.add(request_id)
4426-
logger.warning(
4427-
f"Cancelled timed-out context KV transfer for "
4428-
f"request {request.py_request_id}; waiting for "
4429-
"C++ transfer status to report final cleanup")
4418+
if (not request.py_kv_transfer_timed_out
4419+
or request_id in completed_req_ids
4420+
or request_id in self._disagg_timed_out_ctx_cancelled_ids):
4421+
continue
4422+
4423+
is_cancelled = self.kv_cache_transceiver.cancel_request(request)
4424+
if not is_cancelled:
4425+
continue
4426+
4427+
if is_disagg_inflight_cancel_enabled():
4428+
self._disagg_timed_out_ctx_cancelled_ids.add(request_id)
4429+
logger.warning(f"Cancelled timed-out context KV transfer for "
4430+
f"request {request.py_request_id}; waiting for "
4431+
"C++ transfer status to report final cleanup")
4432+
else:
4433+
# Preserve the legacy timeout behavior when in-flight
4434+
# cancellation is disabled: a queued transfer that can be
4435+
# cancelled is immediately released from the async manager.
4436+
request.py_kv_transfer_start_time = None
4437+
request.state = LlmRequestState.DISAGG_CONTEXT_COMPLETE
4438+
self._end_transfer_and_maybe_terminate(request)
44304439

44314440
self._check_cache_transfer_errors("context requests")
44324441

@@ -4966,10 +4975,22 @@ def _handle_responses(self, emit_first_iter: bool = True):
49664975
requests_to_terminate.append(request)
49674976
continue
49684977

4969-
# Check if an in-progress generation transfer needs cancellation due to timeout.
4970-
if (is_disagg_inflight_cancel_enabled()
4971-
and request.is_disagg_generation_transmission_in_progress
4972-
and request.py_kv_transfer_timed_out):
4978+
# Check if a generation request needs cleanup due to KV cache transfer timeout.
4979+
if request.py_kv_transfer_timed_out:
4980+
defer_timeout_cleanup = (
4981+
is_disagg_inflight_cancel_enabled()
4982+
and request.is_disagg_generation_transmission_in_progress)
4983+
if not defer_timeout_cleanup:
4984+
is_cancelled = self.kv_cache_transceiver.cancel_request(
4985+
request)
4986+
if is_cancelled:
4987+
self._handle_errors(
4988+
error_msg=
4989+
f"Request {request.py_request_id} timed out",
4990+
requests=[request],
4991+
charge_budget=False)
4992+
continue
4993+
49734994
if (request.py_request_id
49744995
not in self._disagg_timed_out_gen_cancelled_ids):
49754996
is_cancelled = self.kv_cache_transceiver.cancel_request(

0 commit comments

Comments
 (0)