Skip to content

Commit 2c4bcdd

Browse files
committed
Release recv buffers on transfer exceptions
Signed-off-by: Jin Li <59594262+liji-nv@users.noreply.github.com>
1 parent ff1505b commit 2c4bcdd

1 file changed

Lines changed: 16 additions & 8 deletions

File tree

cpp/tensorrt_llm/batch_manager/dataTransceiver.cpp

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1093,17 +1093,25 @@ class CacheReceiver::Impl
10931093
TLLM_CUDA_CHECK(cudaSetDevice(mDeviceId));
10941094
auto session = sendRequestInfo(llmRequest);
10951095
session.setTime(TransferSession::kTimeRequestInfo);
1096-
bool isReady = receiveReadySignal(session);
1097-
if (!isReady)
1096+
try
1097+
{
1098+
bool isReady = receiveReadySignal(session);
1099+
if (!isReady)
1100+
{
1101+
// Reuse the error state for the cancelled request.
1102+
llmRequest.setState(LlmRequestState::kDISAGG_TRANS_ERROR);
1103+
llmRequest.setKvCacheTransferEnd(std::chrono::steady_clock::now());
1104+
session.releasePreAssignedRecvBuffers();
1105+
return;
1106+
}
1107+
receiveSync(session);
1108+
session.clearPreAssignedRecvBuffers();
1109+
}
1110+
catch (...)
10981111
{
1099-
// Reuse the error state for the cancelled request.
1100-
llmRequest.setState(LlmRequestState::kDISAGG_TRANS_ERROR);
1101-
llmRequest.setKvCacheTransferEnd(std::chrono::steady_clock::now());
11021112
session.releasePreAssignedRecvBuffers();
1103-
return;
1113+
throw;
11041114
}
1105-
receiveSync(session);
1106-
session.clearPreAssignedRecvBuffers();
11071115
llmRequest.setKvCacheTransferEnd(std::chrono::steady_clock::now());
11081116

11091117
TLLM_LOG_DEBUG(mpi::MpiComm::world().getRank(),

0 commit comments

Comments
 (0)