Skip to content

Commit 3029f52

Browse files
fix(grpc): drain log stream gracefully on success path in solve_lp/mip
stop_log_streaming() calls TryCancel() which races against the server's final log drain: poll_for_completion() returns as soon as the job is marked complete, then TryCancel() kills the stream before the server has finished sending remaining lines (e.g. "Best objective …"). Add drain_log_streaming(): polls log_stream_done_ (set when the thread exits naturally after receiving the job_complete sentinel) for up to 5s, then falls back to stop_log_streaming(). Use it on the success path in solve_lp and solve_mip so all final log lines are received before the stream is torn down. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> Signed-off-by: Ramakrishna Prabhu <ramakrishnap@nvidia.com>
1 parent 25af2f2 commit 3029f52

2 files changed

Lines changed: 34 additions & 2 deletions

File tree

cpp/src/grpc/client/grpc_client.cpp

Lines changed: 29 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -281,12 +281,14 @@ void grpc_client_t::start_log_streaming(const std::string& job_id)
281281
}
282282

283283
stop_logs_.store(false);
284+
log_stream_done_.store(false);
284285
log_thread_ = std::make_unique<std::thread>([this, job_id]() {
285286
stream_logs(job_id, 0, [this](const std::string& line, bool /*job_complete*/) {
286287
if (stop_logs_.load()) return false;
287288
if (config_.log_callback) { config_.log_callback(line); }
288289
return true;
289290
});
291+
log_stream_done_.store(true);
290292
});
291293
}
292294

@@ -308,6 +310,21 @@ void grpc_client_t::stop_log_streaming()
308310
if (t && t->joinable()) { t->join(); }
309311
}
310312

313+
void grpc_client_t::drain_log_streaming()
314+
{
315+
// On the success path the server drains all pending log lines and then sends
316+
// a job_complete sentinel; the streaming thread exits naturally when that
317+
// sentinel arrives. Poll for natural completion (up to kDrainTimeout) so
318+
// callers receive the final log lines (e.g. "Best objective …") before we
319+
// force-cancel. Fall back to stop_log_streaming() if the deadline passes.
320+
static constexpr auto kDrainTimeout = std::chrono::seconds(5);
321+
auto deadline = std::chrono::steady_clock::now() + kDrainTimeout;
322+
while (!log_stream_done_.load() && std::chrono::steady_clock::now() < deadline) {
323+
std::this_thread::sleep_for(std::chrono::milliseconds(10));
324+
}
325+
stop_log_streaming();
326+
}
327+
311328
// =============================================================================
312329
// Proto → Client Enum Conversion
313330
// =============================================================================
@@ -848,7 +865,13 @@ remote_lp_result_t<i_t, f_t> grpc_client_t::solve_lp(
848865

849866
start_log_streaming(sub.job_id);
850867
auto poll = poll_for_completion(sub.job_id);
851-
stop_log_streaming();
868+
// On success, wait for the server to send the job_complete sentinel so all
869+
// final log lines are received before the stream is torn down.
870+
if (poll.completed) {
871+
drain_log_streaming();
872+
} else {
873+
stop_log_streaming();
874+
}
852875

853876
if (!poll.completed) { return {.error_message = poll.error_message}; }
854877

@@ -875,7 +898,11 @@ remote_mip_result_t<i_t, f_t> grpc_client_t::solve_mip(
875898

876899
start_log_streaming(sub.job_id);
877900
auto poll = poll_for_completion(sub.job_id);
878-
stop_log_streaming();
901+
if (poll.completed) {
902+
drain_log_streaming();
903+
} else {
904+
stop_log_streaming();
905+
}
879906

880907
if (!poll.completed) { return {.error_message = poll.error_message}; }
881908

cpp/src/grpc/client/grpc_client.hpp

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,10 @@ class grpc_client_t {
408408
// Activated when config_.stream_logs is true and config_.log_callback is set.
409409
void start_log_streaming(const std::string& job_id);
410410
void stop_log_streaming();
411+
// Graceful variant: waits up to kLogDrainTimeout for the server to send the
412+
// job_complete sentinel before force-cancelling. Use on the success path so
413+
// final log lines (e.g. "Best objective …") are not dropped.
414+
void drain_log_streaming();
411415

412416
// Shared polling loop used by solve_lp and solve_mip.
413417
struct poll_result_t {
@@ -419,6 +423,7 @@ class grpc_client_t {
419423

420424
std::unique_ptr<std::thread> log_thread_;
421425
std::atomic<bool> stop_logs_{false};
426+
std::atomic<bool> log_stream_done_{false};
422427
mutable std::mutex log_context_mutex_;
423428
// Points to the grpc::ClientContext* of the in-flight StreamLogs RPC (if
424429
// any). Typed as void* to avoid exposing grpc headers in the public API.

0 commit comments

Comments
 (0)