Skip to content

Allow thread in grpc _InferStream be daemon #867

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
pavelkochnev wants to merge 2 commits into
base: main
Choose a base branch
from
Open

Allow thread in grpc _InferStream be daemon #867

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
a2ebf84
Allow thread in grpc _InferStream be daemon
pavelkochnev Nov 19, 2025
f31baef
Set grpc _InferStream thread daemon property to None by default
pavelkochnev Nov 19, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions src/python/library/tritonclient/grpc/_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1741,7 +1741,8 @@ def wrapped_callback(call_future):
raise_error_grpc(rpc_error)

def start_stream(
self, callback, stream_timeout=None, headers=None, compression_algorithm=None
self, callback, stream_timeout=None, headers=None, compression_algorithm=None,
daemon=None,
):
"""Starts a grpc bi-directional stream to send streaming inferences.
Note: When using stream, user must ensure the InferenceServerClient.close()
Expand All @@ -1768,6 +1769,8 @@ def start_stream(
Optional grpc compression algorithm to be used on client side.
Currently supports "deflate", "gzip" and None. By default, no
compression is used.
daemon : bool
Make internal thread daemonic if set True. By default set to None.

Raises
------
Expand All @@ -1784,7 +1787,7 @@ def start_stream(
)
metadata = self._get_metadata(headers)

self._stream = _InferStream(callback, self._verbose)
self._stream = _InferStream(callback, self._verbose, daemon=daemon)

try:
response_iterator = self._client_stub.ModelStreamInfer(
Expand Down
7 changes: 5 additions & 2 deletions src/python/library/tritonclient/grpc/_infer_stream.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,16 +52,19 @@ class _InferStream:
'error' would be None for a successful inference.
verbose : bool
Enables verbose mode if set True.
daemon : bool
Make handler thread daemonic if set True (default None).
"""

def __init__(self, callback, verbose):
def __init__(self, callback, verbose, daemon=None):
self._callback = callback
self._verbose = verbose
self._request_queue = queue.Queue()
self._handler = None
self._cancelled = False
self._active = True
self._response_iterator = None
self.daemon = daemon

def __del__(self):
self.close(cancel_requests=True)
Expand Down Expand Up @@ -100,7 +103,7 @@ def _init_handler(self, response_iterator):
if self._handler is not None:
raise_error("Attempted to initialize already initialized InferStream")
# Create a new thread to handle the gRPC response stream
self._handler = threading.Thread(target=self._process_response)
self._handler = threading.Thread(target=self._process_response, daemon=self.daemon)
self._handler.start()
if self._verbose:
print("stream started...")
Expand Down