From da287fcb3f26f3501a5a18bb38c7ad59e7cacca1 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 16 Feb 2026 12:33:27 +0530 Subject: [PATCH 1/9] Update --- qa/L0_backend_python/model_readiness/test.sh | 67 ++- .../model_readiness/test_model_readiness.py | 464 +++++++++++++++++- .../model.py | 46 ++ .../is_model_ready_fn_raises_error/model.py | 44 ++ .../is_model_ready_fn_returns_false/model.py | 45 ++ .../model.py | 44 ++ .../is_model_ready_fn_returns_true/model.py | 45 ++ .../config.pbtxt | 57 +++ .../model.py | 62 +++ .../is_model_ready_fn_timeout/model.py | 46 ++ 10 files changed, 906 insertions(+), 14 deletions(-) create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py diff --git a/qa/L0_backend_python/model_readiness/test.sh b/qa/L0_backend_python/model_readiness/test.sh index cc87aeacd8..3ecbc7d29c 100755 --- a/qa/L0_backend_python/model_readiness/test.sh +++ b/qa/L0_backend_python/model_readiness/test.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -47,7 +47,7 @@ cp ../../python_models/$MODEL_NAME/config.pbtxt ./models/$MODEL_NAME/config.pbtx for SIGNAL in 11 9; do echo -e "\n***\n*** Testing model_readiness with Signal $SIGNAL\n***" SERVER_LOG="./model_readiness_signal_${SIGNAL}_server.log" - CLIENT_LOG="./model_readiness_${SIGNAL}_client.log" + CLIENT_LOG="./model_readiness_signal_${SIGNAL}_client.log" run_server if [ "$SERVER_PID" == "0" ]; then @@ -107,10 +107,69 @@ for SIGNAL in 11 9; do kill_server done +# +# Test User-Defined Model Readiness Function +# +echo -e "\n***\n*** Testing User-Defined is_model_ready() Function\n***" + +# Define all test models +USER_READY_MODELS=( + "is_model_ready_fn_returns_true" + "is_model_ready_fn_returns_false" + "is_model_ready_fn_raises_error" + "is_model_ready_fn_returns_non_boolean" + "is_model_ready_fn_timeout" + "is_model_ready_fn_coroutine_returns_true" + "is_model_ready_fn_returns_true_decoupled" +) + +# Create model directories and copy files +for MODEL in "${USER_READY_MODELS[@]}"; do + mkdir -p ./models/$MODEL/1/ + cp ./test_models/$MODEL/model.py ./models/$MODEL/1/model.py + if [ "$MODEL" == "is_model_ready_fn_returns_true_decoupled" ]; then + cp ./test_models/$MODEL/config.pbtxt ./models/$MODEL/config.pbtxt + else + cp ./models/identity_fp32/config.pbtxt ./models/$MODEL/config.pbtxt + sed -i "s/^name:.*/name: \"$MODEL\"/" ./models/$MODEL/config.pbtxt + fi +done + +# Start server with all models +SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --log-verbose=1" +SERVER_LOG="./test_user_defined_model_readiness_function_server.log" +CLIENT_LOG="./test_user_defined_model_readiness_function_client.log" + +run_server +if [ "$SERVER_PID" == "0" ]; then + cat $SERVER_LOG + echo -e "\n***\n*** Failed to start $SERVER\n***" + exit 1 +fi + +set +e + +echo "Running TestUserDefinedModelReadinessFunction..." +python3 -m unittest test_model_readiness.TestUserDefinedModelReadinessFunction -v >> ${CLIENT_LOG} 2>&1 +TEST_EXIT_CODE=$? + +if [ $TEST_EXIT_CODE -ne 0 ]; then + echo -e "\n***\n*** TestUserDefinedModelReadinessFunction FAILED\n***" + cat ${CLIENT_LOG} + RET=1 +else + echo -e "\n***\n*** TestUserDefinedModelReadinessFunction PASSED\n***" +fi + +set -e +kill_server + + +# Final result if [ $RET -eq 0 ]; then - echo -e "\n***\n*** Test Passed\n***" + echo -e "\n***\n*** All Model Readiness Tests Passed\n***" else - echo -e "\n***\n*** Test FAILED\n***" + echo -e "\n***\n*** Model Readiness Tests FAILED\n***" fi exit $RET diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index 65d1c81d8a..e64704bc17 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -1,4 +1,4 @@ -# Copyright 2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2025-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -25,28 +25,102 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import unittest - +import numpy as np +import queue +import time +import threading import tritonclient.grpc as grpcclient import tritonclient.http as httpclient +from tritonclient.utils import InferenceServerException +from functools import partial + +URL_HTTP = "localhost:8000" +URL_GRPC = "localhost:8001" +DEFAULT_RESPONSE_TIMEOUT = 60 + + +class UserData: + def __init__(self): + self._response_queue = queue.Queue() + + +def callback(user_data, result, error): + if error: + user_data._response_queue.put(error) + else: + user_data._response_queue.put(result) + + +def prepare_infer_args(input_value): + """ + Create InferInput/InferRequestedOutput lists + """ + input_data = np.array([[input_value]], dtype=np.int32) + infer_input = [grpcclient.InferInput("IN", input_data.shape, "INT32")] + infer_input[0].set_data_from_numpy(input_data) + outputs = [grpcclient.InferRequestedOutput("OUT")] + return infer_input, outputs + + +def collect_responses(user_data, expected_responses_count): + """ + Collect responses from user_data until the final response flag is seen. + """ + errors = [] + responses = [] + recv_count = 0 + while recv_count < expected_responses_count: + try: + result = user_data._response_queue.get(timeout=DEFAULT_RESPONSE_TIMEOUT) + except queue.Empty: + raise Exception( + f"No response received within {DEFAULT_RESPONSE_TIMEOUT} seconds." + ) + if type(result) == InferenceServerException: + errors.append(result) + break + else: + responses.append(result.as_numpy("OUT")[0]) + recv_count = recv_count + 1 + + return errors, responses + + +def call_inference_identity_model(model_name, protocol, client): + """Helper to test inference functionality""" + shape = (1, 8) + input_data = np.ones(shape, dtype=np.float32) + + if protocol == "http": + inputs = [httpclient.InferInput("INPUT0", input_data.shape, "FP32")] + else: + inputs = [grpcclient.InferInput("INPUT0", input_data.shape, "FP32")] + + inputs[0].set_data_from_numpy(input_data) + result = client.infer(model_name, inputs) + output_data = result.as_numpy("OUTPUT0") + + np.testing.assert_array_almost_equal( + input_data, + output_data, + err_msg=f"Inference output mismatch for {model_name}", + ) class TestModelReadiness(unittest.TestCase): def setUp(self): self.model_name = "identity_fp32" - self.url_http = "localhost:8000" - self.url_grpc = "localhost:8001" - self.client_http = httpclient.InferenceServerClient(url=self.url_http) - self.client_grpc = grpcclient.InferenceServerClient(url=self.url_grpc) + self.client_http = httpclient.InferenceServerClient(url=URL_HTTP) + self.client_grpc = grpcclient.InferenceServerClient(url=URL_GRPC) def test_model_ready(self): - print(f"\nTesting if model '{self.model_name}' is READY ...") - # Check HTTP try: is_ready = self.client_http.is_model_ready(self.model_name) self.assertTrue( is_ready, f"[HTTP] Model {self.model_name} should be READY but is NOT" ) + call_inference_identity_model(self.model_name, "http", self.client_http) except Exception as e: self.fail(f"[HTTP] Unexpected error: {str(e)}") @@ -56,12 +130,11 @@ def test_model_ready(self): self.assertTrue( is_ready, f"[gRPC] Model {self.model_name} should be READY but is NOT" ) + call_inference_identity_model(self.model_name, "grpc", self.client_grpc) except Exception as e: self.fail(f"[gRPC] Unexpected error: {str(e)}") def test_model_not_ready(self): - print(f"\nTesting if model '{self.model_name}' is NOT READY ...") - # Check HTTP try: is_ready = self.client_http.is_model_ready(self.model_name) @@ -83,5 +156,376 @@ def test_model_not_ready(self): self.fail(f"[gRPC] Unexpected error: {str(e)}") +class TestUserDefinedModelReadinessFunction(unittest.TestCase): + """ + Test user-defined is_model_ready() function + """ + + def setUp(self): + self.client_http = httpclient.InferenceServerClient(url=URL_HTTP) + self.client_grpc = grpcclient.InferenceServerClient(url=URL_GRPC) + + def _run_inference_decoupled(self, index, model_name, expected_responses_count): + """ + Helper function for streaming inference. + """ + user_data = UserData() + with grpcclient.InferenceServerClient(URL_GRPC) as triton_client: + try: + inputs, outputs = prepare_infer_args(expected_responses_count) + triton_client.start_stream(callback=partial(callback, user_data)) + triton_client.async_stream_infer( + model_name=model_name, inputs=inputs, outputs=outputs + ) + + # Collect and verify responses + errors, responses = collect_responses( + user_data, expected_responses_count + ) + self.assertEqual( + len(responses), + expected_responses_count, + f"Index: {index} - Expected {expected_responses_count} responses, got {len(responses)}", + ) + self.assertEqual( + len(errors), + 0, + f"Index: {index} - Expected 0 errors, got {len(errors)}", + ) + + # Verify correctness of successful responses + for idx, output in enumerate(responses): + self.assertEqual( + output, + expected_responses_count, + msg=f"Response {idx} has incorrect value - {output}", + ) + finally: + triton_client.stop_stream() + + def test_multiple_concurrent_ready_and_infer_requests_decoupled(self): + model_name = "is_model_ready_fn_returns_true_decoupled" + num_requests = 16 + response_count = num_requests + readiness_errors = [] + infer_errors = [] + + def readiness_wrapper(index, model_name): + try: + with grpcclient.InferenceServerClient(url=URL_GRPC) as triton_client: + is_ready = triton_client.is_model_ready(model_name) + if not is_ready: + raise AssertionError( + f"Index: {index} - GRPC client - Model {model_name} should be READY" + ) + except Exception as e: + readiness_errors.append((index, str(e))) + + def inference_wrapper(index, model_name): + try: + self._run_inference_decoupled(index, model_name, response_count) + except Exception as e: + infer_errors.append((index, str(e))) + + # Launch concurrent threads + threads = [] + for i in range(num_requests): + # Start threads with slight delay + time.sleep(0.1) + t1 = threading.Thread( + target=inference_wrapper, args=(i, model_name), name=f"infer-{i}" + ) + t2 = threading.Thread( + target=readiness_wrapper, args=(i, model_name), name=f"ready-{i}" + ) + threads.extend([t1, t2]) + t1.start() + t2.start() + + # Wait for all requests to complete + for t in threads: + t.join(timeout=120) + + for t in threads: + self.assertFalse(t.is_alive(), f"Threads are not completed: {t.name}") + + self.assertEqual( + len(readiness_errors), 0, f"Readiness errors: {readiness_errors}" + ) + self.assertEqual(len(infer_errors), 0, f"Inference errors: {infer_errors}") + + def test_is_model_ready_coroutine_returns_true(self): + model_name = "is_model_ready_fn_coroutine_returns_true" + for _ in range(5): + self.assertTrue( + self.client_http.is_model_ready(model_name), + f"HTTP - Model {model_name} (coroutine) should be READY", + ) + self.assertTrue( + self.client_grpc.is_model_ready(model_name), + f"gRPC - Model {model_name} (coroutine) should be READY", + ) + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_is_model_ready_returns_true(self): + model_name = "is_model_ready_fn_returns_true" + num_requests = 10 + + # Send multiple requests in sequence to ensure consistent behavior + for i in range(num_requests): + self.assertTrue( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be READY", + ) + self.assertTrue( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be READY", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_is_model_ready_returns_false(self): + model_name = "is_model_ready_fn_returns_false" + num_requests = 10 + + # Send multiple requests in sequence to ensure consistent behavior + for i in range(num_requests): + self.assertFalse( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be NOT READY", + ) + self.assertFalse( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be NOT READY", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_is_model_ready_raises_exception(self): + model_name = "is_model_ready_fn_raises_error" + num_requests = 10 + + # Send multiple requests in sequence to ensure consistent behavior + for i in range(num_requests): + self.assertFalse( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be NOT READY (exception)", + ) + self.assertFalse( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (exception)", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + # Test good model afterwards to ensure server is healthy + model_name = "is_model_ready_fn_returns_true" + for i in range(num_requests): + self.assertTrue( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be READY", + ) + self.assertTrue( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be READY", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_is_model_ready_returns_non_boolean(self): + model_name = "is_model_ready_fn_returns_non_boolean" + num_requests = 10 + + # Send multiple requests in sequence to ensure consistent behavior + for i in range(num_requests): + self.assertFalse( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be NOT READY (wrong return type)", + ) + self.assertFalse( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (wrong return type)", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + # Test good model afterwards to ensure server is healthy + model_name = "is_model_ready_fn_returns_true" + for i in range(num_requests): + self.assertTrue( + self.client_http.is_model_ready(model_name), + f"iteration {i} - HTTP client - Model {model_name} should be READY", + ) + self.assertTrue( + self.client_grpc.is_model_ready(model_name), + f"iteration {i} - GRPC client - Model {model_name} should be READY", + ) + + # Inference should work normally + # readiness check functionality should not affect inference + call_inference_identity_model(model_name, "http", self.client_http) + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_is_model_ready_takes_longs_time(self): + model_name = "is_model_ready_fn_timeout" + num_requests = 10 + + # Send multiple requests in sequence to ensure consistent behavior + for i in range(num_requests): + # This call should time out and return NOT_READY. + # Note: the stub will continue running is_model_ready() + # in the background (similar to the inference flow) + # even after the backend readiness timeout expires. + is_ready = self.client_http.is_model_ready(model_name) + self.assertFalse( + is_ready, + f"iteration {i} - HTTP client - Model {model_name} should timeout and return NOT READY", + ) + + call_inference_identity_model(model_name, "http", self.client_http) + + # This call should not create another internal IPC message. + # It must wait for the in-flight readiness check + # and return READY once that check completes. + is_ready = self.client_grpc.is_model_ready(model_name) + self.assertTrue( + is_ready, + f"iteration {i} - GRPC client - Model {model_name} should be READY", + ) + + call_inference_identity_model(model_name, "grpc", self.client_grpc) + + def test_multiple_concurrent_ready_and_infer_requests(self): + model_name = "is_model_ready_fn_returns_true" + ready_results = {"http": [], "grpc": []} + ready_errors = {"http": [], "grpc": []} + infer_results = {"http": [], "grpc": []} + infer_errors = {"http": [], "grpc": []} + num_requests = 16 + + def check_model_readiness(protocol, index): + try: + if protocol == "http": + with httpclient.InferenceServerClient(url=URL_HTTP) as client_http: + is_ready = client_http.is_model_ready(model_name) + ready_results["http"].append((index, is_ready)) + else: + with grpcclient.InferenceServerClient(url=URL_GRPC) as client_grpc: + is_ready = client_grpc.is_model_ready(model_name) + ready_results["grpc"].append((index, is_ready)) + except Exception as e: + ready_errors[protocol].append((index, str(e))) + + def do_inference(protocol, index): + try: + if protocol == "http": + with httpclient.InferenceServerClient(url=URL_HTTP) as client_http: + start = time.time() + call_inference_identity_model(model_name, protocol, client_http) + elapsed = time.time() - start + infer_results["http"].append((index, True, elapsed)) + else: + with grpcclient.InferenceServerClient(url=URL_GRPC) as client_grpc: + start = time.time() + call_inference_identity_model(model_name, protocol, client_grpc) + elapsed = time.time() - start + infer_results["grpc"].append((index, True, elapsed)) + except Exception as e: + infer_errors[protocol].append((index, str(e))) + + # Launch concurrent ready checks + http_threads = [] + for i in range(num_requests): + t1 = threading.Thread(target=check_model_readiness, args=("http", i)) + t2 = threading.Thread(target=do_inference, args=("http", i)) + http_threads.extend([t1, t2]) + t1.start() + t2.start() + + # Wait for all requests to complete + for t in http_threads: + t.join(timeout=60) + + for t in http_threads: + self.assertFalse(t.is_alive(), f"HTTP threads are not completed") + + time.sleep(5) + + grpc_threads = [] + for i in range(num_requests): + t1 = threading.Thread(target=check_model_readiness, args=("grpc", i)) + t2 = threading.Thread(target=do_inference, args=("grpc", i)) + grpc_threads.extend([t1, t2]) + t1.start() + t2.start() + + # Wait for all requests to complete + for t in grpc_threads: + t.join(timeout=60) + + for t in grpc_threads: + self.assertFalse(t.is_alive(), f"gRPC threads are not completed") + + time.sleep(5) + + # Verify no errors in readiness checks + self.assertEqual( + len(ready_errors["http"]), 0, f"HTTP errors: {ready_errors['http']}" + ) + self.assertEqual( + len(ready_errors["grpc"]), 0, f"gRPC errors: {ready_errors['grpc']}" + ) + self.assertEqual( + len(ready_results["http"]), + num_requests, + f"Expected {num_requests} HTTP results", + ) + self.assertEqual( + len(ready_results["grpc"]), + num_requests, + f"Expected {num_requests} gRPC results", + ) + + # All should be True + for idx, ready in ready_results["http"]: + self.assertTrue(ready, f"HTTP check {idx} should be ready") + for idx, ready in ready_results["grpc"]: + self.assertTrue(ready, f"gRPC check {idx} should be ready") + + # Verify no errors in inference + self.assertEqual( + len(infer_errors["http"]), 0, f"Errors occurred: {infer_errors['http']}" + ) + self.assertEqual( + len(infer_errors["grpc"]), 0, f"Errors occurred: {infer_errors['grpc']}" + ) + self.assertEqual( + len(infer_results["http"]), + num_requests, + f"Expected {num_requests} HTTP inference results", + ) + self.assertEqual( + len(infer_results["grpc"]), + num_requests, + f"Expected {num_requests} gRPC inference results", + ) + + if __name__ == "__main__": unittest.main() diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py new file mode 100644 index 0000000000..b678077f04 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py @@ -0,0 +1,46 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import asyncio +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + async def is_model_ready(self) -> bool: + """Async coroutine: stub runs it via RunCoroutine""" + await asyncio.sleep(0.1) + return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py new file mode 100644 index 0000000000..f6e6630877 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py @@ -0,0 +1,44 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + def is_model_ready(self): + """Raises an exception - simulates health check failure""" + raise RuntimeError("Internal check failed – model is not ready") diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py new file mode 100644 index 0000000000..de4a106561 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py @@ -0,0 +1,45 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils +import time + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + def is_model_ready(self) -> bool: + # Add slight delay 200ms - under 5s timeout + time.sleep(0.2) + return False diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py new file mode 100644 index 0000000000..9716bc8f99 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py @@ -0,0 +1,44 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + def is_model_ready(self): + """Returns string instead of boolean""" + return "ready" diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py new file mode 100644 index 0000000000..d5c145b382 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py @@ -0,0 +1,45 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils +import time + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + def is_model_ready(self) -> bool: + # Add slight delay 200ms - under 5s timeout + time.sleep(0.2) + return True \ No newline at end of file diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt new file mode 100644 index 0000000000..1537b804ab --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt @@ -0,0 +1,57 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +backend: "python" +max_batch_size: 1 + +input [ + { + name: "IN" + data_type: TYPE_INT32 + dims: [ 1 ] + } +] + +output [ + { + name: "OUT" + data_type: TYPE_INT32 + dims: [ 1 ] + } +] + +instance_group [ + { + count: 1 + kind: KIND_CPU + } +] + +model_transaction_policy { + decoupled: true +} + diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py new file mode 100644 index 0000000000..a076e7f9b8 --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py @@ -0,0 +1,62 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import numpy as np +import time +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """ + Decoupled model that produces N responses based on input value. + """ + + def execute(self, requests): + for request in requests: + # Get input - number of responses to produce + in_tensor = pb_utils.get_input_tensor_by_name(request, "IN") + count = in_tensor.as_numpy().item() + + response_sender = request.get_response_sender() + out_tensor = pb_utils.Tensor("OUT", np.array([count], dtype=np.int32)) + + # Produce 'count' responses, each with 'count' as the output value + for i in range(count): + # Simulate some processing delay + time.sleep(0.1) + response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) + response_sender.send(response) + + # Send final flag + response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) + + return None + + def is_model_ready(self) -> bool: + # Simulate some processing delay + time.sleep(0.2) + return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py new file mode 100644 index 0000000000..1746f6a86c --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py @@ -0,0 +1,46 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +import triton_python_backend_utils as pb_utils +import time + + +class TritonPythonModel: + def execute(self, requests): + """ + Identity model in Python backend. + """ + responses = [] + for request in requests: + input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") + out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) + responses.append(pb_utils.InferenceResponse([out_tensor])) + return responses + + def is_model_ready(self): + """Takes longer than default 5 seconds timeout""" + time.sleep(10) + return True From e3903c9f81fbebb8f8f344c390867e0ca5048c4b Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 16 Feb 2026 12:45:40 +0530 Subject: [PATCH 2/9] Update --- qa/L0_backend_python/model_readiness/test_model_readiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index e64704bc17..ca3d3ed158 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -206,7 +206,7 @@ def _run_inference_decoupled(self, index, model_name, expected_responses_count): def test_multiple_concurrent_ready_and_infer_requests_decoupled(self): model_name = "is_model_ready_fn_returns_true_decoupled" num_requests = 16 - response_count = num_requests + response_count = 8 readiness_errors = [] infer_errors = [] From 346fb370109b3f05d82c2816437ad295d7b49e9f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 16 Feb 2026 07:35:27 +0000 Subject: [PATCH 3/9] Fix pre-commit errors --- .../model_readiness/test_model_readiness.py | 9 +++++---- .../is_model_ready_fn_coroutine_returns_true/model.py | 1 + .../test_models/is_model_ready_fn_raises_error/model.py | 2 +- .../test_models/is_model_ready_fn_returns_false/model.py | 6 ++++-- .../test_models/is_model_ready_fn_returns_true/model.py | 8 +++++--- .../is_model_ready_fn_returns_true_decoupled/model.py | 3 ++- .../test_models/is_model_ready_fn_timeout/model.py | 3 ++- 7 files changed, 20 insertions(+), 12 deletions(-) diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index ca3d3ed158..89becb3ae0 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -24,15 +24,16 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import unittest -import numpy as np import queue -import time import threading +import time +import unittest +from functools import partial + +import numpy as np import tritonclient.grpc as grpcclient import tritonclient.http as httpclient from tritonclient.utils import InferenceServerException -from functools import partial URL_HTTP = "localhost:8000" URL_GRPC = "localhost:8001" diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py index b678077f04..e7700b5e28 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py @@ -25,6 +25,7 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio + import triton_python_backend_utils as pb_utils diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py index f6e6630877..1ad6e43a45 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py @@ -38,7 +38,7 @@ def execute(self, requests): out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) responses.append(pb_utils.InferenceResponse([out_tensor])) return responses - + def is_model_ready(self): """Raises an exception - simulates health check failure""" raise RuntimeError("Internal check failed – model is not ready") diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py index de4a106561..c030b30932 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py @@ -24,9 +24,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import triton_python_backend_utils as pb_utils import time +import triton_python_backend_utils as pb_utils + + class TritonPythonModel: def execute(self, requests): """ @@ -41,5 +43,5 @@ def execute(self, requests): def is_model_ready(self) -> bool: # Add slight delay 200ms - under 5s timeout - time.sleep(0.2) + time.sleep(0.2) return False diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py index d5c145b382..81952aea34 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py @@ -24,9 +24,11 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import triton_python_backend_utils as pb_utils import time +import triton_python_backend_utils as pb_utils + + class TritonPythonModel: def execute(self, requests): """ @@ -41,5 +43,5 @@ def execute(self, requests): def is_model_ready(self) -> bool: # Add slight delay 200ms - under 5s timeout - time.sleep(0.2) - return True \ No newline at end of file + time.sleep(0.2) + return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py index a076e7f9b8..a2a0cdc267 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py @@ -25,8 +25,9 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import numpy as np import time + +import numpy as np import triton_python_backend_utils as pb_utils diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py index 1746f6a86c..6358a42002 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py @@ -24,9 +24,10 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -import triton_python_backend_utils as pb_utils import time +import triton_python_backend_utils as pb_utils + class TritonPythonModel: def execute(self, requests): From 766c1b7f5ef2d396b972f0451a17df0fd6192688 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Sat, 21 Feb 2026 23:30:15 +0530 Subject: [PATCH 4/9] Fix typo --- qa/L0_backend_python/model_readiness/test_model_readiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index 89becb3ae0..7b5a4e7bb1 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -383,7 +383,7 @@ def test_is_model_ready_returns_non_boolean(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_takes_longs_time(self): + def test_is_model_ready_takes_long_time(self): model_name = "is_model_ready_fn_timeout" num_requests = 10 From b1c141e3f4f29092a00206ac60207204cdee3b58 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Sun, 22 Feb 2026 23:25:38 +0530 Subject: [PATCH 5/9] Improvements --- qa/L0_backend_python/model_readiness/test.sh | 58 ++++++++++++------- .../model_readiness/test_model_readiness.py | 40 +++++-------- .../is_model_ready_fn_raises_error/model.py | 44 -------------- .../is_model_ready_fn_returns_false/model.py | 47 --------------- .../model.py | 44 -------------- .../is_model_ready_fn_returns_true/model.py | 47 --------------- .../model.py => readiness_coroutine_model.py} | 23 ++++++-- .../model.py => readiness_model.py} | 35 +++++++++-- 8 files changed, 98 insertions(+), 240 deletions(-) delete mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py delete mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py delete mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py delete mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py rename qa/L0_backend_python/model_readiness/test_models/{is_model_ready_fn_coroutine_returns_true/model.py => readiness_coroutine_model.py} (77%) rename qa/L0_backend_python/model_readiness/test_models/{is_model_ready_fn_timeout/model.py => readiness_model.py} (62%) diff --git a/qa/L0_backend_python/model_readiness/test.sh b/qa/L0_backend_python/model_readiness/test.sh index 3ecbc7d29c..52809d9bd6 100755 --- a/qa/L0_backend_python/model_readiness/test.sh +++ b/qa/L0_backend_python/model_readiness/test.sh @@ -112,28 +112,46 @@ done # echo -e "\n***\n*** Testing User-Defined is_model_ready() Function\n***" -# Define all test models -USER_READY_MODELS=( - "is_model_ready_fn_returns_true" - "is_model_ready_fn_returns_false" - "is_model_ready_fn_raises_error" - "is_model_ready_fn_returns_non_boolean" - "is_model_ready_fn_timeout" - "is_model_ready_fn_coroutine_returns_true" - "is_model_ready_fn_returns_true_decoupled" -) - -# Create model directories and copy files -for MODEL in "${USER_READY_MODELS[@]}"; do - mkdir -p ./models/$MODEL/1/ - cp ./test_models/$MODEL/model.py ./models/$MODEL/1/model.py - if [ "$MODEL" == "is_model_ready_fn_returns_true_decoupled" ]; then - cp ./test_models/$MODEL/config.pbtxt ./models/$MODEL/config.pbtxt +# Helper function to set up test models with different readiness behaviors based on config parameters +setup_readiness_test_model() { + local model_name=$1 + local return_value=$2 + local delay_secs=$3 + + mkdir -p ./models/$model_name/1/ + if [ "$model_name" == "is_model_ready_fn_coroutine_returns_true" ]; then + cp ./test_models/readiness_coroutine_model.py ./models/$model_name/1/model.py else - cp ./models/identity_fp32/config.pbtxt ./models/$MODEL/config.pbtxt - sed -i "s/^name:.*/name: \"$MODEL\"/" ./models/$MODEL/config.pbtxt + cp ./test_models/readiness_model.py ./models/$model_name/1/model.py fi -done + cp ./models/identity_fp32/config.pbtxt ./models/$model_name/config.pbtxt + sed -i "s/^name:.*/name: \"$model_name\"/" ./models/$model_name/config.pbtxt + cat >> ./models/$model_name/config.pbtxt << EOF +parameters: { + key: "READINESS_FN_RETURN_VALUE" + value: { string_value: "$return_value" } +} +parameters: { + key: "READINESS_FN_DELAY_SECS" + value: { string_value: "$delay_secs" } +} +EOF +} + +# Create readiness test models using shared model.py + config parameters +setup_readiness_test_model "is_model_ready_fn_returns_true" "true" "0.1" +setup_readiness_test_model "is_model_ready_fn_returns_false" "false" "0.1" +setup_readiness_test_model "is_model_ready_fn_raises_error" "exception" "0.1" +setup_readiness_test_model "is_model_ready_fn_returns_non_boolean" "non_boolean" "0.1" +setup_readiness_test_model "is_model_ready_fn_timeout" "true" "8" +setup_readiness_test_model "is_model_ready_fn_coroutine_returns_true" "coroutine" "0.1" + +# Decoupled model has a unique execute() and its own config +mkdir -p ./models/is_model_ready_fn_returns_true_decoupled/1/ +cp ./test_models/is_model_ready_fn_returns_true_decoupled/model.py \ + ./models/is_model_ready_fn_returns_true_decoupled/1/model.py +cp ./test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt \ + ./models/is_model_ready_fn_returns_true_decoupled/config.pbtxt # Start server with all models SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --log-verbose=1" diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index 7b5a4e7bb1..2c12e31ada 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -53,9 +53,7 @@ def callback(user_data, result, error): def prepare_infer_args(input_value): - """ - Create InferInput/InferRequestedOutput lists - """ + """Create InferInput and InferRequestedOutput lists for decoupled inference.""" input_data = np.array([[input_value]], dtype=np.int32) infer_input = [grpcclient.InferInput("IN", input_data.shape, "INT32")] infer_input[0].set_data_from_numpy(input_data) @@ -65,7 +63,7 @@ def prepare_infer_args(input_value): def collect_responses(user_data, expected_responses_count): """ - Collect responses from user_data until the final response flag is seen. + Collect up to `expected_responses_count` responses from user_data. """ errors = [] responses = [] @@ -88,7 +86,7 @@ def collect_responses(user_data, expected_responses_count): def call_inference_identity_model(model_name, protocol, client): - """Helper to test inference functionality""" + """Send an inference request and verify the output matches the input.""" shape = (1, 8) input_data = np.ones(shape, dtype=np.float32) @@ -167,9 +165,7 @@ def setUp(self): self.client_grpc = grpcclient.InferenceServerClient(url=URL_GRPC) def _run_inference_decoupled(self, index, model_name, expected_responses_count): - """ - Helper function for streaming inference. - """ + """Send a decoupled streaming inference request and verify responses.""" user_data = UserData() with grpcclient.InferenceServerClient(URL_GRPC) as triton_client: try: @@ -284,8 +280,7 @@ def test_is_model_ready_returns_true(self): f"iteration {i} - GRPC client - Model {model_name} should be READY", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) @@ -304,8 +299,7 @@ def test_is_model_ready_returns_false(self): f"iteration {i} - GRPC client - Model {model_name} should be NOT READY", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) @@ -324,12 +318,11 @@ def test_is_model_ready_raises_exception(self): f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (exception)", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - # Test good model afterwards to ensure server is healthy + # Verify a healthy model is still ready to confirm server stability. model_name = "is_model_ready_fn_returns_true" for i in range(num_requests): self.assertTrue( @@ -341,8 +334,7 @@ def test_is_model_ready_raises_exception(self): f"iteration {i} - GRPC client - Model {model_name} should be READY", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) @@ -361,12 +353,11 @@ def test_is_model_ready_returns_non_boolean(self): f"iteration {i} - GRPC client - Model {model_name} should be NOT READY (wrong return type)", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - # Test good model afterwards to ensure server is healthy + # Verify a healthy model is still ready to confirm server stability. model_name = "is_model_ready_fn_returns_true" for i in range(num_requests): self.assertTrue( @@ -378,8 +369,7 @@ def test_is_model_ready_returns_non_boolean(self): f"iteration {i} - GRPC client - Model {model_name} should be READY", ) - # Inference should work normally - # readiness check functionality should not affect inference + # Verify inference is unaffected by readiness checks. call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) @@ -450,7 +440,7 @@ def do_inference(protocol, index): except Exception as e: infer_errors[protocol].append((index, str(e))) - # Launch concurrent ready checks + # Launch concurrent readiness and inference requests. http_threads = [] for i in range(num_requests): t1 = threading.Thread(target=check_model_readiness, args=("http", i)) @@ -466,8 +456,6 @@ def do_inference(protocol, index): for t in http_threads: self.assertFalse(t.is_alive(), f"HTTP threads are not completed") - time.sleep(5) - grpc_threads = [] for i in range(num_requests): t1 = threading.Thread(target=check_model_readiness, args=("grpc", i)) @@ -483,8 +471,6 @@ def do_inference(protocol, index): for t in grpc_threads: self.assertFalse(t.is_alive(), f"gRPC threads are not completed") - time.sleep(5) - # Verify no errors in readiness checks self.assertEqual( len(ready_errors["http"]), 0, f"HTTP errors: {ready_errors['http']}" diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py deleted file mode 100644 index 1ad6e43a45..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_raises_error/model.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - def execute(self, requests): - """ - Identity model in Python backend. - """ - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - def is_model_ready(self): - """Raises an exception - simulates health check failure""" - raise RuntimeError("Internal check failed – model is not ready") diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py deleted file mode 100644 index c030b30932..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_false/model.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import time - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - def execute(self, requests): - """ - Identity model in Python backend. - """ - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - def is_model_ready(self) -> bool: - # Add slight delay 200ms - under 5s timeout - time.sleep(0.2) - return False diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py deleted file mode 100644 index 9716bc8f99..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_non_boolean/model.py +++ /dev/null @@ -1,44 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - def execute(self, requests): - """ - Identity model in Python backend. - """ - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - def is_model_ready(self): - """Returns string instead of boolean""" - return "ready" diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py deleted file mode 100644 index 81952aea34..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true/model.py +++ /dev/null @@ -1,47 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -import time - -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - def execute(self, requests): - """ - Identity model in Python backend. - """ - responses = [] - for request in requests: - input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") - out_tensor = pb_utils.Tensor("OUTPUT0", input_tensor.as_numpy()) - responses.append(pb_utils.InferenceResponse([out_tensor])) - return responses - - def is_model_ready(self) -> bool: - # Add slight delay 200ms - under 5s timeout - time.sleep(0.2) - return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py similarity index 77% rename from qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py rename to qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py index e7700b5e28..2b2640d40f 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_coroutine_returns_true/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py @@ -25,15 +25,27 @@ # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. import asyncio +import json import triton_python_backend_utils as pb_utils class TritonPythonModel: + """ + Parameterized test model for async is_model_ready() testing. + + Behavior is controlled via config.pbtxt parameters: + READINESS_FN_DELAY_SECS - seconds to await before returning (e.g. "0.1") + """ + + def initialize(self, args): + model_config = json.loads(args["model_config"]) + params = model_config.get("parameters", {}) + self.readiness_delay_secs = float( + params.get("READINESS_FN_DELAY_SECS", {}).get("string_value", "0.1") + ) + def execute(self, requests): - """ - Identity model in Python backend. - """ responses = [] for request in requests: input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") @@ -41,7 +53,6 @@ def execute(self, requests): responses.append(pb_utils.InferenceResponse([out_tensor])) return responses - async def is_model_ready(self) -> bool: - """Async coroutine: stub runs it via RunCoroutine""" - await asyncio.sleep(0.1) + async def is_model_ready(self): + await asyncio.sleep(self.readiness_delay_secs) return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_model.py similarity index 62% rename from qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py rename to qa/L0_backend_python/model_readiness/test_models/readiness_model.py index 6358a42002..3061b66d60 100644 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_timeout/model.py +++ b/qa/L0_backend_python/model_readiness/test_models/readiness_model.py @@ -24,16 +24,32 @@ # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE # OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +import json import time import triton_python_backend_utils as pb_utils class TritonPythonModel: + """ + Parameterized test model for user-defined is_model_ready() testing. + + Behavior is controlled via config.pbtxt parameters: + READINESS_FN_RETURN_VALUE - "true", "false", "exception", or "non_boolean" + READINESS_FN_DELAY_SECS - seconds to sleep before returning (e.g. "0.1") + """ + + def initialize(self, args): + model_config = json.loads(args["model_config"]) + params = model_config.get("parameters", {}) + self.readiness_return_value = params.get("READINESS_FN_RETURN_VALUE", {}).get( + "string_value", "true" + ) + self.readiness_delay_secs = float( + params.get("READINESS_FN_DELAY_SECS", {}).get("string_value", "0") + ) + def execute(self, requests): - """ - Identity model in Python backend. - """ responses = [] for request in requests: input_tensor = pb_utils.get_input_tensor_by_name(request, "INPUT0") @@ -42,6 +58,15 @@ def execute(self, requests): return responses def is_model_ready(self): - """Takes longer than default 5 seconds timeout""" - time.sleep(10) + if self.readiness_delay_secs > 0: + time.sleep(self.readiness_delay_secs) + + if self.readiness_return_value == "true": + return True + elif self.readiness_return_value == "false": + return False + elif self.readiness_return_value == "exception": + raise RuntimeError("Internal check failed – model is not ready") + elif self.readiness_return_value == "non_boolean": + return "ready" return True From 39e800b1f0b681a6a8f9db4707df262d7a0a24b8 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 3 Mar 2026 18:15:55 +0530 Subject: [PATCH 6/9] Update --- qa/L0_backend_python/model_readiness/test.sh | 26 ++++---- .../model_readiness/test_model_readiness.py | 22 +++---- .../model.py | 63 ------------------- .../config.pbtxt | 0 .../test_models/readiness_coroutine_model.py | 4 +- .../test_models/readiness_model.py | 4 +- 6 files changed, 28 insertions(+), 91 deletions(-) delete mode 100644 qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py rename qa/L0_backend_python/model_readiness/test_models/{is_model_ready_fn_returns_true_decoupled => is_ready_fn_returns_true_decoupled}/config.pbtxt (100%) diff --git a/qa/L0_backend_python/model_readiness/test.sh b/qa/L0_backend_python/model_readiness/test.sh index 52809d9bd6..b25cfab6ec 100755 --- a/qa/L0_backend_python/model_readiness/test.sh +++ b/qa/L0_backend_python/model_readiness/test.sh @@ -110,7 +110,7 @@ done # # Test User-Defined Model Readiness Function # -echo -e "\n***\n*** Testing User-Defined is_model_ready() Function\n***" +echo -e "\n***\n*** Testing User-Defined is_ready() Function\n***" # Helper function to set up test models with different readiness behaviors based on config parameters setup_readiness_test_model() { @@ -119,7 +119,7 @@ setup_readiness_test_model() { local delay_secs=$3 mkdir -p ./models/$model_name/1/ - if [ "$model_name" == "is_model_ready_fn_coroutine_returns_true" ]; then + if [ "$model_name" == "is_ready_fn_coroutine_returns_true" ]; then cp ./test_models/readiness_coroutine_model.py ./models/$model_name/1/model.py else cp ./test_models/readiness_model.py ./models/$model_name/1/model.py @@ -139,19 +139,19 @@ EOF } # Create readiness test models using shared model.py + config parameters -setup_readiness_test_model "is_model_ready_fn_returns_true" "true" "0.1" -setup_readiness_test_model "is_model_ready_fn_returns_false" "false" "0.1" -setup_readiness_test_model "is_model_ready_fn_raises_error" "exception" "0.1" -setup_readiness_test_model "is_model_ready_fn_returns_non_boolean" "non_boolean" "0.1" -setup_readiness_test_model "is_model_ready_fn_timeout" "true" "8" -setup_readiness_test_model "is_model_ready_fn_coroutine_returns_true" "coroutine" "0.1" +setup_readiness_test_model "is_ready_fn_returns_true" "true" "0.1" +setup_readiness_test_model "is_ready_fn_returns_false" "false" "0.1" +setup_readiness_test_model "is_ready_fn_raises_error" "exception" "0.1" +setup_readiness_test_model "is_ready_fn_returns_non_boolean" "non_boolean" "0.1" +setup_readiness_test_model "is_ready_fn_timeout" "true" "8" +setup_readiness_test_model "is_ready_fn_coroutine_returns_true" "coroutine" "0.1" # Decoupled model has a unique execute() and its own config -mkdir -p ./models/is_model_ready_fn_returns_true_decoupled/1/ -cp ./test_models/is_model_ready_fn_returns_true_decoupled/model.py \ - ./models/is_model_ready_fn_returns_true_decoupled/1/model.py -cp ./test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt \ - ./models/is_model_ready_fn_returns_true_decoupled/config.pbtxt +mkdir -p ./models/is_ready_fn_returns_true_decoupled/1/ +cp ./test_models/is_ready_fn_returns_true_decoupled/model.py \ + ./models/is_ready_fn_returns_true_decoupled/1/model.py +cp ./test_models/is_ready_fn_returns_true_decoupled/config.pbtxt \ + ./models/is_ready_fn_returns_true_decoupled/config.pbtxt # Start server with all models SERVER_ARGS="--model-repository=$(pwd)/models --backend-directory=${BACKEND_DIR} --log-verbose=1" diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index 2c12e31ada..ea448aa8f4 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -157,7 +157,7 @@ def test_model_not_ready(self): class TestUserDefinedModelReadinessFunction(unittest.TestCase): """ - Test user-defined is_model_ready() function + Test user-defined is_ready() function """ def setUp(self): @@ -201,7 +201,7 @@ def _run_inference_decoupled(self, index, model_name, expected_responses_count): triton_client.stop_stream() def test_multiple_concurrent_ready_and_infer_requests_decoupled(self): - model_name = "is_model_ready_fn_returns_true_decoupled" + model_name = "is_ready_fn_returns_true_decoupled" num_requests = 16 response_count = 8 readiness_errors = [] @@ -252,7 +252,7 @@ def inference_wrapper(index, model_name): self.assertEqual(len(infer_errors), 0, f"Inference errors: {infer_errors}") def test_is_model_ready_coroutine_returns_true(self): - model_name = "is_model_ready_fn_coroutine_returns_true" + model_name = "is_ready_fn_coroutine_returns_true" for _ in range(5): self.assertTrue( self.client_http.is_model_ready(model_name), @@ -266,7 +266,7 @@ def test_is_model_ready_coroutine_returns_true(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_is_model_ready_returns_true(self): - model_name = "is_model_ready_fn_returns_true" + model_name = "is_ready_fn_returns_true" num_requests = 10 # Send multiple requests in sequence to ensure consistent behavior @@ -285,7 +285,7 @@ def test_is_model_ready_returns_true(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_is_model_ready_returns_false(self): - model_name = "is_model_ready_fn_returns_false" + model_name = "is_ready_fn_returns_false" num_requests = 10 # Send multiple requests in sequence to ensure consistent behavior @@ -304,7 +304,7 @@ def test_is_model_ready_returns_false(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_is_model_ready_raises_exception(self): - model_name = "is_model_ready_fn_raises_error" + model_name = "is_ready_fn_raises_error" num_requests = 10 # Send multiple requests in sequence to ensure consistent behavior @@ -323,7 +323,7 @@ def test_is_model_ready_raises_exception(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) # Verify a healthy model is still ready to confirm server stability. - model_name = "is_model_ready_fn_returns_true" + model_name = "is_ready_fn_returns_true" for i in range(num_requests): self.assertTrue( self.client_http.is_model_ready(model_name), @@ -339,7 +339,7 @@ def test_is_model_ready_raises_exception(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_is_model_ready_returns_non_boolean(self): - model_name = "is_model_ready_fn_returns_non_boolean" + model_name = "is_ready_fn_returns_non_boolean" num_requests = 10 # Send multiple requests in sequence to ensure consistent behavior @@ -358,7 +358,7 @@ def test_is_model_ready_returns_non_boolean(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) # Verify a healthy model is still ready to confirm server stability. - model_name = "is_model_ready_fn_returns_true" + model_name = "is_ready_fn_returns_true" for i in range(num_requests): self.assertTrue( self.client_http.is_model_ready(model_name), @@ -374,7 +374,7 @@ def test_is_model_ready_returns_non_boolean(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_is_model_ready_takes_long_time(self): - model_name = "is_model_ready_fn_timeout" + model_name = "is_ready_fn_timeout" num_requests = 10 # Send multiple requests in sequence to ensure consistent behavior @@ -403,7 +403,7 @@ def test_is_model_ready_takes_long_time(self): call_inference_identity_model(model_name, "grpc", self.client_grpc) def test_multiple_concurrent_ready_and_infer_requests(self): - model_name = "is_model_ready_fn_returns_true" + model_name = "is_ready_fn_returns_true" ready_results = {"http": [], "grpc": []} ready_errors = {"http": [], "grpc": []} infer_results = {"http": [], "grpc": []} diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py b/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py deleted file mode 100644 index a2a0cdc267..0000000000 --- a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/model.py +++ /dev/null @@ -1,63 +0,0 @@ -# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# -# Redistribution and use in source and binary forms, with or without -# modification, are permitted provided that the following conditions -# are met: -# * Redistributions of source code must retain the above copyright -# notice, this list of conditions and the following disclaimer. -# * Redistributions in binary form must reproduce the above copyright -# notice, this list of conditions and the following disclaimer in the -# documentation and/or other materials provided with the distribution. -# * Neither the name of NVIDIA CORPORATION nor the names of its -# contributors may be used to endorse or promote products derived -# from this software without specific prior written permission. -# -# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY -# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, -# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, -# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY -# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - - -import time - -import numpy as np -import triton_python_backend_utils as pb_utils - - -class TritonPythonModel: - """ - Decoupled model that produces N responses based on input value. - """ - - def execute(self, requests): - for request in requests: - # Get input - number of responses to produce - in_tensor = pb_utils.get_input_tensor_by_name(request, "IN") - count = in_tensor.as_numpy().item() - - response_sender = request.get_response_sender() - out_tensor = pb_utils.Tensor("OUT", np.array([count], dtype=np.int32)) - - # Produce 'count' responses, each with 'count' as the output value - for i in range(count): - # Simulate some processing delay - time.sleep(0.1) - response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) - response_sender.send(response) - - # Send final flag - response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) - - return None - - def is_model_ready(self) -> bool: - # Simulate some processing delay - time.sleep(0.2) - return True diff --git a/qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt b/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/config.pbtxt similarity index 100% rename from qa/L0_backend_python/model_readiness/test_models/is_model_ready_fn_returns_true_decoupled/config.pbtxt rename to qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/config.pbtxt diff --git a/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py index 2b2640d40f..ce33afe700 100644 --- a/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py +++ b/qa/L0_backend_python/model_readiness/test_models/readiness_coroutine_model.py @@ -32,7 +32,7 @@ class TritonPythonModel: """ - Parameterized test model for async is_model_ready() testing. + Parameterized test model for async is_ready() testing. Behavior is controlled via config.pbtxt parameters: READINESS_FN_DELAY_SECS - seconds to await before returning (e.g. "0.1") @@ -53,6 +53,6 @@ def execute(self, requests): responses.append(pb_utils.InferenceResponse([out_tensor])) return responses - async def is_model_ready(self): + async def is_ready(self): await asyncio.sleep(self.readiness_delay_secs) return True diff --git a/qa/L0_backend_python/model_readiness/test_models/readiness_model.py b/qa/L0_backend_python/model_readiness/test_models/readiness_model.py index 3061b66d60..444bd9e983 100644 --- a/qa/L0_backend_python/model_readiness/test_models/readiness_model.py +++ b/qa/L0_backend_python/model_readiness/test_models/readiness_model.py @@ -32,7 +32,7 @@ class TritonPythonModel: """ - Parameterized test model for user-defined is_model_ready() testing. + Parameterized test model for user-defined is_ready() testing. Behavior is controlled via config.pbtxt parameters: READINESS_FN_RETURN_VALUE - "true", "false", "exception", or "non_boolean" @@ -57,7 +57,7 @@ def execute(self, requests): responses.append(pb_utils.InferenceResponse([out_tensor])) return responses - def is_model_ready(self): + def is_ready(self): if self.readiness_delay_secs > 0: time.sleep(self.readiness_delay_secs) From aa5eb07f1375dcdfe0791053e43d2f634476f2a0 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 3 Mar 2026 18:17:29 +0530 Subject: [PATCH 7/9] Update --- .../model_readiness/test_model_readiness.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index ea448aa8f4..417842abd1 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -251,7 +251,7 @@ def inference_wrapper(index, model_name): ) self.assertEqual(len(infer_errors), 0, f"Inference errors: {infer_errors}") - def test_is_model_ready_coroutine_returns_true(self): + def test_is_ready_coroutine_returns_true(self): model_name = "is_ready_fn_coroutine_returns_true" for _ in range(5): self.assertTrue( @@ -265,7 +265,7 @@ def test_is_model_ready_coroutine_returns_true(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_returns_true(self): + def test_is_ready_returns_true(self): model_name = "is_ready_fn_returns_true" num_requests = 10 @@ -284,7 +284,7 @@ def test_is_model_ready_returns_true(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_returns_false(self): + def test_is_ready_returns_false(self): model_name = "is_ready_fn_returns_false" num_requests = 10 @@ -303,7 +303,7 @@ def test_is_model_ready_returns_false(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_raises_exception(self): + def test_is_ready_raises_exception(self): model_name = "is_ready_fn_raises_error" num_requests = 10 @@ -338,7 +338,7 @@ def test_is_model_ready_raises_exception(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_returns_non_boolean(self): + def test_is_ready_returns_non_boolean(self): model_name = "is_ready_fn_returns_non_boolean" num_requests = 10 @@ -373,7 +373,7 @@ def test_is_model_ready_returns_non_boolean(self): call_inference_identity_model(model_name, "http", self.client_http) call_inference_identity_model(model_name, "grpc", self.client_grpc) - def test_is_model_ready_takes_long_time(self): + def test_is_ready_takes_long_time(self): model_name = "is_ready_fn_timeout" num_requests = 10 From e8eac521a03ebc43af70b26f31aa1b2b478e2f57 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 3 Mar 2026 18:18:55 +0530 Subject: [PATCH 8/9] Update --- qa/L0_backend_python/model_readiness/test_model_readiness.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_backend_python/model_readiness/test_model_readiness.py b/qa/L0_backend_python/model_readiness/test_model_readiness.py index 417842abd1..5330eb1c83 100644 --- a/qa/L0_backend_python/model_readiness/test_model_readiness.py +++ b/qa/L0_backend_python/model_readiness/test_model_readiness.py @@ -380,7 +380,7 @@ def test_is_ready_takes_long_time(self): # Send multiple requests in sequence to ensure consistent behavior for i in range(num_requests): # This call should time out and return NOT_READY. - # Note: the stub will continue running is_model_ready() + # Note: the stub will continue running is_ready() # in the background (similar to the inference flow) # even after the backend readiness timeout expires. is_ready = self.client_http.is_model_ready(model_name) From 3e1b8c6b3b69265506af6514be3e6aa4d1ecddc4 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 3 Mar 2026 18:38:37 +0530 Subject: [PATCH 9/9] Update --- .../model.py | 63 +++++++++++++++++++ 1 file changed, 63 insertions(+) create mode 100644 qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py diff --git a/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py b/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py new file mode 100644 index 0000000000..e2a4dfcb0d --- /dev/null +++ b/qa/L0_backend_python/model_readiness/test_models/is_ready_fn_returns_true_decoupled/model.py @@ -0,0 +1,63 @@ +# Copyright 2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +import time + +import numpy as np +import triton_python_backend_utils as pb_utils + + +class TritonPythonModel: + """ + Decoupled model that produces N responses based on input value. + """ + + def execute(self, requests): + for request in requests: + # Get input - number of responses to produce + in_tensor = pb_utils.get_input_tensor_by_name(request, "IN") + count = in_tensor.as_numpy().item() + + response_sender = request.get_response_sender() + out_tensor = pb_utils.Tensor("OUT", np.array([count], dtype=np.int32)) + + # Produce 'count' responses, each with 'count' as the output value + for i in range(count): + # Simulate some processing delay + time.sleep(0.1) + response = pb_utils.InferenceResponse(output_tensors=[out_tensor]) + response_sender.send(response) + + # Send final flag + response_sender.send(flags=pb_utils.TRITONSERVER_RESPONSE_COMPLETE_FINAL) + + return None + + def is_ready(self) -> bool: + # Simulate some processing delay + time.sleep(0.2) + return True