From 03c20933498d2948e83e16663e69d61634b99232 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 9 Dec 2025 21:49:02 +0530 Subject: [PATCH 1/9] Update --- python/openai/openai_frontend/engine/triton_engine.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 179b963b57..a036cf7373 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -823,14 +823,12 @@ def _validate_chat_request( raise ClientError("logit bias is not currently supported") # Logprobs are only supported for vLLM backend currently - if metadata.backend != "vllm" and ( - request.logprobs is not None or request.top_logprobs is not None - ): + if metadata.backend != "vllm" and (request.logprobs or request.top_logprobs): raise ClientError( "logprobs are currently available only for the vLLM backend" ) - if request.top_logprobs is not None and not request.logprobs: + if request.top_logprobs and not request.logprobs: raise ClientError("`top_logprobs` can only be used when `logprobs` is True") self._verify_chat_tool_call_settings(request=request) From a06c448f210bfb42cf8ae41f1f29a72c7d206d77 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 9 Dec 2025 22:12:05 +0530 Subject: [PATCH 2/9] Update --- python/openai/tests/test_chat_completions.py | 25 +++++++++++++++++++- python/openai/tests/test_completions.py | 13 ++++++++++ python/openai/tests/test_openai_client.py | 6 +++++ 3 files changed, 43 insertions(+), 1 deletion(-) diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index e708689e0e..eee18bf354 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -160,6 +160,19 @@ def test_chat_completions_sampling_parameters( assert response.json()["detail"] == "logit bias is not currently supported" return + # TRT-LLM backend doesn't support logprobs + if ( + param_key == "logprobs" + and param_value is True + and model == "tensorrt_llm_bls" + ): + assert response.status_code == 400 + assert ( + "logprobs are currently available only for the vLLM backend" + in response.json()["detail"] + ) + return + assert response.status_code == 200 assert response.json()["choices"][0]["message"]["content"] assert response.json()["choices"][0]["message"]["role"] == "assistant" @@ -620,9 +633,19 @@ def test_chat_completions_logprobs_false( @pytest.mark.parametrize("top_logprobs_value", [0, 5]) def test_chat_completions_top_logprobs_without_logprobs( - self, client, model: str, messages: List[dict], top_logprobs_value: int + self, + client, + model: str, + messages: List[dict], + top_logprobs_value: int, + backend: str, ): """Test that top_logprobs without logprobs raises validation error.""" + if backend != "vllm": + pytest.skip( + reason="logprobs are currently available only for the vLLM backend" + ) + response = client.post( "/v1/chat/completions", json={ diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index ea1cf62166..4dcad4c2d2 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -87,6 +87,19 @@ def test_completions_sampling_parameters( assert response.json()["detail"] == "logit bias is not supported" return + # TRT-LLM backend doesn't support logprobs + if ( + sampling_parameter == "logprobs" + and value is not None + and model == "tensorrt_llm_bls" + ): + assert response.status_code == 400 + assert ( + "logprobs are currently available only for the vLLM backend" + in response.json()["detail"] + ) + return + assert response.status_code == 200 assert response.json()["choices"][0]["text"].strip() diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index e5ce4ae55e..8b1323b57b 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -669,10 +669,16 @@ async def test_top_logprobs_requires_logprobs( model: str, messages: List[dict], top_logprobs_value: int, + backend: str, ): """ Test that top_logprobs without logprobs raises an error """ + if backend != "vllm": + pytest.skip( + reason="logprobs are currently available only for the vLLM backend" + ) + with pytest.raises(openai.BadRequestError) as exc_info: await client.chat.completions.create( model=model, From 24db0ac3b010a8f9d487c5b4e69f810c4d54f483 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 10 Dec 2025 16:47:50 +0530 Subject: [PATCH 3/9] Temporary workaround to fix CI --- qa/L0_openai/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index a1db293436..5a7bdf5067 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -49,6 +49,8 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then + pip uninstall torch torchvision + pip install onnx==1.19.1 torch torchvision # TODO: Remove this when the next stable version of TRT-LLM is available TENSORRTLLM_DIR="/workspace/TensorRT-LLM" TENSORRTLLM_VERSION="v1.2.0rc2" From 56e848c000bc85564d7877699bcddf47f6953eef Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Wed, 10 Dec 2025 21:46:21 +0530 Subject: [PATCH 4/9] Test --- qa/L0_openai/test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index 5a7bdf5067..de63cc115b 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -49,7 +49,7 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then - pip uninstall torch torchvision + pip uninstall -y torch torchvision onnx pip install onnx==1.19.1 torch torchvision # TODO: Remove this when the next stable version of TRT-LLM is available TENSORRTLLM_DIR="/workspace/TensorRT-LLM" From 92c37adbd1166cd8778f415956ace7cca381954f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Dec 2025 10:58:47 +0530 Subject: [PATCH 5/9] Update --- python/openai/openai_frontend/engine/triton_engine.py | 2 +- python/openai/tests/test_lora.py | 6 ++++++ python/openai/tests/test_openai_client.py | 4 ++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index a036cf7373..cd8e904f66 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -828,7 +828,7 @@ def _validate_chat_request( "logprobs are currently available only for the vLLM backend" ) - if request.top_logprobs and not request.logprobs: + if request.top_logprobs is not None and not request.logprobs: raise ClientError("`top_logprobs` can only be used when `logprobs` is True") self._verify_chat_tool_call_settings(request=request) diff --git a/python/openai/tests/test_lora.py b/python/openai/tests/test_lora.py index 550273cf52..d6322ee4b2 100644 --- a/python/openai/tests/test_lora.py +++ b/python/openai/tests/test_lora.py @@ -181,6 +181,12 @@ def _create_model_repository_mock_llm(self): data_type: TYPE_BOOL dims: [1] optional: true + }, + { + name: "return_logprobs" + data_type: TYPE_BOOL + dims: [1] + optional: true } ] output [ diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 8b1323b57b..62fc6bfa18 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -569,7 +569,7 @@ async def test_chat_completion_logprobs( assert stream_tokens_list == non_stream_tokens_list, "Tokens should match" assert np.allclose( - stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-2 + stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-1 ), "Logprob values should be close" @pytest.mark.asyncio @@ -658,7 +658,7 @@ async def test_completion_logprobs( assert stream_text_offsets == logprobs.text_offset, "Text offsets should match" assert stream_top_logprobs == logprobs.top_logprobs, "Top logprobs should match" assert np.allclose( - stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-2 + stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-1 ), "Token logprob values should be close" @pytest.mark.parametrize("top_logprobs_value", [0, 5]) From de077219034d63f5817a3fd19ce5b87faf32fb24 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Dec 2025 16:04:13 +0530 Subject: [PATCH 6/9] Undo temporary change --- qa/L0_openai/test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index de63cc115b..a1db293436 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -49,8 +49,6 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then - pip uninstall -y torch torchvision onnx - pip install onnx==1.19.1 torch torchvision # TODO: Remove this when the next stable version of TRT-LLM is available TENSORRTLLM_DIR="/workspace/TensorRT-LLM" TENSORRTLLM_VERSION="v1.2.0rc2" From f53a67a627c31a5e89ab562c39a26859f6cf67f7 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Dec 2025 16:10:55 +0530 Subject: [PATCH 7/9] Update --- python/openai/openai_frontend/engine/triton_engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index cd8e904f66..9d515079b9 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -823,7 +823,9 @@ def _validate_chat_request( raise ClientError("logit bias is not currently supported") # Logprobs are only supported for vLLM backend currently - if metadata.backend != "vllm" and (request.logprobs or request.top_logprobs): + if metadata.backend != "vllm" and ( + request.logprobs or request.top_logprobs is not None + ): raise ClientError( "logprobs are currently available only for the vLLM backend" ) From dba2b61fe29a3744275fa5915f3ccbe7b7ecb95e Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Dec 2025 16:12:06 +0530 Subject: [PATCH 8/9] Temporary change to run CI --- qa/L0_openai/test.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index a1db293436..de63cc115b 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -49,6 +49,8 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then + pip uninstall -y torch torchvision onnx + pip install onnx==1.19.1 torch torchvision # TODO: Remove this when the next stable version of TRT-LLM is available TENSORRTLLM_DIR="/workspace/TensorRT-LLM" TENSORRTLLM_VERSION="v1.2.0rc2" From 301277eec1bbf5b2021bf594a1fa09e5df359b24 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 11 Dec 2025 17:07:36 +0530 Subject: [PATCH 9/9] undo temporary changes --- qa/L0_openai/test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/qa/L0_openai/test.sh b/qa/L0_openai/test.sh index de63cc115b..a1db293436 100755 --- a/qa/L0_openai/test.sh +++ b/qa/L0_openai/test.sh @@ -49,8 +49,6 @@ function install_deps() { pip install -r requirements-test.txt if [ "${IMAGE_KIND}" == "TRTLLM" ]; then - pip uninstall -y torch torchvision onnx - pip install onnx==1.19.1 torch torchvision # TODO: Remove this when the next stable version of TRT-LLM is available TENSORRTLLM_DIR="/workspace/TensorRT-LLM" TENSORRTLLM_VERSION="v1.2.0rc2"