From 2669511045edd5a11b1f475fdd94481e3f517f64 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 11:01:32 +0530 Subject: [PATCH 01/19] Support `logprobs` for vLLM models in OpenAI API --- .../openai_frontend/engine/triton_engine.py | 83 +++++++- .../openai_frontend/engine/utils/triton.py | 196 +++++++++++++++++- python/openai/tests/test_chat_completions.py | 92 +++++++- python/openai/tests/test_completions.py | 77 ++++++- python/openai/tests/test_openai_client.py | 193 +++++++++++++++++ 5 files changed, 621 insertions(+), 20 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 6a96c5652d..217bc01ea7 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -57,6 +57,8 @@ _create_trtllm_generate_request, _create_vllm_embedding_request, _create_vllm_generate_request, + _get_openai_chat_format_logprobs_from_vllm_response, + _get_openai_completion_format_logprobs_from_vllm_response, _get_output, _get_usage_from_response, _get_vllm_lora_names, @@ -86,6 +88,7 @@ FinishReason, Function1, Function2, + Logprobs2, Model, ObjectType, ) @@ -248,13 +251,22 @@ async def chat( response, metadata.backend, RequestKind.GENERATION ) + # Parse logprobs if requested + logprobs_data = None + if request.logprobs: + openai_logprobs = _get_openai_chat_format_logprobs_from_vllm_response( + response + ) + if openai_logprobs: + logprobs_data = Logprobs2(content=openai_logprobs) + return CreateChatCompletionResponse( id=request_id, choices=[ ChatCompletionChoice( index=0, message=response_message, - logprobs=None, + logprobs=logprobs_data, finish_reason=finish_reason, ) ], @@ -352,10 +364,17 @@ async def completion( response, metadata.backend, RequestKind.GENERATION ) + # Parse logprobs if requested + logprobs_data = None + if request.logprobs is not None and request.logprobs > 0: + logprobs_data = _get_openai_completion_format_logprobs_from_vllm_response( + response + ) + choice = Choice( finish_reason=FinishReason.stop, index=0, - logprobs=None, + logprobs=logprobs_data, text=text, ) return CreateCompletionResponse( @@ -587,6 +606,15 @@ async def _streaming_chat_iterator( ) previous_text = current_text + # Parse logprobs for this chunk if requested + chunk_logprobs = None + if request.logprobs: + openai_logprobs = _get_openai_chat_format_logprobs_from_vllm_response( + response + ) + if openai_logprobs: + chunk_logprobs = Logprobs2(content=openai_logprobs) + # if the response delta is None (e.g. because it was a # "control token" for tool calls or the parser otherwise # wasn't ready to send a token, then @@ -600,7 +628,7 @@ async def _streaming_chat_iterator( choice = ChatCompletionStreamingResponseChoice( index=0, delta=response_delta, - logprobs=None, + logprobs=chunk_logprobs, finish_reason=finish_reason, ) @@ -772,8 +800,19 @@ def _validate_chat_request( f"Received n={request.n}, but only single choice (n=1) is currently supported" ) - if request.logit_bias is not None or request.logprobs: - raise ClientError("logit bias and log probs not currently supported") + if request.logit_bias is not None: + raise ClientError("logit bias is not currently supported") + + # Logprobs are only supported for vLLM backend currently + if metadata.backend == "tensorrtllm" and ( + request.logprobs is not None or request.top_logprobs is not None + ): + raise ClientError( + "logprobs are currently not supported for TensorRT-LLM backend" + ) + + if request.top_logprobs and not request.logprobs: + raise ClientError("`top_logprobs` can only be used when `logprobs` is True") self._verify_chat_tool_call_settings(request=request) @@ -828,16 +867,34 @@ async def _streaming_completion_iterator( model = request.model include_usage = request.stream_options and request.stream_options.include_usage usage_accumulator = _StreamingUsageAccumulator(backend) + current_offset = 0 async for response in responses: if include_usage: usage_accumulator.update(response) text = _get_output(response) + + # Parse logprobs for this chunk if requested + chunk_logprobs = None + if request.logprobs is not None and request.logprobs > 0: + chunk_logprobs = ( + _get_openai_completion_format_logprobs_from_vllm_response( + response + ) + ) + # Adjust text offsets based on accumulated output + if chunk_logprobs and chunk_logprobs.text_offset: + chunk_logprobs.text_offset = [ + offset + current_offset for offset in chunk_logprobs.text_offset + ] + + current_offset += len(text) + choice = Choice( finish_reason=FinishReason.stop if response.final else None, index=0, - logprobs=None, + logprobs=chunk_logprobs, text=text, ) chunk = CreateCompletionResponse( @@ -923,8 +980,18 @@ def _validate_completion_request( f"Received best_of={request.best_of}, but only single choice (best_of=1) is currently supported" ) - if request.logit_bias is not None or request.logprobs is not None: - raise ClientError("logit bias and log probs not supported") + if request.logit_bias is not None: + raise ClientError("logit bias is not supported") + + # Logprobs are only supported for vLLM backend currently + if ( + request.logprobs is not None + and request.logprobs > 0 + and metadata.backend == "tensorrtllm" + ): + raise ClientError( + "logprobs are currently not supported for TensorRT-LLM backend" + ) if request.stream_options and not request.stream: raise ClientError("`stream_options` can only be used when `stream` is True") diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 2880875a9e..a0d93b4fb9 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -27,10 +27,11 @@ import json import os import re +import sys from dataclasses import asdict, dataclass, field from enum import Enum from pathlib import Path -from typing import Iterable, List, Optional, Union +from typing import Iterable, List, Optional, Union, Dict import numpy as np import tritonserver @@ -42,7 +43,10 @@ CreateChatCompletionRequest, CreateCompletionRequest, CreateEmbeddingRequest, + ChatCompletionTokenLogprob, EmbeddingUsage, + Logprobs, + TopLogprob, ) from utils.utils import ClientError, ServerError @@ -82,6 +86,8 @@ def _create_vllm_generate_request( "max_completion_tokens", # will be handled explicitly "max_tokens", + "logprobs", + "top_logprobs", } # NOTE: The exclude_none is important, as internals may not support @@ -91,6 +97,7 @@ def _create_vllm_generate_request( exclude_none=True, ) + request_logprobs = False # Indicates CreateChatCompletionRequest if hasattr(request, "max_completion_tokens"): if request.max_completion_tokens is not None: @@ -101,11 +108,31 @@ def _create_vllm_generate_request( # If neither is set, use a default value for max_tokens else: sampling_parameters["max_tokens"] = default_max_tokens + + # Handle logprobs for chat completions + # OpenAI API: logprobs (bool), top_logprobs (int 0-20) + # vLLM API: logprobs (int) - number of top token logprobs to return + if request.logprobs and request.top_logprobs is not None: + sampling_parameters["logprobs"] = request.top_logprobs + request_logprobs = True + elif request.logprobs: + # If logprobs=True but top_logprobs not specified, default to 1 + sampling_parameters["logprobs"] = 1 + request_logprobs = True # Indicates CreateCompletionRequest - elif request.max_tokens is not None: - sampling_parameters["max_tokens"] = request.max_tokens else: - sampling_parameters["max_tokens"] = default_max_tokens + if request.max_tokens is not None: + sampling_parameters["max_tokens"] = request.max_tokens + else: + sampling_parameters["max_tokens"] = default_max_tokens + + # Handle logprobs for completions + # OpenAI API: logprobs (int 0-5) - number of top token log probs + # vLLM API: logprobs (int) - same behavior, pass directly + if request.logprobs is not None and request.logprobs > 0: + sampling_parameters["logprobs"] = request.logprobs + request_logprobs = True + inputs["return_logprobs"] = np.bool_([request_logprobs]) if lora_name is not None: sampling_parameters["lora_name"] = lora_name @@ -375,6 +402,167 @@ def _get_output(response: tritonserver._api._response.InferenceResponse) -> str: return "" +def _get_logprobs_from_response( + response: tritonserver._api._response.InferenceResponse, +) -> Optional[List[Dict]]: + """ + Extracts logprobs from a Triton inference response (vLLM backend). + + Returns: + List of dictionaries containing logprobs data, or None if not available. + Format: [ + { + token_id: { + "logprob": float, + "rank": int, + "decoded_token": str + } + }, + ... + ] + """ + if "logprobs" not in response.outputs: + return None + + logprobs_tensor = response.outputs["logprobs"] + if logprobs_tensor is None: + return None + + # The logprobs are stored as JSON string (vLLM backend) + logprobs_str = _to_string(logprobs_tensor) + + if logprobs_str == "null": + return None + + try: + logprobs_data = json.loads(logprobs_str) + return logprobs_data + except json.JSONDecodeError: + return None + + +def _get_openai_chat_format_logprobs_from_vllm_response( + response: tritonserver._api._response.InferenceResponse, +) -> Optional[List[ChatCompletionTokenLogprob]]: + """ + Convert logprobs from a Triton inference response (vLLM backend) to OpenAI chat completion format. + + Args: + response: Triton inference response containing logprobs output. + + Returns: + List of ChatCompletionTokenLogprob objects, or None if no logprobs available. + """ + vllm_logprobs = _get_logprobs_from_response(response) + + if not vllm_logprobs: + return None + + openai_logprobs = [] + for token_logprobs_dict in vllm_logprobs: + if not token_logprobs_dict: + continue + + # Sort by rank to identify the selected token (rank=1 is always the chosen token) + sorted_tokens = sorted( + token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) + ) + + if not sorted_tokens: + continue + + # The first token (lowest rank) is the selected token + selected_token_id, selected_token_data = sorted_tokens[0] + selected_token = selected_token_data["decoded_token"] + selected_logprob = selected_token_data["logprob"] + + # Convert to bytes representation + token_bytes = list(selected_token.encode("utf-8")) + + top_logprobs_list = [] + for token_id, token_data in sorted_tokens: + decoded_token = token_data["decoded_token"] + top_logprobs_list.append( + TopLogprob( + token=decoded_token, + logprob=token_data["logprob"], + bytes=list(decoded_token.encode("utf-8")), + ) + ) + + openai_logprobs.append( + ChatCompletionTokenLogprob( + token=selected_token, + logprob=selected_logprob, + bytes=token_bytes, + top_logprobs=top_logprobs_list, + ) + ) + + return openai_logprobs + + +def _get_openai_completion_format_logprobs_from_vllm_response( + response: tritonserver._api._response.InferenceResponse, +) -> Optional[Logprobs]: + """ + Convert logprobs from a Triton inference response (vLLM backend) to OpenAI completion format. + + Args: + response: Triton inference response containing logprobs output. + + Returns: + Logprobs object for completions API, or None if no logprobs available. + """ + vllm_logprobs = _get_logprobs_from_response(response) + + if not vllm_logprobs: + return None + + text_offset = [] + token_logprobs = [] + tokens = [] + top_logprobs = [] + + current_offset = 0 + for token_logprobs_dict in vllm_logprobs: + if not token_logprobs_dict: + continue + + # Sort by rank to identify the selected token (rank=1 is always the chosen token) + sorted_tokens = sorted( + token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) + ) + + if not sorted_tokens: + continue + + # The first token (lowest rank) is the selected token + selected_token_id, selected_token_data = sorted_tokens[0] + selected_token = selected_token_data["decoded_token"] + selected_logprob = selected_token_data["logprob"] + + text_offset.append(current_offset) + token_logprobs.append(selected_logprob) + tokens.append(selected_token) + + # Build top_logprobs dict for this position + top_logprobs_dict = {} + for token_id, token_data in sorted_tokens: + decoded_token = token_data["decoded_token"] + top_logprobs_dict[decoded_token] = token_data["logprob"] + top_logprobs.append(top_logprobs_dict) + + current_offset += len(selected_token) + + return Logprobs( + text_offset=text_offset, + token_logprobs=token_logprobs, + tokens=tokens, + top_logprobs=top_logprobs, + ) + + def _validate_triton_responses_non_streaming( responses: List[tritonserver._api._response.InferenceResponse], ): diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 2b410c3cc2..a519894ff8 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -154,12 +154,12 @@ def test_chat_completions_sampling_parameters( ) # FIXME: Add support and remove this check - unsupported_parameters = ["logprobs", "logit_bias"] + unsupported_parameters = ["logit_bias"] if param_key in unsupported_parameters: assert response.status_code == 400 assert ( response.json()["detail"] - == "logit bias and log probs not currently supported" + == "logit bias is not currently supported" ) return @@ -522,10 +522,6 @@ def test_multi_lora(self): def test_request_n_choices(self): pass - @pytest.mark.skip(reason="Not Implemented Yet") - def test_request_logprobs(self): - pass - @pytest.mark.skip(reason="Not Implemented Yet") def test_request_logit_bias(self): pass @@ -548,6 +544,90 @@ def test_usage_response(self, client, model: str, messages: List[dict]): usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] ) + def test_chat_completions_logprobs(self, client, backend: str, model: str, messages: List[dict]): + """Test logprobs parameter for chat completions.""" + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "logprobs": True, + "top_logprobs": 2, + "max_tokens": 10, + }, + ) + + # TRT-LLM should raise an error + if backend == "tensorrtllm": + assert response.status_code == 400 + assert "logprobs are currently not supported for TensorRT-LLM backend" in response.json()["detail"] + return + + assert response.status_code == 200 + response_json = response.json() + + # Check that logprobs are present in the response + choice = response_json["choices"][0] + assert "logprobs" in choice + logprobs = choice["logprobs"] + + if logprobs is not None: + assert "content" in logprobs + content = logprobs["content"] + assert isinstance(content, list) + assert len(content) > 0 + + # Validate structure of each token logprob + for token_logprob in content: + assert "token" in token_logprob + assert "logprob" in token_logprob + assert "bytes" in token_logprob + assert "top_logprobs" in token_logprob + + assert isinstance(token_logprob["token"], str) + assert isinstance(token_logprob["logprob"], (int, float)) + assert isinstance(token_logprob["bytes"], list) + assert isinstance(token_logprob["top_logprobs"], list) + + # Validate top_logprobs structure + for top_logprob in token_logprob["top_logprobs"]: + assert "token" in top_logprob + assert "logprob" in top_logprob + assert "bytes" in top_logprob + + # Test that logprobs=False returns no logprobs + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "logprobs": False, + "max_tokens": 10, + }, + ) + + assert response.status_code == 200 + response_json = response.json() + + # logprobs should be None when logprobs=False + choice = response_json["choices"][0] + assert choice.get("logprobs") is None + + # Test that top_logprobs without logprobs raises an error + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "top_logprobs": 2, + "max_tokens": 10, + }, + ) + + # Should raise validation error + assert response.status_code == 400 + assert "`top_logprobs` can only be used when `logprobs` is True" in response.json()["detail"] + # For tests that won't use the same pytest fixture for server startup across # the whole class test suite. diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index fca7521ad8..913147dfff 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -81,10 +81,10 @@ def test_completions_sampling_parameters( print("Response:", response.json()) # FIXME: Add support and remove this check - unsupported_parameters = ["logprobs", "logit_bias"] + unsupported_parameters = ["logit_bias"] if sampling_parameter in unsupported_parameters: assert response.status_code == 400 - assert response.json()["detail"] == "logit bias and log probs not supported" + assert response.json()["detail"] == "logit bias is not supported" return assert response.status_code == 200 @@ -384,3 +384,76 @@ def test_usage_response(self, client, model: str, prompt: str): assert ( usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] ) + + def test_completions_logprobs(self, client, backend: str, model: str, prompt: str): + """Test logprobs parameter for completions.""" + response = client.post( + "/v1/completions", + json={ + "model": model, + "prompt": prompt, + "logprobs": 3, + "max_tokens": 10, + }, + ) + + # TRT-LLM should raise an error + if backend == "tensorrtllm": + assert response.status_code == 400 + assert "logprobs are not currently supported for TensorRT-LLM backend" in response.json()["detail"] + return + + assert response.status_code == 200 + response_json = response.json() + + # Check that logprobs are present in the response + choice = response_json["choices"][0] + assert "logprobs" in choice + logprobs = choice["logprobs"] + + if logprobs is not None: + assert "text_offset" in logprobs + assert "token_logprobs" in logprobs + assert "tokens" in logprobs + assert "top_logprobs" in logprobs + + assert isinstance(logprobs["text_offset"], list) + assert isinstance(logprobs["token_logprobs"], list) + assert isinstance(logprobs["tokens"], list) + assert isinstance(logprobs["top_logprobs"], list) + + # All lists should have the same length + num_tokens = len(logprobs["tokens"]) + assert len(logprobs["text_offset"]) == num_tokens + assert len(logprobs["token_logprobs"]) == num_tokens + assert len(logprobs["top_logprobs"]) == num_tokens + + # Validate each token + for i in range(num_tokens): + assert isinstance(logprobs["tokens"][i], str) + assert isinstance(logprobs["token_logprobs"][i], (int, float)) + assert isinstance(logprobs["text_offset"][i], int) + assert isinstance(logprobs["top_logprobs"][i], dict) + + # Validate top_logprobs dict contains token -> logprob mappings + for token, logprob in logprobs["top_logprobs"][i].items(): + assert isinstance(token, str) + assert isinstance(logprob, (int, float)) + + # Test that logprobs=0 returns no logprobs + response = client.post( + "/v1/completions", + json={ + "model": model, + "prompt": prompt, + "logprobs": 0, + "max_tokens": 10, + }, + ) + + assert response.status_code == 200 + response_json = response.json() + + # logprobs should be None when logprobs=0 + choice = response_json["choices"][0] + assert choice.get("logprobs") is None diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 5ffcbe4f1d..98439eda24 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -487,3 +487,196 @@ async def test_stream_options_without_streaming( stream_options={"include_usage": True}, ) assert "`stream_options` can only be used when `stream` is True" in str(e.value) + + @pytest.mark.asyncio + async def test_chat_completion_logprobs( + self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] + ): + """Test logprobs for chat completions.""" + # TRT-LLM should raise an error + if backend == "tensorrtllm": + with pytest.raises(openai.BadRequestError) as exc_info: + await client.chat.completions.create( + model=model, + messages=messages, + logprobs=True, + top_logprobs=2, + max_tokens=10, + ) + assert ( + "logprobs are currently not supported for TensorRT-LLM backend" + in str(exc_info.value) + ) + return + + chat_completion = await client.chat.completions.create( + model=model, + messages=messages, + logprobs=True, + top_logprobs=2, + max_tokens=10, + ) + + assert chat_completion.choices[0].message.content + assert chat_completion.choices[0].logprobs is not None + + logprobs = chat_completion.choices[0].logprobs + assert logprobs.content is not None + assert len(logprobs.content) > 0 + + # Validate each token logprob + for token_logprob in logprobs.content: + assert token_logprob.token + assert isinstance(token_logprob.logprob, float) + assert isinstance(token_logprob.bytes, list) + assert token_logprob.top_logprobs is not None + assert len(token_logprob.top_logprobs) > 0 + + @pytest.mark.asyncio + async def test_completion_logprobs( + self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str + ): + """Test logprobs for completions.""" + # TRT-LLM should raise an error + if backend == "tensorrtllm": + with pytest.raises(openai.BadRequestError) as exc_info: + await client.completions.create( + model=model, + prompt=prompt, + logprobs=3, + max_tokens=10, + ) + assert ( + "logprobs are currently not supported for TensorRT-LLM backend" + in str(exc_info.value) + ) + return + + completion = await client.completions.create( + model=model, + prompt=prompt, + logprobs=3, + max_tokens=10, + ) + + assert completion.choices[0].text + assert completion.choices[0].logprobs is not None + + logprobs = completion.choices[0].logprobs + assert logprobs.tokens is not None + assert logprobs.token_logprobs is not None + assert logprobs.text_offset is not None + assert logprobs.top_logprobs is not None + + num_tokens = len(logprobs.tokens) + assert len(logprobs.token_logprobs) == num_tokens + assert len(logprobs.text_offset) == num_tokens + assert len(logprobs.top_logprobs) == num_tokens + + @pytest.mark.asyncio + async def test_chat_streaming_with_logprobs( + self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] + ): + """Test streaming chat completions with logprobs.""" + # TRT-LLM should raise an error + if backend == "tensorrtllm": + with pytest.raises(openai.BadRequestError) as exc_info: + stream = await client.chat.completions.create( + model=model, + messages=messages, + max_tokens=10, + stream=True, + logprobs=True, + top_logprobs=2, + ) + # Try to consume the stream (error should happen before streaming starts) + async for _ in stream: + pass + assert ( + "logprobs are currently not supported for TensorRT-LLM backend" + in str(exc_info.value) + ) + return + + stream = await client.chat.completions.create( + model=model, + messages=messages, + max_tokens=10, + stream=True, + logprobs=True, + top_logprobs=2, + ) + + chunks_with_logprobs = 0 + async for chunk in stream: + if chunk.choices[0].logprobs is not None: + chunks_with_logprobs += 1 + logprobs = chunk.choices[0].logprobs + assert logprobs.content is not None + assert len(logprobs.content) > 0 + + # At least some chunks should have logprobs + assert chunks_with_logprobs > 0 + + @pytest.mark.asyncio + async def test_completion_streaming_with_logprobs( + self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str + ): + """Test streaming completions with logprobs.""" + # TRT-LLM should raise an error + if backend == "tensorrtllm": + with pytest.raises(openai.BadRequestError) as exc_info: + stream = await client.completions.create( + model=model, + prompt=prompt, + max_tokens=10, + stream=True, + logprobs=3, + ) + # Try to consume the stream (error should happen before streaming starts) + async for _ in stream: + pass + assert ( + "logprobs are currently not supported for TensorRT-LLM backend" + in str(exc_info.value) + ) + return + + stream = await client.completions.create( + model=model, + prompt=prompt, + max_tokens=10, + stream=True, + logprobs=3, + ) + + chunks_with_logprobs = 0 + async for chunk in stream: + if chunk.choices[0].logprobs is not None: + chunks_with_logprobs += 1 + logprobs = chunk.choices[0].logprobs + assert logprobs.tokens is not None + assert logprobs.token_logprobs is not None + assert logprobs.text_offset is not None + assert logprobs.top_logprobs is not None + + # At least some chunks should have logprobs + assert chunks_with_logprobs > 0 + + @pytest.mark.asyncio + async def test_top_logprobs_requires_logprobs( + self, client: openai.AsyncOpenAI, model: str, messages: List[dict] + ): + """ + Test that top_logprobs without logprobs raises an error + """ + with pytest.raises(openai.BadRequestError) as exc_info: + await client.chat.completions.create( + model=model, + messages=messages, + top_logprobs=2, # Without logprobs=True + max_tokens=5, + ) + assert "`top_logprobs` can only be used when `logprobs` is True" in str( + exc_info.value + ) From f7b3b49e155899b02eaa590f9a99ff164b27b1cb Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 11:16:06 +0530 Subject: [PATCH 02/19] Fix pre-commit errors --- python/openai/tests/test_chat_completions.py | 31 ++++++++++++-------- python/openai/tests/test_completions.py | 19 +++++++----- 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index a519894ff8..fe86e90309 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -157,10 +157,7 @@ def test_chat_completions_sampling_parameters( unsupported_parameters = ["logit_bias"] if param_key in unsupported_parameters: assert response.status_code == 400 - assert ( - response.json()["detail"] - == "logit bias is not currently supported" - ) + assert response.json()["detail"] == "logit bias is not currently supported" return assert response.status_code == 200 @@ -544,7 +541,9 @@ def test_usage_response(self, client, model: str, messages: List[dict]): usage["total_tokens"] == usage["prompt_tokens"] + usage["completion_tokens"] ) - def test_chat_completions_logprobs(self, client, backend: str, model: str, messages: List[dict]): + def test_chat_completions_logprobs( + self, client, backend: str, model: str, messages: List[dict] + ): """Test logprobs parameter for chat completions.""" response = client.post( "/v1/chat/completions", @@ -560,35 +559,38 @@ def test_chat_completions_logprobs(self, client, backend: str, model: str, messa # TRT-LLM should raise an error if backend == "tensorrtllm": assert response.status_code == 400 - assert "logprobs are currently not supported for TensorRT-LLM backend" in response.json()["detail"] + assert ( + "logprobs are currently not supported for TensorRT-LLM backend" + in response.json()["detail"] + ) return assert response.status_code == 200 response_json = response.json() - + # Check that logprobs are present in the response choice = response_json["choices"][0] assert "logprobs" in choice logprobs = choice["logprobs"] - + if logprobs is not None: assert "content" in logprobs content = logprobs["content"] assert isinstance(content, list) assert len(content) > 0 - + # Validate structure of each token logprob for token_logprob in content: assert "token" in token_logprob assert "logprob" in token_logprob assert "bytes" in token_logprob assert "top_logprobs" in token_logprob - + assert isinstance(token_logprob["token"], str) assert isinstance(token_logprob["logprob"], (int, float)) assert isinstance(token_logprob["bytes"], list) assert isinstance(token_logprob["top_logprobs"], list) - + # Validate top_logprobs structure for top_logprob in token_logprob["top_logprobs"]: assert "token" in top_logprob @@ -608,7 +610,7 @@ def test_chat_completions_logprobs(self, client, backend: str, model: str, messa assert response.status_code == 200 response_json = response.json() - + # logprobs should be None when logprobs=False choice = response_json["choices"][0] assert choice.get("logprobs") is None @@ -626,7 +628,10 @@ def test_chat_completions_logprobs(self, client, backend: str, model: str, messa # Should raise validation error assert response.status_code == 400 - assert "`top_logprobs` can only be used when `logprobs` is True" in response.json()["detail"] + assert ( + "`top_logprobs` can only be used when `logprobs` is True" + in response.json()["detail"] + ) # For tests that won't use the same pytest fixture for server startup across diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 600c1ecc94..858b956477 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -413,41 +413,44 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st # TRT-LLM should raise an error if backend == "tensorrtllm": assert response.status_code == 400 - assert "logprobs are not currently supported for TensorRT-LLM backend" in response.json()["detail"] + assert ( + "logprobs are not currently supported for TensorRT-LLM backend" + in response.json()["detail"] + ) return assert response.status_code == 200 response_json = response.json() - + # Check that logprobs are present in the response choice = response_json["choices"][0] assert "logprobs" in choice logprobs = choice["logprobs"] - + if logprobs is not None: assert "text_offset" in logprobs assert "token_logprobs" in logprobs assert "tokens" in logprobs assert "top_logprobs" in logprobs - + assert isinstance(logprobs["text_offset"], list) assert isinstance(logprobs["token_logprobs"], list) assert isinstance(logprobs["tokens"], list) assert isinstance(logprobs["top_logprobs"], list) - + # All lists should have the same length num_tokens = len(logprobs["tokens"]) assert len(logprobs["text_offset"]) == num_tokens assert len(logprobs["token_logprobs"]) == num_tokens assert len(logprobs["top_logprobs"]) == num_tokens - + # Validate each token for i in range(num_tokens): assert isinstance(logprobs["tokens"][i], str) assert isinstance(logprobs["token_logprobs"][i], (int, float)) assert isinstance(logprobs["text_offset"][i], int) assert isinstance(logprobs["top_logprobs"][i], dict) - + # Validate top_logprobs dict contains token -> logprob mappings for token, logprob in logprobs["top_logprobs"][i].items(): assert isinstance(token, str) @@ -466,7 +469,7 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st assert response.status_code == 200 response_json = response.json() - + # logprobs should be None when logprobs=0 choice = response_json["choices"][0] assert choice.get("logprobs") is None From 3c0772034aa7ff827814226ca53185ee693a233f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 11:19:23 +0530 Subject: [PATCH 03/19] Fix pre-commit --- python/openai/openai_frontend/engine/triton_engine.py | 4 +--- python/openai/openai_frontend/engine/utils/triton.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index aff4fd50cf..f075eb17bf 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -897,9 +897,7 @@ async def _streaming_completion_iterator( chunk_logprobs = None if request.logprobs is not None and request.logprobs > 0: chunk_logprobs = ( - _get_openai_completion_format_logprobs_from_vllm_response( - response - ) + _get_openai_completion_format_logprobs_from_vllm_response(response) ) # Adjust text offsets based on accumulated output if chunk_logprobs and chunk_logprobs.text_offset: diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 69f5179b28..395ac82eee 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -31,19 +31,19 @@ from dataclasses import asdict, dataclass, field from enum import Enum from pathlib import Path -from typing import Iterable, List, Optional, Union, Dict +from typing import Dict, Iterable, List, Optional, Union import numpy as np import tritonserver from pydantic import BaseModel from schemas.openai import ( ChatCompletionNamedToolChoice, + ChatCompletionTokenLogprob, ChatCompletionToolChoiceOption1, CompletionUsage, CreateChatCompletionRequest, CreateCompletionRequest, CreateEmbeddingRequest, - ChatCompletionTokenLogprob, EmbeddingUsage, Logprobs, TopLogprob, From b3e063201a8305a692378b77de5563a245bedd54 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 12:19:55 +0530 Subject: [PATCH 04/19] Update --- .../openai_frontend/engine/triton_engine.py | 8 +-- python/openai/tests/test_chat_completions.py | 24 +++++++- python/openai/tests/test_completions.py | 23 +++++++- python/openai/tests/test_openai_client.py | 55 +++++++++++++++---- 4 files changed, 88 insertions(+), 22 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index f075eb17bf..94077d93f1 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -822,11 +822,11 @@ def _validate_chat_request( raise ClientError("logit bias is not currently supported") # Logprobs are only supported for vLLM backend currently - if metadata.backend == "tensorrtllm" and ( + if metadata.backend != "vllm" and ( request.logprobs is not None or request.top_logprobs is not None ): raise ClientError( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" ) if request.top_logprobs and not request.logprobs: @@ -1003,10 +1003,10 @@ def _validate_completion_request( if ( request.logprobs is not None and request.logprobs > 0 - and metadata.backend == "tensorrtllm" + and metadata.backend != "vllm" ): raise ClientError( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" ) if request.stream_options and not request.stream: diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index fe86e90309..5db77862c0 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -556,11 +556,11 @@ def test_chat_completions_logprobs( }, ) - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": assert response.status_code == 400 assert ( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in response.json()["detail"] ) return @@ -633,6 +633,24 @@ def test_chat_completions_logprobs( in response.json()["detail"] ) + # Test that top_logprobs > 20 is rejected by schema validation + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "logprobs": True, + "top_logprobs": 25, # Exceeds maximum of 20 + "max_tokens": 5, + }, + ) + + # Should raise schema validation error + assert response.status_code == 422 + assert "Input should be less than or equal to 20" in str( + response.json()["detail"] + ) + # For tests that won't use the same pytest fixture for server startup across # the whole class test suite. diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 858b956477..403ae3b3d3 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -410,11 +410,11 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st }, ) - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": assert response.status_code == 400 assert ( - "logprobs are not currently supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in response.json()["detail"] ) return @@ -473,3 +473,20 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st # logprobs should be None when logprobs=0 choice = response_json["choices"][0] assert choice.get("logprobs") is None + + # Test that logprobs > 5 is rejected by schema validation + response = client.post( + "/v1/completions", + json={ + "model": model, + "prompt": prompt, + "logprobs": 7, + "max_tokens": 5, + }, + ) + + # Should raise schema validation error + assert response.status_code == 422 + assert "Input should be less than or equal to 5" in str( + response.json()["detail"] + ) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 8b3212471b..111dd55265 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -488,8 +488,8 @@ async def test_chat_completion_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] ): """Test logprobs for chat completions.""" - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": with pytest.raises(openai.BadRequestError) as exc_info: await client.chat.completions.create( model=model, @@ -499,7 +499,7 @@ async def test_chat_completion_logprobs( max_tokens=10, ) assert ( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in str(exc_info.value) ) return @@ -532,8 +532,8 @@ async def test_completion_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str ): """Test logprobs for completions.""" - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": with pytest.raises(openai.BadRequestError) as exc_info: await client.completions.create( model=model, @@ -542,7 +542,7 @@ async def test_completion_logprobs( max_tokens=10, ) assert ( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in str(exc_info.value) ) return @@ -573,8 +573,8 @@ async def test_chat_streaming_with_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] ): """Test streaming chat completions with logprobs.""" - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": with pytest.raises(openai.BadRequestError) as exc_info: stream = await client.chat.completions.create( model=model, @@ -588,7 +588,7 @@ async def test_chat_streaming_with_logprobs( async for _ in stream: pass assert ( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in str(exc_info.value) ) return @@ -618,8 +618,8 @@ async def test_completion_streaming_with_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str ): """Test streaming completions with logprobs.""" - # TRT-LLM should raise an error - if backend == "tensorrtllm": + # Non-vLLM backends should raise an error + if backend != "vllm": with pytest.raises(openai.BadRequestError) as exc_info: stream = await client.completions.create( model=model, @@ -632,7 +632,7 @@ async def test_completion_streaming_with_logprobs( async for _ in stream: pass assert ( - "logprobs are currently not supported for TensorRT-LLM backend" + "logprobs are currently available only for the vLLM backend" in str(exc_info.value) ) return @@ -675,3 +675,34 @@ async def test_top_logprobs_requires_logprobs( assert "`top_logprobs` can only be used when `logprobs` is True" in str( exc_info.value ) + + @pytest.mark.asyncio + async def test_chat_top_logprobs_exceeds_max( + self, client: openai.AsyncOpenAI, model: str, messages: List[dict] + ): + """Test that top_logprobs > 20 raises schema validation error.""" + with pytest.raises(openai.BadRequestError) as exc_info: + await client.chat.completions.create( + model=model, + messages=messages, + logprobs=True, + top_logprobs=25, # Exceeds maximum of 20 + max_tokens=5, + ) + # Pydantic validation error + assert "less than or equal to 20" in str(exc_info.value).lower() + + @pytest.mark.asyncio + async def test_completion_logprobs_exceeds_max( + self, client: openai.AsyncOpenAI, model: str, prompt: str + ): + """Test that logprobs > 5 raises schema validation error.""" + with pytest.raises(openai.BadRequestError) as exc_info: + await client.completions.create( + model=model, + prompt=prompt, + logprobs=7, # Exceeds maximum of 5 + max_tokens=5, + ) + # Pydantic validation error + assert "less than or equal to 5" in str(exc_info.value).lower() From 8d667e6e51ca6715b11ba2308262cb67209378c1 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 12:31:23 +0530 Subject: [PATCH 05/19] Update --- python/openai/tests/test_openai_client.py | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 111dd55265..671fddd2f8 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -498,9 +498,8 @@ async def test_chat_completion_logprobs( top_logprobs=2, max_tokens=10, ) - assert ( - "logprobs are currently available only for the vLLM backend" - in str(exc_info.value) + assert "logprobs are currently available only for the vLLM backend" in str( + exc_info.value ) return @@ -541,9 +540,8 @@ async def test_completion_logprobs( logprobs=3, max_tokens=10, ) - assert ( - "logprobs are currently available only for the vLLM backend" - in str(exc_info.value) + assert "logprobs are currently available only for the vLLM backend" in str( + exc_info.value ) return @@ -587,9 +585,8 @@ async def test_chat_streaming_with_logprobs( # Try to consume the stream (error should happen before streaming starts) async for _ in stream: pass - assert ( - "logprobs are currently available only for the vLLM backend" - in str(exc_info.value) + assert "logprobs are currently available only for the vLLM backend" in str( + exc_info.value ) return @@ -631,9 +628,8 @@ async def test_completion_streaming_with_logprobs( # Try to consume the stream (error should happen before streaming starts) async for _ in stream: pass - assert ( - "logprobs are currently available only for the vLLM backend" - in str(exc_info.value) + assert "logprobs are currently available only for the vLLM backend" in str( + exc_info.value ) return From 1a9f3e1c02ffa08c4b94ec42f79d1203caf8fccf Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 13:38:12 +0530 Subject: [PATCH 06/19] Update --- python/openai/tests/test_openai_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 671fddd2f8..03961c4a67 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -677,7 +677,7 @@ async def test_chat_top_logprobs_exceeds_max( self, client: openai.AsyncOpenAI, model: str, messages: List[dict] ): """Test that top_logprobs > 20 raises schema validation error.""" - with pytest.raises(openai.BadRequestError) as exc_info: + with pytest.raises(openai.UnprocessableEntityError) as exc_info: await client.chat.completions.create( model=model, messages=messages, @@ -693,7 +693,7 @@ async def test_completion_logprobs_exceeds_max( self, client: openai.AsyncOpenAI, model: str, prompt: str ): """Test that logprobs > 5 raises schema validation error.""" - with pytest.raises(openai.BadRequestError) as exc_info: + with pytest.raises(openai.UnprocessableEntityError) as exc_info: await client.completions.create( model=model, prompt=prompt, From ae8ab4dccb58e9d6bfa81f7d0e9d8e6f90147b69 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 17:41:43 +0530 Subject: [PATCH 07/19] Temp test --- .../vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json | 2 +- .../openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json index b7ce0ee199..d872f1cda8 100644 --- a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json +++ b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json @@ -1 +1 @@ -{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.9} \ No newline at end of file +{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.4} \ No newline at end of file diff --git a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json index df85a05da0..a441d12ca1 100644 --- a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json +++ b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json @@ -1 +1 @@ -{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.9} +{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.4} From 3fa4326f212b5b8492f86d0992162cc6655e2a3b Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 17:48:25 +0530 Subject: [PATCH 08/19] Undo temp --- .../vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json | 2 +- .../openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json index d872f1cda8..b7ce0ee199 100644 --- a/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json +++ b/python/openai/tests/vllm_mistral_models/mistral-nemo-instruct-2407/1/model.json @@ -1 +1 @@ -{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.4} \ No newline at end of file +{"model": "mistralai/Mistral-Nemo-Instruct-2407", "gpu_memory_utilization": 0.9} \ No newline at end of file diff --git a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json index a441d12ca1..df85a05da0 100644 --- a/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json +++ b/python/openai/tests/vllm_models/llama-3.1-8b-instruct/1/model.json @@ -1 +1 @@ -{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.4} +{"model": "meta-llama/Meta-Llama-3.1-8B-Instruct", "gpu_memory_utilization": 0.9} From e88a905185f30249b271070e4b819e08ab8f05b5 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 24 Nov 2025 18:11:30 +0530 Subject: [PATCH 09/19] Update --- .../openai_frontend/engine/triton_engine.py | 2 +- python/openai/tests/test_chat_completions.py | 32 ++++++++++--------- python/openai/tests/test_openai_client.py | 9 ++++-- 3 files changed, 25 insertions(+), 18 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 94077d93f1..88a6ec061f 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -829,7 +829,7 @@ def _validate_chat_request( "logprobs are currently available only for the vLLM backend" ) - if request.top_logprobs and not request.logprobs: + if request.top_logprobs is not None and not request.logprobs: raise ClientError("`top_logprobs` can only be used when `logprobs` is True") self._verify_chat_tool_call_settings(request=request) diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 5db77862c0..7c78814851 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -616,22 +616,24 @@ def test_chat_completions_logprobs( assert choice.get("logprobs") is None # Test that top_logprobs without logprobs raises an error - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "top_logprobs": 2, - "max_tokens": 10, - }, - ) + # Test multiple values including edge case top_logprobs=0 + for top_logprobs_value in [0, 2, 10]: + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "top_logprobs": top_logprobs_value, + "max_tokens": 10, + }, + ) - # Should raise validation error - assert response.status_code == 400 - assert ( - "`top_logprobs` can only be used when `logprobs` is True" - in response.json()["detail"] - ) + # Should raise validation error for any value when logprobs is not True + assert response.status_code == 400 + assert ( + "`top_logprobs` can only be used when `logprobs` is True" + in response.json()["detail"] + ) # Test that top_logprobs > 20 is rejected by schema validation response = client.post( diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 03961c4a67..c584f52a19 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -654,9 +654,14 @@ async def test_completion_streaming_with_logprobs( # At least some chunks should have logprobs assert chunks_with_logprobs > 0 + @pytest.mark.parametrize("top_logprobs_value", [0, 5]) @pytest.mark.asyncio async def test_top_logprobs_requires_logprobs( - self, client: openai.AsyncOpenAI, model: str, messages: List[dict] + self, + client: openai.AsyncOpenAI, + model: str, + messages: List[dict], + top_logprobs_value: int, ): """ Test that top_logprobs without logprobs raises an error @@ -665,7 +670,7 @@ async def test_top_logprobs_requires_logprobs( await client.chat.completions.create( model=model, messages=messages, - top_logprobs=2, # Without logprobs=True + top_logprobs=top_logprobs_value, # Without logprobs=True max_tokens=5, ) assert "`top_logprobs` can only be used when `logprobs` is True" in str( From 45e1687fd2863a09a82664967115178a077b94c8 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 25 Nov 2025 10:38:37 +0530 Subject: [PATCH 10/19] Update --- python/openai/openai_frontend/engine/utils/triton.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 395ac82eee..c74a0830f3 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -471,9 +471,6 @@ def _get_openai_chat_format_logprobs_from_vllm_response( token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) ) - if not sorted_tokens: - continue - # The first token (lowest rank) is the selected token selected_token_id, selected_token_data = sorted_tokens[0] selected_token = selected_token_data["decoded_token"] @@ -537,9 +534,6 @@ def _get_openai_completion_format_logprobs_from_vllm_response( token_logprobs_dict.items(), key=lambda x: x[1].get("rank", sys.maxsize) ) - if not sorted_tokens: - continue - # The first token (lowest rank) is the selected token selected_token_id, selected_token_data = sorted_tokens[0] selected_token = selected_token_data["decoded_token"] From 1d3cc0029579c91e48970d8dd562c3a4738d6713 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 10:47:38 +0530 Subject: [PATCH 11/19] Update --- .../openai_frontend/engine/triton_engine.py | 6 +- .../openai/openai_frontend/schemas/openai.py | 6 +- python/openai/tests/test_chat_completions.py | 96 ++++++----- python/openai/tests/test_completions.py | 66 +++---- python/openai/tests/test_openai_client.py | 161 +++++++++--------- 5 files changed, 173 insertions(+), 162 deletions(-) diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 88a6ec061f..4cbfe3bba7 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -68,6 +68,7 @@ from schemas.openai import ( ChatCompletionChoice, ChatCompletionFinishReason, + ChatCompletionLogprobs, ChatCompletionMessageToolCall, ChatCompletionMessageToolCallChunk, ChatCompletionNamedToolChoice, @@ -88,7 +89,6 @@ FinishReason, Function1, Function2, - Logprobs2, Model, ObjectType, ) @@ -265,7 +265,7 @@ async def chat( response ) if openai_logprobs: - logprobs_data = Logprobs2(content=openai_logprobs) + logprobs_data = ChatCompletionLogprobs(content=openai_logprobs) return CreateChatCompletionResponse( id=request_id, @@ -631,7 +631,7 @@ async def _streaming_chat_iterator( response ) if openai_logprobs: - chunk_logprobs = Logprobs2(content=openai_logprobs) + chunk_logprobs = ChatCompletionLogprobs(content=openai_logprobs) # if the response delta is None (e.g. because it was a # "control token" for tool calls or the parser otherwise diff --git a/python/openai/openai_frontend/schemas/openai.py b/python/openai/openai_frontend/schemas/openai.py index 81ff6e93b3..57fb7b1017 100644 --- a/python/openai/openai_frontend/schemas/openai.py +++ b/python/openai/openai_frontend/schemas/openai.py @@ -530,7 +530,7 @@ class ChatCompletionTokenLogprob(BaseModel): ) -class Logprobs2(BaseModel): +class ChatCompletionLogprobs(BaseModel): content: List[ChatCompletionTokenLogprob] = Field( ..., description="A list of message content tokens with log probability information.", @@ -539,7 +539,7 @@ class Logprobs2(BaseModel): class ChatCompletionStreamingResponseChoice(BaseModel): delta: ChatCompletionStreamResponseDelta - logprobs: Optional[Logprobs2] = Field( + logprobs: Optional[ChatCompletionLogprobs] = Field( None, description="Log probability information for the choice." ) finish_reason: ChatCompletionFinishReason | None = Field( @@ -730,7 +730,7 @@ class ChatCompletionChoice(BaseModel): ..., description="The index of the choice in the list of choices." ) message: ChatCompletionResponseMessage - logprobs: Logprobs2 | None = Field( + logprobs: ChatCompletionLogprobs | None = Field( ..., description="Log probability information for the choice." ) diff --git a/python/openai/tests/test_chat_completions.py b/python/openai/tests/test_chat_completions.py index 7c78814851..0dd55295d3 100644 --- a/python/openai/tests/test_chat_completions.py +++ b/python/openai/tests/test_chat_completions.py @@ -573,31 +573,34 @@ def test_chat_completions_logprobs( assert "logprobs" in choice logprobs = choice["logprobs"] - if logprobs is not None: - assert "content" in logprobs - content = logprobs["content"] - assert isinstance(content, list) - assert len(content) > 0 - - # Validate structure of each token logprob - for token_logprob in content: - assert "token" in token_logprob - assert "logprob" in token_logprob - assert "bytes" in token_logprob - assert "top_logprobs" in token_logprob - - assert isinstance(token_logprob["token"], str) - assert isinstance(token_logprob["logprob"], (int, float)) - assert isinstance(token_logprob["bytes"], list) - assert isinstance(token_logprob["top_logprobs"], list) - - # Validate top_logprobs structure - for top_logprob in token_logprob["top_logprobs"]: - assert "token" in top_logprob - assert "logprob" in top_logprob - assert "bytes" in top_logprob - - # Test that logprobs=False returns no logprobs + assert logprobs is not None + assert "content" in logprobs + content = logprobs["content"] + assert isinstance(content, list) + assert len(content) > 0 + + # Validate structure of each token logprob + for token_logprob in content: + assert "token" in token_logprob + assert "logprob" in token_logprob + assert "bytes" in token_logprob + assert "top_logprobs" in token_logprob + + assert isinstance(token_logprob["token"], str) + assert isinstance(token_logprob["logprob"], (int, float)) + assert isinstance(token_logprob["bytes"], list) + assert isinstance(token_logprob["top_logprobs"], list) + + # Validate top_logprobs structure + for top_logprob in token_logprob["top_logprobs"]: + assert "token" in top_logprob + assert "logprob" in top_logprob + assert "bytes" in top_logprob + + def test_chat_completions_logprobs_false( + self, client, model: str, messages: List[dict] + ): + """Test that logprobs=False returns no logprobs.""" response = client.post( "/v1/chat/completions", json={ @@ -615,27 +618,32 @@ def test_chat_completions_logprobs( choice = response_json["choices"][0] assert choice.get("logprobs") is None - # Test that top_logprobs without logprobs raises an error - # Test multiple values including edge case top_logprobs=0 - for top_logprobs_value in [0, 2, 10]: - response = client.post( - "/v1/chat/completions", - json={ - "model": model, - "messages": messages, - "top_logprobs": top_logprobs_value, - "max_tokens": 10, - }, - ) + @pytest.mark.parametrize("top_logprobs_value", [0, 5]) + def test_chat_completions_top_logprobs_without_logprobs( + self, client, model: str, messages: List[dict], top_logprobs_value: int + ): + """Test that top_logprobs without logprobs raises validation error.""" + response = client.post( + "/v1/chat/completions", + json={ + "model": model, + "messages": messages, + "top_logprobs": top_logprobs_value, + "max_tokens": 10, + }, + ) - # Should raise validation error for any value when logprobs is not True - assert response.status_code == 400 - assert ( - "`top_logprobs` can only be used when `logprobs` is True" - in response.json()["detail"] - ) + # Should raise validation error for any value when logprobs is not True + assert response.status_code == 400 + assert ( + "`top_logprobs` can only be used when `logprobs` is True" + in response.json()["detail"] + ) - # Test that top_logprobs > 20 is rejected by schema validation + def test_chat_completions_top_logprobs_validation( + self, client, model: str, messages: List[dict] + ): + """Test that top_logprobs > 20 is rejected by schema validation.""" response = client.post( "/v1/chat/completions", json={ diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 403ae3b3d3..ea1cf62166 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -427,36 +427,37 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st assert "logprobs" in choice logprobs = choice["logprobs"] - if logprobs is not None: - assert "text_offset" in logprobs - assert "token_logprobs" in logprobs - assert "tokens" in logprobs - assert "top_logprobs" in logprobs - - assert isinstance(logprobs["text_offset"], list) - assert isinstance(logprobs["token_logprobs"], list) - assert isinstance(logprobs["tokens"], list) - assert isinstance(logprobs["top_logprobs"], list) - - # All lists should have the same length - num_tokens = len(logprobs["tokens"]) - assert len(logprobs["text_offset"]) == num_tokens - assert len(logprobs["token_logprobs"]) == num_tokens - assert len(logprobs["top_logprobs"]) == num_tokens - - # Validate each token - for i in range(num_tokens): - assert isinstance(logprobs["tokens"][i], str) - assert isinstance(logprobs["token_logprobs"][i], (int, float)) - assert isinstance(logprobs["text_offset"][i], int) - assert isinstance(logprobs["top_logprobs"][i], dict) - - # Validate top_logprobs dict contains token -> logprob mappings - for token, logprob in logprobs["top_logprobs"][i].items(): - assert isinstance(token, str) - assert isinstance(logprob, (int, float)) - - # Test that logprobs=0 returns no logprobs + assert logprobs is not None + assert "text_offset" in logprobs + assert "token_logprobs" in logprobs + assert "tokens" in logprobs + assert "top_logprobs" in logprobs + + assert isinstance(logprobs["text_offset"], list) + assert isinstance(logprobs["token_logprobs"], list) + assert isinstance(logprobs["tokens"], list) + assert isinstance(logprobs["top_logprobs"], list) + + # All lists should have the same length + num_tokens = len(logprobs["tokens"]) + assert len(logprobs["text_offset"]) == num_tokens + assert len(logprobs["token_logprobs"]) == num_tokens + assert len(logprobs["top_logprobs"]) == num_tokens + + # Validate each token + for i in range(num_tokens): + assert isinstance(logprobs["tokens"][i], str) + assert isinstance(logprobs["token_logprobs"][i], (int, float)) + assert isinstance(logprobs["text_offset"][i], int) + assert isinstance(logprobs["top_logprobs"][i], dict) + + # Validate top_logprobs dict contains token -> logprob mappings + for token, logprob in logprobs["top_logprobs"][i].items(): + assert isinstance(token, str) + assert isinstance(logprob, (int, float)) + + def test_completions_logprobs_zero(self, client, model: str, prompt: str): + """Test that logprobs=0 returns no logprobs.""" response = client.post( "/v1/completions", json={ @@ -474,13 +475,14 @@ def test_completions_logprobs(self, client, backend: str, model: str, prompt: st choice = response_json["choices"][0] assert choice.get("logprobs") is None - # Test that logprobs > 5 is rejected by schema validation + def test_completions_logprobs_validation(self, client, model: str, prompt: str): + """Test that logprobs > 5 is rejected by schema validation.""" response = client.post( "/v1/completions", json={ "model": model, "prompt": prompt, - "logprobs": 7, + "logprobs": 7, # Exceeds maximum of 5 "max_tokens": 5, }, ) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index c584f52a19..66d5cec11b 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -487,7 +487,7 @@ async def test_stream_options_without_streaming( async def test_chat_completion_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] ): - """Test logprobs for chat completions.""" + """Test logprobs for chat completions and compare streaming vs non-streaming.""" # Non-vLLM backends should raise an error if backend != "vllm": with pytest.raises(openai.BadRequestError) as exc_info: @@ -503,12 +503,18 @@ async def test_chat_completion_logprobs( ) return + # Test non-streaming + seed = 0 + temperature = 0.0 chat_completion = await client.chat.completions.create( model=model, messages=messages, logprobs=True, top_logprobs=2, max_tokens=10, + temperature=temperature, + seed=seed, + stream=False, ) assert chat_completion.choices[0].message.content @@ -526,6 +532,40 @@ async def test_chat_completion_logprobs( assert token_logprob.top_logprobs is not None assert len(token_logprob.top_logprobs) > 0 + # Test streaming and compare with non-streaming + stream = await client.chat.completions.create( + model=model, + messages=messages, + logprobs=True, + top_logprobs=2, + max_tokens=10, + temperature=temperature, + seed=seed, + stream=True, + ) + + chunks = [] + stream_logprobs = [] + async for chunk in stream: + if chunk.choices[0].delta.content: + chunks.append(chunk.choices[0].delta.content) + if chunk.choices[0].logprobs and chunk.choices[0].logprobs.content: + stream_logprobs.extend(chunk.choices[0].logprobs.content) + + # Assert streaming output matches non-streaming + streamed_output = "".join(chunks) + assert streamed_output == chat_completion.choices[0].message.content + + # Assert logprobs from streaming match non-streaming + assert len(stream_logprobs) > 0, "Streaming should produce logprobs" + assert len(stream_logprobs) == len(logprobs.content) + for i, (stream_token, non_stream_token) in enumerate( + zip(stream_logprobs, logprobs.content) + ): + assert stream_token.token == non_stream_token.token + assert stream_token.logprob == non_stream_token.logprob + assert stream_token.bytes == non_stream_token.bytes + @pytest.mark.asyncio async def test_completion_logprobs( self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str @@ -545,11 +585,17 @@ async def test_completion_logprobs( ) return + # Test non-streaming + seed = 0 + temperature = 0.0 completion = await client.completions.create( model=model, prompt=prompt, logprobs=3, max_tokens=10, + temperature=temperature, + seed=seed, + stream=False, ) assert completion.choices[0].text @@ -566,93 +612,48 @@ async def test_completion_logprobs( assert len(logprobs.text_offset) == num_tokens assert len(logprobs.top_logprobs) == num_tokens - @pytest.mark.asyncio - async def test_chat_streaming_with_logprobs( - self, client: openai.AsyncOpenAI, backend: str, model: str, messages: List[dict] - ): - """Test streaming chat completions with logprobs.""" - # Non-vLLM backends should raise an error - if backend != "vllm": - with pytest.raises(openai.BadRequestError) as exc_info: - stream = await client.chat.completions.create( - model=model, - messages=messages, - max_tokens=10, - stream=True, - logprobs=True, - top_logprobs=2, - ) - # Try to consume the stream (error should happen before streaming starts) - async for _ in stream: - pass - assert "logprobs are currently available only for the vLLM backend" in str( - exc_info.value - ) - return - - stream = await client.chat.completions.create( - model=model, - messages=messages, - max_tokens=10, - stream=True, - logprobs=True, - top_logprobs=2, - ) - - chunks_with_logprobs = 0 - async for chunk in stream: - if chunk.choices[0].logprobs is not None: - chunks_with_logprobs += 1 - logprobs = chunk.choices[0].logprobs - assert logprobs.content is not None - assert len(logprobs.content) > 0 - - # At least some chunks should have logprobs - assert chunks_with_logprobs > 0 - - @pytest.mark.asyncio - async def test_completion_streaming_with_logprobs( - self, client: openai.AsyncOpenAI, backend: str, model: str, prompt: str - ): - """Test streaming completions with logprobs.""" - # Non-vLLM backends should raise an error - if backend != "vllm": - with pytest.raises(openai.BadRequestError) as exc_info: - stream = await client.completions.create( - model=model, - prompt=prompt, - max_tokens=10, - stream=True, - logprobs=3, - ) - # Try to consume the stream (error should happen before streaming starts) - async for _ in stream: - pass - assert "logprobs are currently available only for the vLLM backend" in str( - exc_info.value - ) - return - + # Test streaming and compare with non-streaming stream = await client.completions.create( model=model, prompt=prompt, + logprobs=3, max_tokens=10, + temperature=temperature, + seed=seed, stream=True, - logprobs=3, ) - chunks_with_logprobs = 0 + chunks = [] + stream_tokens = [] + stream_token_logprobs = [] + stream_text_offsets = [] + stream_top_logprobs = [] + async for chunk in stream: - if chunk.choices[0].logprobs is not None: - chunks_with_logprobs += 1 - logprobs = chunk.choices[0].logprobs - assert logprobs.tokens is not None - assert logprobs.token_logprobs is not None - assert logprobs.text_offset is not None - assert logprobs.top_logprobs is not None - - # At least some chunks should have logprobs - assert chunks_with_logprobs > 0 + if chunk.choices[0].text: + chunks.append(chunk.choices[0].text) + if chunk.choices[0].logprobs: + lp = chunk.choices[0].logprobs + if lp.tokens: + stream_tokens.extend(lp.tokens) + if lp.token_logprobs: + stream_token_logprobs.extend(lp.token_logprobs) + if lp.text_offset: + stream_text_offsets.extend(lp.text_offset) + if lp.top_logprobs: + stream_top_logprobs.extend(lp.top_logprobs) + + # Assert streaming output matches non-streaming + streamed_output = "".join(chunks) + assert streamed_output == completion.choices[0].text + + # Assert logprobs from streaming match non-streaming + assert len(stream_tokens) > 0, "Streaming should produce logprobs" + assert len(stream_tokens) == len(logprobs.tokens) + assert stream_tokens == logprobs.tokens + assert stream_token_logprobs == logprobs.token_logprobs + assert stream_text_offsets == logprobs.text_offset + assert stream_top_logprobs == logprobs.top_logprobs @pytest.mark.parametrize("top_logprobs_value", [0, 5]) @pytest.mark.asyncio From 3d616eae5675057215e377f401c8a79759f14bd6 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 15:05:30 +0530 Subject: [PATCH 12/19] Update --- .../openai/tests/test_models/mock_llm/config.pbtxt | 6 ++++++ python/openai/tests/test_openai_client.py | 13 +++++++++++-- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_models/mock_llm/config.pbtxt b/python/openai/tests/test_models/mock_llm/config.pbtxt index 5f665ff543..f0c51a3bc1 100644 --- a/python/openai/tests/test_models/mock_llm/config.pbtxt +++ b/python/openai/tests/test_models/mock_llm/config.pbtxt @@ -41,6 +41,12 @@ input [ name: "stream" data_type: TYPE_BOOL dims: [ 1, 1 ] + }, + { + name: "return_logprobs" + data_type: TYPE_BOOL + dims: [ 1, 1 ] + optional: true } ] diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 66d5cec11b..94e8dbb4b3 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -563,7 +563,10 @@ async def test_chat_completion_logprobs( zip(stream_logprobs, logprobs.content) ): assert stream_token.token == non_stream_token.token - assert stream_token.logprob == non_stream_token.logprob + # Use approximate comparison for floating point logprobs + assert ( + abs(stream_token.logprob - non_stream_token.logprob) < 1e-5 + ), f"Logprob mismatch: {stream_token.logprob} vs {non_stream_token.logprob}" assert stream_token.bytes == non_stream_token.bytes @pytest.mark.asyncio @@ -651,9 +654,15 @@ async def test_completion_logprobs( assert len(stream_tokens) > 0, "Streaming should produce logprobs" assert len(stream_tokens) == len(logprobs.tokens) assert stream_tokens == logprobs.tokens - assert stream_token_logprobs == logprobs.token_logprobs assert stream_text_offsets == logprobs.text_offset assert stream_top_logprobs == logprobs.top_logprobs + # Use approximate comparison for floating point logprobs + for stream_logprob, non_stream_logprob in zip( + stream_token_logprobs, logprobs.token_logprobs + ): + assert ( + abs(stream_logprob - non_stream_logprob) < 1e-5 + ), f"Logprob mismatch: {stream_logprob} vs {non_stream_logprob}" @pytest.mark.parametrize("top_logprobs_value", [0, 5]) @pytest.mark.asyncio From c0abe37e7def11427128eeaa3947e5f0e27f0160 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 15:06:51 +0530 Subject: [PATCH 13/19] Update --- python/openai/tests/test_models/mock_llm/config.pbtxt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/openai/tests/test_models/mock_llm/config.pbtxt b/python/openai/tests/test_models/mock_llm/config.pbtxt index f0c51a3bc1..cac10b5de0 100644 --- a/python/openai/tests/test_models/mock_llm/config.pbtxt +++ b/python/openai/tests/test_models/mock_llm/config.pbtxt @@ -1,4 +1,4 @@ -# Copyright 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions From d37d5853cc249b1ac159e0e10417aac7cbfd524f Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 19:16:13 +0530 Subject: [PATCH 14/19] Update --- python/openai/tests/test_openai_client.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 94e8dbb4b3..721189b654 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -563,9 +563,9 @@ async def test_chat_completion_logprobs( zip(stream_logprobs, logprobs.content) ): assert stream_token.token == non_stream_token.token - # Use approximate comparison for floating point logprobs + # Use approximate comparison for floating point logprobs (tolerance 0.001) assert ( - abs(stream_token.logprob - non_stream_token.logprob) < 1e-5 + abs(stream_token.logprob - non_stream_token.logprob) < 1e-3 ), f"Logprob mismatch: {stream_token.logprob} vs {non_stream_token.logprob}" assert stream_token.bytes == non_stream_token.bytes @@ -656,12 +656,12 @@ async def test_completion_logprobs( assert stream_tokens == logprobs.tokens assert stream_text_offsets == logprobs.text_offset assert stream_top_logprobs == logprobs.top_logprobs - # Use approximate comparison for floating point logprobs + # Use approximate comparison for floating point logprobs (tolerance 0.001) for stream_logprob, non_stream_logprob in zip( stream_token_logprobs, logprobs.token_logprobs ): assert ( - abs(stream_logprob - non_stream_logprob) < 1e-5 + abs(stream_logprob - non_stream_logprob) < 1e-3 ), f"Logprob mismatch: {stream_logprob} vs {non_stream_logprob}" @pytest.mark.parametrize("top_logprobs_value", [0, 5]) From fc8c51df2665dad78839c06262b204d7fc163d67 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 21:02:54 +0530 Subject: [PATCH 15/19] update --- python/openai/tests/test_openai_client.py | 47 ++++++++++++----------- 1 file changed, 24 insertions(+), 23 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 721189b654..13b278675e 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -556,18 +556,17 @@ async def test_chat_completion_logprobs( streamed_output = "".join(chunks) assert streamed_output == chat_completion.choices[0].message.content - # Assert logprobs from streaming match non-streaming + # Assert both streaming and non-streaming produce logprobs assert len(stream_logprobs) > 0, "Streaming should produce logprobs" - assert len(stream_logprobs) == len(logprobs.content) - for i, (stream_token, non_stream_token) in enumerate( - zip(stream_logprobs, logprobs.content) - ): - assert stream_token.token == non_stream_token.token - # Use approximate comparison for floating point logprobs (tolerance 0.001) - assert ( - abs(stream_token.logprob - non_stream_token.logprob) < 1e-3 - ), f"Logprob mismatch: {stream_token.logprob} vs {non_stream_token.logprob}" - assert stream_token.bytes == non_stream_token.bytes + assert len(stream_logprobs) == len(logprobs.content), "Same number of tokens" + + # Validate streaming logprobs structure + for stream_token in stream_logprobs: + assert stream_token.token + assert isinstance(stream_token.logprob, float) + assert isinstance(stream_token.bytes, list) + assert stream_token.top_logprobs is not None + assert len(stream_token.top_logprobs) > 0 @pytest.mark.asyncio async def test_completion_logprobs( @@ -650,19 +649,21 @@ async def test_completion_logprobs( streamed_output = "".join(chunks) assert streamed_output == completion.choices[0].text - # Assert logprobs from streaming match non-streaming + # Assert both streaming and non-streaming produce logprobs assert len(stream_tokens) > 0, "Streaming should produce logprobs" - assert len(stream_tokens) == len(logprobs.tokens) - assert stream_tokens == logprobs.tokens - assert stream_text_offsets == logprobs.text_offset - assert stream_top_logprobs == logprobs.top_logprobs - # Use approximate comparison for floating point logprobs (tolerance 0.001) - for stream_logprob, non_stream_logprob in zip( - stream_token_logprobs, logprobs.token_logprobs - ): - assert ( - abs(stream_logprob - non_stream_logprob) < 1e-3 - ), f"Logprob mismatch: {stream_logprob} vs {non_stream_logprob}" + assert len(stream_tokens) == len(logprobs.tokens), "Same number of tokens" + + # Validate streaming logprobs structure + assert len(stream_token_logprobs) == len(stream_tokens) + assert len(stream_text_offsets) == len(stream_tokens) + assert len(stream_top_logprobs) == len(stream_tokens) + + for i in range(len(stream_tokens)): + assert isinstance(stream_tokens[i], str) + assert isinstance(stream_token_logprobs[i], float) + assert isinstance(stream_text_offsets[i], int) + assert isinstance(stream_top_logprobs[i], dict) + assert len(stream_top_logprobs[i]) > 0 @pytest.mark.parametrize("top_logprobs_value", [0, 5]) @pytest.mark.asyncio From 3cf11faf6e7665baebef25358dd71cf957361621 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Thu, 27 Nov 2025 21:07:20 +0530 Subject: [PATCH 16/19] Update --- python/openai/tests/test_openai_client.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index 13b278675e..aa5f0093f4 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -559,7 +559,7 @@ async def test_chat_completion_logprobs( # Assert both streaming and non-streaming produce logprobs assert len(stream_logprobs) > 0, "Streaming should produce logprobs" assert len(stream_logprobs) == len(logprobs.content), "Same number of tokens" - + # Validate streaming logprobs structure for stream_token in stream_logprobs: assert stream_token.token @@ -652,12 +652,12 @@ async def test_completion_logprobs( # Assert both streaming and non-streaming produce logprobs assert len(stream_tokens) > 0, "Streaming should produce logprobs" assert len(stream_tokens) == len(logprobs.tokens), "Same number of tokens" - + # Validate streaming logprobs structure assert len(stream_token_logprobs) == len(stream_tokens) assert len(stream_text_offsets) == len(stream_tokens) assert len(stream_top_logprobs) == len(stream_tokens) - + for i in range(len(stream_tokens)): assert isinstance(stream_tokens[i], str) assert isinstance(stream_token_logprobs[i], float) From 9b10ef950de4ef548d3930d1a0520aba8e362f46 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Mon, 1 Dec 2025 11:34:39 +0530 Subject: [PATCH 17/19] Update --- python/openai/tests/test_openai_client.py | 40 ++++++++++------------- 1 file changed, 18 insertions(+), 22 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index aa5f0093f4..bc5ecf1a88 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -26,6 +26,7 @@ from typing import List +import numpy as np import openai import pytest @@ -560,13 +561,16 @@ async def test_chat_completion_logprobs( assert len(stream_logprobs) > 0, "Streaming should produce logprobs" assert len(stream_logprobs) == len(logprobs.content), "Same number of tokens" - # Validate streaming logprobs structure - for stream_token in stream_logprobs: - assert stream_token.token - assert isinstance(stream_token.logprob, float) - assert isinstance(stream_token.bytes, list) - assert stream_token.top_logprobs is not None - assert len(stream_token.top_logprobs) > 0 + # Compare tokens and logprob values (using np.allclose for float comparison) + stream_tokens_list = [t.token for t in stream_logprobs] + non_stream_tokens_list = [t.token for t in logprobs.content] + stream_logprobs_values = [t.logprob for t in stream_logprobs] + non_stream_logprobs_values = [t.logprob for t in logprobs.content] + + assert stream_tokens_list == non_stream_tokens_list, "Tokens should match" + assert np.allclose( + stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-3 + ), "Logprob values should be close" @pytest.mark.asyncio async def test_completion_logprobs( @@ -649,21 +653,13 @@ async def test_completion_logprobs( streamed_output = "".join(chunks) assert streamed_output == completion.choices[0].text - # Assert both streaming and non-streaming produce logprobs - assert len(stream_tokens) > 0, "Streaming should produce logprobs" - assert len(stream_tokens) == len(logprobs.tokens), "Same number of tokens" - - # Validate streaming logprobs structure - assert len(stream_token_logprobs) == len(stream_tokens) - assert len(stream_text_offsets) == len(stream_tokens) - assert len(stream_top_logprobs) == len(stream_tokens) - - for i in range(len(stream_tokens)): - assert isinstance(stream_tokens[i], str) - assert isinstance(stream_token_logprobs[i], float) - assert isinstance(stream_text_offsets[i], int) - assert isinstance(stream_top_logprobs[i], dict) - assert len(stream_top_logprobs[i]) > 0 + # Compare values (using np.allclose for float comparison) + assert stream_tokens == logprobs.tokens, "Tokens should match" + assert stream_text_offsets == logprobs.text_offset, "Text offsets should match" + assert stream_top_logprobs == logprobs.top_logprobs, "Top logprobs should match" + assert np.allclose( + stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-3 + ), "Token logprob values should be close" @pytest.mark.parametrize("top_logprobs_value", [0, 5]) @pytest.mark.asyncio From 2e075d7e4b07aaf9048dc33c11638bde2c298013 Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 2 Dec 2025 15:54:04 +0530 Subject: [PATCH 18/19] update --- python/openai/tests/test_openai_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index bc5ecf1a88..e1fa3fa197 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -569,7 +569,7 @@ async def test_chat_completion_logprobs( assert stream_tokens_list == non_stream_tokens_list, "Tokens should match" assert np.allclose( - stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-3 + stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=2e-1 ), "Logprob values should be close" @pytest.mark.asyncio @@ -658,7 +658,7 @@ async def test_completion_logprobs( assert stream_text_offsets == logprobs.text_offset, "Text offsets should match" assert stream_top_logprobs == logprobs.top_logprobs, "Top logprobs should match" assert np.allclose( - stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-3 + stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=2e-1 ), "Token logprob values should be close" @pytest.mark.parametrize("top_logprobs_value", [0, 5]) From d644d84d75438d07e0c9ff4d70c95610d084820e Mon Sep 17 00:00:00 2001 From: Sai Kiran Polisetty Date: Tue, 2 Dec 2025 15:20:30 +0000 Subject: [PATCH 19/19] Update --- python/openai/tests/test_openai_client.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/python/openai/tests/test_openai_client.py b/python/openai/tests/test_openai_client.py index e1fa3fa197..e5ce4ae55e 100644 --- a/python/openai/tests/test_openai_client.py +++ b/python/openai/tests/test_openai_client.py @@ -569,7 +569,7 @@ async def test_chat_completion_logprobs( assert stream_tokens_list == non_stream_tokens_list, "Tokens should match" assert np.allclose( - stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=2e-1 + stream_logprobs_values, non_stream_logprobs_values, rtol=0, atol=1e-2 ), "Logprob values should be close" @pytest.mark.asyncio @@ -658,7 +658,7 @@ async def test_completion_logprobs( assert stream_text_offsets == logprobs.text_offset, "Text offsets should match" assert stream_top_logprobs == logprobs.top_logprobs, "Top logprobs should match" assert np.allclose( - stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=2e-1 + stream_token_logprobs, logprobs.token_logprobs, rtol=0, atol=1e-2 ), "Token logprob values should be close" @pytest.mark.parametrize("top_logprobs_value", [0, 5])