diff --git a/python/openai/openai_frontend/engine/triton_engine.py b/python/openai/openai_frontend/engine/triton_engine.py index 0d735be5c0..d194fc2e1f 100644 --- a/python/openai/openai_frontend/engine/triton_engine.py +++ b/python/openai/openai_frontend/engine/triton_engine.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -991,6 +991,11 @@ def _validate_completion_request( if not isinstance(request.prompt, str): raise ClientError("only single string input is supported") + if "best_of" in request.model_fields_set and metadata.backend == "vllm": + raise ClientError( + "best_of is no longer supported in vLLM backend, removed from vLLM V1 engine" + ) + if request.n and request.n > 1: raise ClientError( f"Received n={request.n}, but only single choice (n=1) is currently supported" diff --git a/python/openai/openai_frontend/engine/utils/triton.py b/python/openai/openai_frontend/engine/utils/triton.py index 344ad374c9..8dd7663f3a 100644 --- a/python/openai/openai_frontend/engine/utils/triton.py +++ b/python/openai/openai_frontend/engine/utils/triton.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -100,6 +100,8 @@ def _create_vllm_generate_request( "max_tokens", "logprobs", "top_logprobs", + # not supported for vLLM backend (removed from vLLM V1) but supported for TRT-LLM/Python backend + "best_of", } # NOTE: The exclude_none is important, as internals may not support diff --git a/python/openai/requirements.txt b/python/openai/requirements.txt index cf63446574..4b4f631f9b 100644 --- a/python/openai/requirements.txt +++ b/python/openai/requirements.txt @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -31,6 +31,9 @@ fastapi==0.121.2 httpx==0.27.2 openai==1.107.3 partial-json-parser # used for parsing partial JSON outputs + +# FIXME [TRI-641]: The latest stable version of scipy is 1.17.0 which caused segfault during tests. See TRI-620. +scipy==1.16.3 # Minimum starlette version needed to address CVE(s): # https://github.com/advisories/GHSA-f96h-pmfr-66vw # https://github.com/advisories/GHSA-7f5h-v6xp-fcq8 diff --git a/python/openai/tests/conftest.py b/python/openai/tests/conftest.py index cff670f3ec..a1195000ed 100644 --- a/python/openai/tests/conftest.py +++ b/python/openai/tests/conftest.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -37,6 +37,7 @@ def pytest_configure(config): config.addinivalue_line( "markers", "openai: mark test to run with OpenAI server (subprocess)" ) + config.addinivalue_line("markers", "asyncio: mark test as an asyncio test") ### TEST ENVIRONMENT SETUP ### diff --git a/python/openai/tests/test_completions.py b/python/openai/tests/test_completions.py index 4dcad4c2d2..9d496476e1 100644 --- a/python/openai/tests/test_completions.py +++ b/python/openai/tests/test_completions.py @@ -1,4 +1,4 @@ -# Copyright 2024-2025, NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# Copyright 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. # # Redistribution and use in source and binary forms, with or without # modification, are permitted provided that the following conditions @@ -57,7 +57,6 @@ def test_completions_defaults(self, client, model: str, prompt: str): ("top_p", 0.9), ("frequency_penalty", 0.5), ("presence_penalty", 0.2), - ("best_of", 1), ("n", 1), # logprobs is an integer for completions ("logprobs", 5), @@ -359,7 +358,12 @@ def test_no_prompt(self, client, model: str): ], ) def test_completions_multiple_choices( - self, client, sampling_parameter_dict: dict, model: str, prompt: str + self, + client, + sampling_parameter_dict: dict, + backend: str, + model: str, + prompt: str, ): response = client.post( "/v1/completions", @@ -370,7 +374,11 @@ def test_completions_multiple_choices( # FIXME: Add support and test for success # Expected to fail when n or best_of > 1, only single choice supported for now assert response.status_code == 400 - assert "only single choice" in response.json()["detail"] + if backend == "vllm" and "best_of" in sampling_parameter_dict: + error_message = "best_of is no longer supported in vLLM backend" + else: + error_message = "only single choice" + assert error_message in response.json()["detail"] @pytest.mark.skip(reason="Not Implemented Yet") def test_lora(self):