fix(serve): filter HF generation config and fix request extract merge

Author: lvhan028

lmdeploy/serve/core/generation_config.py

@@ -7,10 +7,17 @@
 import dataclasses
 from typing import Any
 
+from pydantic import BaseModel
+
 from lmdeploy.messages import GenerationConfig
 from lmdeploy.utils import get_logger
 
 logger = get_logger('lmdeploy')
+_GENERATION_CONFIG_FIELDS = {field.name for field in dataclasses.fields(GenerationConfig)}
+
+
+def _filter_gen_config(config: dict[str, Any]) -> dict[str, Any]:
+    return {key: value for key, value in config.items() if key in _GENERATION_CONFIG_FIELDS}
 
 
 def _load_hf_generation_config(path: str, trust_remote_code: bool) -> dict[str, Any]:
@@ -37,6 +44,7 @@ def resolve_default_gen_config(
         config = _load_hf_generation_config(model_path, trust_remote_code)
     else:
         config = _load_hf_generation_config(src, trust_remote_code)
+    config = _filter_gen_config(config)
 
     if config and src != 'lmdeploy':
         source = "the model's `generation_config.json`" if src == 'auto' else src
@@ -61,20 +69,19 @@ def merge_gen_config(
     return merged
 
 
-def extract_request_gen_config(request: Any) -> dict[str, Any]:
-    """Extract non-None GenerationConfig fields present on the request."""
-    values: dict[str, Any] = {}
-    for field in dataclasses.fields(GenerationConfig):
-        if not hasattr(request, field.name):
-            continue
-        value = getattr(request, field.name)
-        if value is not None:
-            values[field.name] = value
-    return values
+def extract_request_gen_config(request: BaseModel) -> dict[str, Any]:
+    """Extract explicit non-None GenerationConfig fields from a request."""
+    # exclude_unset keeps client-supplied fields plus parser-updated fields,
+    # while leaving plain Pydantic defaults available for server defaults.
+    return {
+        key: value
+        for key, value in request.model_dump(exclude_unset=True).items()
+        if key in _GENERATION_CONFIG_FIELDS and value is not None
+    }
 
 
 def build_generation_config(
-    request: Any,
+    request: BaseModel,
     default_gen_config: dict[str, Any],
     *,
     max_new_tokens: int | None = None,
@@ -85,7 +92,7 @@ def build_generation_config(
     request_gen_config = extract_request_gen_config(request)
     for key in extra_kwargs:
         request_gen_config.pop(key, None)
-    merged = merge_gen_config(request_gen_config, default_gen_config)
+    merged = merge_gen_config(request_gen_config, _filter_gen_config(default_gen_config))
     merged.pop('max_new_tokens', None)
     merged.pop('do_sample', None)
     return GenerationConfig(

lmdeploy/serve/openai/protocol.py

@@ -549,12 +549,12 @@ class GenerateReqInput(BaseModel):
     stop: str | list[str] | None = None
     stop_token_ids: list[int] | None = None
     stream: bool | None = False
-    temperature: float = 1.0
+    temperature: float | None = None
     repetition_penalty: float | None = None
     ignore_eos: bool | None = False
-    top_p: float = 1.0
-    top_k: int = 0
-    min_p: float = 0.0
+    top_p: float | None = None
+    top_k: int | None = None
+    min_p: float | None = None
     skip_special_tokens: bool | None = True
     spaces_between_special_tokens: bool | None = True
     include_stop_str_in_output: bool | None = False

lmdeploy/serve/openai/serving_generate.py

@@ -35,11 +35,11 @@ def check_request(request: GenerateReqInput, server_context: 'VariableInterface'
         return f'The session_id {request.session_id!r} is occupied.'
 
     # check sampling settings
-    if not (0 < request.top_p <= 1):
+    if request.top_p is not None and not (0 < request.top_p <= 1):
         return f'The top_p {request.top_p!r} must be in (0, 1].'
-    if request.top_k < 0:
+    if request.top_k is not None and request.top_k < 0:
         return f'The top_k {request.top_k!r} cannot be a negative integer.'
-    if not (0 <= request.temperature <= 2):
+    if request.temperature is not None and not (0 <= request.temperature <= 2):
         return f'The temperature {request.temperature!r} must be in [0, 2]'
 
     return ''

tests/test_lmdeploy/serve/test_generation_config.py

@@ -1,4 +1,5 @@
 # Copyright (c) OpenMMLab. All rights reserved.
+import warnings
 from unittest.mock import patch
 
 from lmdeploy.messages import GenerationConfig
@@ -8,11 +9,31 @@
     merge_gen_config,
     resolve_default_gen_config,
 )
-from lmdeploy.serve.openai.protocol import ChatCompletionRequest, CompletionRequest
+from lmdeploy.serve.openai.protocol import ChatCompletionRequest, CompletionRequest, GenerateReqInput
+from lmdeploy.serve.openai.serving_generate import check_request as check_generate_request
 
 _DEFAULTS = GenerationConfig()
 
 
+class _FakeEngineConfig:
+    logprobs_mode = None
+
+
+class _FakeSessionManager:
+
+    def has(self, session_id):
+        return False
+
+
+class _FakeServerContext:
+
+    def get_engine_config(self):
+        return _FakeEngineConfig()
+
+    def get_session_manager(self):
+        return _FakeSessionManager()
+
+
 def test_merge_gen_config_priority():
     merged = merge_gen_config(
         {'temperature': 0.2},
@@ -26,7 +47,7 @@ def test_merge_gen_config_uses_server_defaults():
     assert merged == {'temperature': 0.5}
 
 
-def test_extract_request_gen_config_only_non_null():
+def test_extract_request_gen_config_only_explicit_fields():
     request = ChatCompletionRequest(model='test', messages='hi', temperature=0.3)
     values = extract_request_gen_config(request)
     assert values == {'temperature': 0.3}
@@ -58,12 +79,65 @@ def test_build_generation_config_uses_generation_config_defaults():
     assert gen_config.top_k == _DEFAULTS.top_k
 
 
+def test_build_generation_config_ignores_unsupported_defaults():
+    request = CompletionRequest(model='test', prompt='hello')
+    gen_config = build_generation_config(
+        request,
+        {
+            'temperature': 0.6,
+            'eos_token_id': 2,
+            'pad_token_id': 0,
+            'transformers_version': '5.12.1',
+        },
+    )
+    assert gen_config.temperature == 0.6
+
+
+def test_completion_request_max_tokens_is_optional():
+    request = CompletionRequest(model='test', prompt='hello')
+    with warnings.catch_warnings():
+        warnings.simplefilter('ignore', DeprecationWarning)
+        assert request.max_tokens is None
+
+
+def test_generate_request_sampling_defaults_match_chat_request():
+    chat_request = ChatCompletionRequest(model='test', messages='hello')
+    generate_request = GenerateReqInput(prompt='hello')
+    for name in ('temperature', 'top_p', 'top_k', 'min_p'):
+        assert getattr(generate_request, name) == getattr(chat_request, name)
+
+
+def test_generate_request_accepts_none_sampling_defaults():
+    request = GenerateReqInput(prompt='hello')
+    assert check_generate_request(request, _FakeServerContext()) == ''
+
+
+def test_generate_request_sampling_merge_uses_server_defaults():
+    request = GenerateReqInput(prompt='hello')
+    gen_config = build_generation_config(
+        request,
+        {
+            'temperature': 0.2,
+            'top_p': 0.3,
+            'top_k': 7,
+            'min_p': 0.1,
+        },
+        max_new_tokens=request.max_tokens,
+    )
+    assert gen_config.temperature == 0.2
+    assert gen_config.top_p == 0.3
+    assert gen_config.top_k == 7
+    assert gen_config.min_p == 0.1
+
+
 @patch('lmdeploy.serve.core.generation_config._load_hf_generation_config')
 def test_resolve_default_gen_config_auto(mock_load):
     mock_load.return_value = {
         'temperature': 0.6,
         'top_p': 0.8,
         'max_new_tokens': 2048,
+        'eos_token_id': 2,
+        'transformers_version': '5.12.1',
     }
     config = resolve_default_gen_config('auto', '/fake/model', False)
     assert config == {