Merge pull request lightspeed-core#200 from asamal4/dynamic-llm-param

[LEADS-211] feat: ability to add/remove model parameters from config

Author: asamal4

docs/configuration.md

@@ -26,18 +26,31 @@ Define a centralized pool of LLM configurations for the Judge Panel feature.
 | Setting | Description |
 |---------|-------------|
 | `llm_pool.defaults.cache_dir` | Cache directory (default: `.caches/llm_cache`) |
-| `llm_pool.defaults.timeout` | Request timeout in seconds |
-| `llm_pool.defaults.num_retries` | Retry attempts |
+| `llm_pool.defaults.timeout` | Request timeout in seconds (default: `300`) |
+| `llm_pool.defaults.num_retries` | Retry attempts (default: `3`) |
 | `llm_pool.defaults.parameters.temperature` | Sampling temperature |
 | `llm_pool.defaults.parameters.max_completion_tokens` | Max tokens in response |
+| `llm_pool.defaults.parameters.*` | Any additional provider-supported LLM parameter (e.g., `top_p`, `frequency_penalty`) |
 | `llm_pool.models.<id>.provider` | LLM provider (required) |
 | `llm_pool.models.<id>.model` | Model name |
-| `llm_pool.models.<id>.parameters.*` | Model-specific parameter overrides |
+| `llm_pool.models.<id>.parameters.*` | Model-specific parameter overrides (merged with defaults, model takes priority) |
+
+**Dynamic Parameters:** The `parameters` dict accepts any key-value pair supported by the LLM provider. Known parameters: `temperature`, `max_completion_tokens`. Unsupported parameters are silently dropped by the provider.
+
+**Removing Defaults:** To remove an inherited default parameter, set it to `null` in the model config:
+```yaml
+models:
+  no-temp-model:
+    provider: openai
+    parameters:
+      temperature: null  # Removes default temperature, won't be sent to API
+```
 
 ```yaml
 llm_pool:
   defaults:
     cache_dir: ".caches/llm_cache"
+    num_retries: 2
     parameters:
       temperature: 0.0
       max_completion_tokens: 512
@@ -48,6 +61,8 @@ llm_pool:
     judge-4.1-mini:
       provider: openai
       model: gpt-4.1-mini
+      parameters:
+        temperature: 0.3                # Overrides default
 ```
 
 ## Judge Panel
@@ -81,8 +96,11 @@ judge_panel:
 
 ---
 
-## Judge LLM configuration
-This section configures LLM as a judge for both Ragas and DeepEval.
+## Judge LLM configuration (legacy)
+
+> **Deprecation notice:** The `llm` section will be replaced by `llm_pool` + `judge_panel`.
+
+This section configures LLM. It is used when `judge_panel` is not configured (even if `llm_pool` exists).
 
 ### LLM
 | Setting (llm.) | Default | Description |
@@ -98,6 +116,8 @@ This section configures LLM as a judge for both Ragas and DeepEval.
 | cache_dir | `".caches/llm_cache"` | Directory with cached LLM responses |
 | cache_enabled | `true` | Is LLM cache enabled? |
 
+Dynamic LLM parameters are only supported through `llm_pool` config. To use dynamic parameters, migrate to `llm_pool`.
+
 **Note**: For RHAIIS, models.corp, or other vLLM-based inference servers, use the `hosted_vllm` provider configuration. `models.corp` additionally requires certificate setup via `ssl_cert_file` configuration option.
 
 ### Embeddings

src/lightspeed_evaluation/core/llm/custom.py

@@ -1,7 +1,7 @@
 """Base Custom LLM class for evaluation framework."""
 
 import logging
-from typing import Any, Optional, Union
+from typing import Any, Union
 
 import litellm
 from litellm.exceptions import InternalServerError
@@ -16,7 +16,13 @@ class BaseCustomLLM:  # pylint: disable=too-few-public-methods
     """Base LLM class with core calling functionality."""
 
     def __init__(self, model_name: str, llm_params: dict[str, Any]):
-        """Initialize with model configuration."""
+        """Initialize with model configuration.
+
+        Args:
+            model_name: Constructed model name (e.g., "openai/gpt-4")
+            llm_params: LLM params from get_llm_params() — operational fields
+                plus a "parameters" dict with inference params.
+        """
         self.model_name = model_name
         self.llm_params = llm_params
 
@@ -29,7 +35,6 @@ def call(
         self,
         prompt: str,
         n: int = 1,
-        temperature: Optional[float] = None,
         return_single: bool = True,
         **kwargs: Any,
     ) -> Union[str, list[str]]:
@@ -38,27 +43,20 @@ def call(
         Args:
             prompt: Text prompt to send
             n: Number of responses to generate (default 1)
-            temperature: Override temperature (uses config default if None)
             return_single: If True and n=1, return single string. If False, always return list.
             **kwargs: Additional LLM parameters
 
         Returns:
             Single string if return_single=True and n=1, otherwise list of strings
         """
-        temp = (
-            temperature
-            if temperature is not None
-            else self.llm_params.get("temperature", 0.0)
-        )
-
+        # Note: Forbidden keys are rejected at LLMParametersConfig load time
         call_params = {
             "model": self.model_name,
             "messages": [{"role": "user", "content": prompt}],
-            "temperature": temp,
             "n": n,
-            "max_completion_tokens": self.llm_params.get("max_completion_tokens"),
             "timeout": self.llm_params.get("timeout"),
-            "num_retries": self.llm_params.get("num_retries", 3),
+            "num_retries": self.llm_params.get("num_retries"),
+            **self.llm_params.get("parameters", {}),
             **kwargs,
         }
 

src/lightspeed_evaluation/core/llm/deepeval.py

@@ -23,7 +23,13 @@ class DeepEvalLLMManager:
     """
 
     def __init__(self, model_name: str, llm_params: dict[str, Any]):
-        """Initialize with LLM parameters from LLMManager."""
+        """Initialize with LLM parameters from LLMManager.
+
+        Args:
+            model_name: Constructed model name (e.g., "openai/gpt-4")
+            llm_params: LLM params from get_llm_params() — operational fields
+                plus a "parameters" dict with inference params.
+        """
         self.model_name = model_name
         self.llm_params = llm_params
 
@@ -34,14 +40,16 @@ def __init__(self, model_name: str, llm_params: dict[str, Any]):
 
         # Note: Token tracking is handled by the patched litellm.completion/acompletion
         # No additional setup needed - the patch was applied at module import time
-
         # Create standard LiteLLMModel - it will use our patched completion functions
+
+        # LiteLLMModel stores **kwargs in self.kwargs and merges them into
+        # every litellm.completion() call
+        # Note: Forbidden keys are rejected at LLMParametersConfig load time
         self.llm_model = LiteLLMModel(
             model=self.model_name,
-            temperature=llm_params.get("temperature", 0.0),
-            max_completion_tokens=llm_params.get("max_completion_tokens"),
-            timeout=llm_params.get("timeout"),
-            num_retries=llm_params.get("num_retries", 3),
+            timeout=self.llm_params.get("timeout"),
+            num_retries=self.llm_params.get("num_retries"),
+            **self.llm_params.get("parameters", {}),
         )
 
         print(f"✅ DeepEval LLM Manager: {self.model_name}")
@@ -65,13 +73,12 @@ def get_llm(self) -> LiteLLMModel:
 
     def get_model_info(self) -> dict[str, Any]:
         """Get information about the configured model."""
-        return {
-            "model_name": self.model_name,
-            "temperature": self.llm_params.get("temperature", 0.0),
-            "max_completion_tokens": self.llm_params.get("max_completion_tokens"),
-            "timeout": self.llm_params.get("timeout"),
-            "num_retries": self.llm_params.get("num_retries", 3),
-        }
+        info: dict[str, Any] = {"model_name": self.model_name}
+        info["timeout"] = self.llm_params.get("timeout")
+        info["num_retries"] = self.llm_params.get("num_retries")
+        # Add inference params (forbidden keys rejected at config load time)
+        info.update(self.llm_params.get("parameters", {}))
+        return info
 
     @staticmethod
     def flush_deepevals_pending_tasks() -> None:

src/lightspeed_evaluation/core/llm/manager.py

@@ -209,15 +209,18 @@ def get_model_name(self) -> str:
         return self.model_name
 
     def get_llm_params(self) -> dict[str, Any]:
-        """Get parameters for LLM completion calls."""
+        """Get all LLM parameters for adapter consumption.
+
+        Returns operational params and inference parameters dict.
+        Adapters read operational fields directly and spread **parameters
+        for inference params (temperature, max_completion_tokens, etc.).
+        """
         return {
             "model": self.model_name,
-            "temperature": self.config.temperature,
-            # Map max_tokens to max_completion_tokens for LLM API
-            "max_completion_tokens": self.config.max_tokens,
             "timeout": self.config.timeout,
             "num_retries": self.config.num_retries,
             "ssl_verify": self.config.ssl_verify,
+            "parameters": dict(self.config.parameters),
         }
 
     def get_config(self) -> LLMConfig:

src/lightspeed_evaluation/core/llm/ragas.py

@@ -42,24 +42,24 @@ def __init__(self, llm_manager: LLMManager):
         # Setup SSL verification for litellm
         setup_litellm_ssl(self.llm_params)
 
-        # Extract inference kwargs to pass to ragas llm_factory
-        # Filter out None values to avoid overriding library defaults unnecessarily
-        inference_kwargs: dict[str, Any] = {}
-        for key in ("temperature", "timeout", "num_retries"):
-            if self.llm_params.get(key) is not None:
-                inference_kwargs[key] = self.llm_params[key]
-
+        # Build inference kwargs from parameters
         # Rename max_completion_tokens to max_tokens for ragas/instructor compatibility
         # (OpenAI rejects requests with both set simultaneously)
-        if self.llm_params.get("max_completion_tokens") is not None:
-            inference_kwargs["max_tokens"] = self.llm_params["max_completion_tokens"]
+        # Note: Forbidden keys are rejected at LLMParametersConfig load time
+        inference_kwargs = dict(self.llm_params.get("parameters", {}))
+        if "max_completion_tokens" in inference_kwargs:
+            inference_kwargs["max_tokens"] = inference_kwargs.pop(
+                "max_completion_tokens"
+            )
 
         # Create LLM using ragas llm_factory with litellm provider
         # MUST use acompletion (async) because ragas 0.4 metrics use async internally
         self.llm = llm_factory(
-            model=self.model_name,
             provider="litellm",
             client=litellm.acompletion,
+            model=self.model_name,
+            timeout=self.llm_params.get("timeout"),
+            num_retries=self.llm_params.get("num_retries"),
             **inference_kwargs,
         )
 
@@ -75,7 +75,9 @@ def get_llm(self) -> Any:
 
     def get_model_info(self) -> dict[str, Any]:
         """Get information about the configured model."""
-        return {
-            "model_name": self.model_name,
-            "temperature": self.llm_params.get("temperature", 0.0),
-        }
+        params = self.llm_params.get("parameters", {})
+        info: dict[str, Any] = {"model_name": self.model_name}
+        # Only include temperature if explicitly set (not removed via null)
+        if "temperature" in params:
+            info["temperature"] = params["temperature"]
+        return info

src/lightspeed_evaluation/core/models/system.py

@@ -46,6 +46,27 @@
 )
 from lightspeed_evaluation.core.system.exceptions import ConfigurationError
 
+# Keys not allowed in llm_pool.parameters
+# These are either request-envelope fields or managed via dedicated config fields
+FORBIDDEN_PARAMETER_KEYS = frozenset(
+    {
+        # Request envelope fields
+        "model",
+        "messages",
+        "n",
+        # Operational fields (set via dedicated config)
+        "timeout",
+        "num_retries",
+        "ssl_verify",
+        "ssl_cert_file",
+        "cache_enabled",
+        "cache_dir",
+        # Provider/client fields
+        "provider",
+        "client",
+    }
+)
+
 
 class LLMConfig(BaseModel):
     """LLM configuration from system configuration."""
@@ -119,6 +140,35 @@ def validate_ssl_cert_file(self) -> "LLMConfig":
     cache_enabled: bool = Field(
         default=True, description="Is caching of 'LLM as a judge' queries enabled?"
     )
+    parameters: dict[str, Any] = Field(
+        default_factory=dict,
+        description="Internal: dynamic LLM parameters for API calls",
+    )
+
+    @model_validator(mode="before")
+    @classmethod
+    def strip_user_parameters(cls, data: Any) -> Any:
+        """Strip parameters from YAML/dict input.
+
+        Dynamic parameters via YAML are only supported through llm_pool.
+        For legacy llm:, parameters is always built from explicit fields.
+        """
+        if isinstance(data, dict):
+            data.pop("parameters", None)
+        return data
+
+    @model_validator(mode="after")
+    def build_parameters_from_fields(self) -> "LLMConfig":
+        """Build parameters dict from explicit temperature and max_tokens.
+
+        For the pool path, resolve_llm_config() overrides this via
+        model_copy(update=...) after construction.
+        """
+        self.parameters = {
+            "temperature": self.temperature,
+            "max_completion_tokens": self.max_tokens,
+        }
+        return self
 
 
 class EmbeddingConfig(BaseModel):
@@ -410,19 +460,37 @@ class LLMParametersConfig(BaseModel):
         description="Maximum tokens in response",
     )
 
+    @model_validator(mode="before")
+    @classmethod
+    def reject_forbidden_keys(cls, data: Any) -> Any:
+        """Reject keys that must be set via dedicated config fields, not parameters."""
+        if not isinstance(data, dict):
+            return data
+        forbidden_found = FORBIDDEN_PARAMETER_KEYS & data.keys()
+        if forbidden_found:
+            raise ConfigurationError(
+                f"Keys not allowed in parameters: {forbidden_found}. "
+                f"Use dedicated config fields instead."
+            )
+        return data
+
     def to_dict(self, exclude_none: bool = True) -> dict[str, Any]:
         """Convert parameters to dict for passing to LLM.
 
         Args:
-            exclude_none: If True, exclude None values from output
+            exclude_none: If True, exclude None values from output.
+                If False, include only explicitly set fields (uses model_fields_set
+                to distinguish user-provided None from unset defaults).
 
         Returns:
             Dict of parameters ready for LLM API call
         """
-        params = self.model_dump()
         if exclude_none:
-            return {k: v for k, v in params.items() if v is not None}
-        return params
+            return {k: v for k, v in self.model_dump().items() if v is not None}
+        # Include only fields the user explicitly set (including None overrides)
+        return {
+            k: v for k, v in self.model_dump().items() if k in self.model_fields_set
+        }
 
 
 class LLMDefaultsConfig(BaseModel):
@@ -581,16 +649,19 @@ def resolve_llm_config(
             )
         entry = self.models[model_id]
 
-        # Merge parameters: defaults -> model entry
+        # Merge parameters: defaults -> individual (individual overrides defaults).
+        # None in individual explicitly removes the default's value.
+        # Note: Forbidden keys are rejected at LLMParametersConfig load time.
         merged_params: dict[str, Any] = {}
         merged_params.update(self.defaults.parameters.to_dict(exclude_none=True))
-        merged_params.update(entry.parameters.to_dict(exclude_none=True))
+        merged_params.update(entry.parameters.to_dict(exclude_none=False))
+        merged_params = {k: v for k, v in merged_params.items() if v is not None}
 
         # Build cache_dir from defaults with model-specific suffix
         suffix = cache_suffix if cache_suffix else model_id
         cache_dir = os.path.join(self.defaults.cache_dir, suffix)
 
-        return LLMConfig(
+        config = LLMConfig(
             provider=entry.provider,
             model=entry.model or model_id,
             temperature=merged_params.get("temperature", DEFAULT_LLM_TEMPERATURE),
@@ -609,6 +680,7 @@ def resolve_llm_config(
             cache_dir=cache_dir,
             # Note: api_base and api_key_path are not propagated yet - requires LLMConfig extension
         )
+        return config.model_copy(update={"parameters": merged_params})
 
 
 class JudgePanelConfig(BaseModel):