fix(litellm): omit temperature for reasoning models

Share the OpenAI-compatible reasoning model detector between the native
OpenAI-compatible provider and LiteLLM-backed providers.

Skip forwarding explicit temperature values through LiteLLM for those
models, including Azure GPT-5 deployments, o-series models, and DeepSeek
reasoning routes.

Author: Sanderhoff-alt

hindsight-api-slim/hindsight_api/engine/providers/litellm_llm.py

@@ -24,13 +24,19 @@
 from hindsight_api.config import DEFAULT_LLM_TIMEOUT, ENV_LLM_TIMEOUT
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError
 from hindsight_api.engine.llm_trace import LLMResponseUsage, stash_response_usage
+from hindsight_api.engine.providers.model_capabilities import supports_openai_compatible_reasoning
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
 from hindsight_api.worker.stage import set_stage
 
 logger = logging.getLogger(__name__)
 
 
+def _model_rejects_temperature(model: str) -> bool:
+    """Return True for reasoning models that reject explicit temperature."""
+    return supports_openai_compatible_reasoning(model)
+
+
 def _usage_from_litellm_response(response: Any) -> LLMResponseUsage:
     """Extract prompt/completion/cached token counts from a LiteLLM (OpenAI-shaped) usage block."""
     usage = getattr(response, "usage", None)
@@ -144,12 +150,18 @@ def _build_common_kwargs(
         for key, value in self._extra_body.items():
             kwargs.setdefault(key, value)
 
+        if self._should_omit_temperature():
+            kwargs.pop("temperature", None)
+
         # Bedrock service tier: flex (50% cheaper), priority, or reserved
         if self.model.startswith("bedrock/") and self.bedrock_service_tier is not None:
             kwargs["service_tier"] = self.bedrock_service_tier
 
         return kwargs
 
+    def _should_omit_temperature(self) -> bool:
+        return _model_rejects_temperature(self.model)
+
     # ── per-model output-tokens cap (shared with Router subclass) ────────────
     # Hindsight's defaults (e.g. retain_max_completion_tokens=64000) target
     # high-capacity models. When a configured deployment supports fewer

hindsight-api-slim/hindsight_api/engine/providers/litellm_router_llm.py

@@ -36,7 +36,7 @@
 import logging
 from typing import Any
 
-from hindsight_api.engine.providers.litellm_llm import LiteLLMLLM
+from hindsight_api.engine.providers.litellm_llm import LiteLLMLLM, _model_rejects_temperature
 
 logger = logging.getLogger(__name__)
 
@@ -94,6 +94,7 @@ def __init__(
         # deployment Router picks. Uses LiteLLM's own per-model registry; unknown
         # models contribute no cap. See LiteLLMLLM._cap_max_completion_tokens.
         self._router_output_cap = self._compute_router_output_cap(config)
+        self._router_omits_temperature = self._config_has_temperature_rejecting_model(config)
 
         logger.info("LiteLLM Router initialized; entrypoint model_name=%r", _ENTRYPOINT_MODEL_NAME)
 
@@ -130,6 +131,19 @@ def _resolve_completion_model(self, response: Any) -> str:
     def _get_model_output_cap(self) -> int | None:
         return self._router_output_cap
 
+    def _should_omit_temperature(self) -> bool:
+        return bool(getattr(self, "_router_omits_temperature", False))
+
+    def _config_has_temperature_rejecting_model(self, config: dict[str, Any]) -> bool:
+        for deployment in (config.get("model_list") or []) if isinstance(config, dict) else []:
+            if not isinstance(deployment, dict):
+                continue
+            params = deployment.get("litellm_params") or {}
+            model_str = params.get("model") if isinstance(params, dict) else None
+            if model_str and _model_rejects_temperature(model_str):
+                return True
+        return False
+
     def _build_common_kwargs(
         self,
         messages: list[dict[str, Any]],
@@ -144,7 +158,7 @@ def _build_common_kwargs(
         }
         if max_completion_tokens is not None:
             kwargs["max_completion_tokens"] = self._cap_max_completion_tokens(max_completion_tokens)
-        if temperature is not None:
+        if temperature is not None and not self._should_omit_temperature():
             kwargs["temperature"] = temperature
         return kwargs
 

hindsight-api-slim/hindsight_api/engine/providers/model_capabilities.py

@@ -0,0 +1,11 @@
+"""Shared provider model capability helpers."""
+
+
+def supports_openai_compatible_reasoning(model: str) -> bool:
+    """Return True for OpenAI-compatible reasoning model names."""
+    model_lower = (model or "").lower()
+    if "deepseek" in model_lower:
+        # DeepSeek v4-flash is the non-thinking route. Treating every
+        # DeepSeek model as reasoning injects unsupported reasoning params.
+        return any(x in model_lower for x in ["v4-pro", "reasoner", "r1", "thinking"])
+    return any(x in model_lower for x in ["gpt-5", "o1", "o3"])

hindsight-api-slim/hindsight_api/engine/providers/openai_compatible_llm.py

@@ -38,6 +38,7 @@
 from hindsight_api.engine.bank_attribution import apply_bank_attribution
 from hindsight_api.engine.llm_interface import LLMInterface, OutputTooLongError, ProviderRateLimitResetError
 from hindsight_api.engine.llm_trace import LLMResponseUsage, stash_response_usage
+from hindsight_api.engine.providers.model_capabilities import supports_openai_compatible_reasoning
 from hindsight_api.engine.response_models import LLMToolCall, LLMToolCallResult, TokenUsage
 from hindsight_api.metrics import get_metrics_collector
 from hindsight_api.worker.stage import set_stage
@@ -594,13 +595,7 @@ async def verify_connection(self) -> None:
 
     def _supports_reasoning_model(self) -> bool:
         """Check if the current model is a reasoning model (o1, o3, GPT-5, DeepSeek)."""
-        model_lower = self.model.lower()
-        if "deepseek" in model_lower:
-            # DeepSeek v4-flash is the non-thinking route. Treating every
-            # DeepSeek model as a reasoning model injects reasoning_effort,
-            # which conflicts with thinking-disabled flash calls.
-            return any(x in model_lower for x in ["v4-pro", "reasoner", "r1", "thinking"])
-        return any(x in model_lower for x in ["gpt-5", "o1", "o3"])
+        return supports_openai_compatible_reasoning(self.model)
 
     def _get_max_reasoning_tokens(self) -> int | None:
         """Get max reasoning tokens for reasoning models."""

hindsight-api-slim/tests/test_llm_extra_body.py

@@ -348,15 +348,15 @@ class StructuredAnswer(BaseModel):
 # ─── LiteLLM ──────────────────────────────────────────────────────────────────
 
 
-def _make_litellm_provider(extra_body=None):
+def _make_litellm_provider(extra_body=None, model="gpt-4o"):
     pytest.importorskip("litellm")
     from hindsight_api.engine.providers.litellm_llm import LiteLLMLLM
 
     return LiteLLMLLM(
         provider="litellm",
         api_key="fake-key",
         base_url="",
-        model="gpt-4o",
+        model=model,
         extra_body=extra_body,
     )
 
@@ -404,6 +404,80 @@ async def test_litellm_explicit_param_wins_over_extra_body():
     assert provider._acompletion.call_args.kwargs.get("temperature") == 0.9
 
 
+@pytest.mark.asyncio
+async def test_litellm_gpt5_omits_explicit_temperature():
+    provider = _make_litellm_provider(model="azure/gpt-5.5")
+    provider._acompletion = AsyncMock(return_value=_fake_litellm_response())
+
+    with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            temperature=0.1,
+            scope="test",
+            max_retries=0,
+        )
+
+    assert "temperature" not in provider._acompletion.call_args.kwargs
+
+
+@pytest.mark.asyncio
+async def test_litellm_o_series_omits_explicit_temperature():
+    provider = _make_litellm_provider(model="openai/o3-mini")
+    provider._acompletion = AsyncMock(return_value=_fake_litellm_response())
+
+    with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            temperature=0.1,
+            scope="test",
+            max_retries=0,
+        )
+
+    assert "temperature" not in provider._acompletion.call_args.kwargs
+
+
+@pytest.mark.asyncio
+async def test_litellm_deepseek_reasoning_omits_temperature_but_flash_keeps_it():
+    reasoner = _make_litellm_provider(model="deepseek/deepseek-reasoner")
+    reasoner._acompletion = AsyncMock(return_value=_fake_litellm_response())
+    flash = _make_litellm_provider(model="deepseek/deepseek-v4-flash")
+    flash._acompletion = AsyncMock(return_value=_fake_litellm_response())
+
+    with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
+        await reasoner.call(
+            messages=[{"role": "user", "content": "hi"}],
+            temperature=0.1,
+            scope="test",
+            max_retries=0,
+        )
+        await flash.call(
+            messages=[{"role": "user", "content": "hi"}],
+            temperature=0.1,
+            scope="test",
+            max_retries=0,
+        )
+
+    assert "temperature" not in reasoner._acompletion.call_args.kwargs
+    assert flash._acompletion.call_args.kwargs.get("temperature") == 0.1
+
+
+@pytest.mark.asyncio
+async def test_litellm_gpt5_omits_extra_body_temperature():
+    provider = _make_litellm_provider(extra_body=EXTRA_BODY, model="azure/gpt-5.5")
+    provider._acompletion = AsyncMock(return_value=_fake_litellm_response())
+
+    with patch("hindsight_api.engine.providers.litellm_llm.get_metrics_collector"):
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            scope="test",
+            max_retries=0,
+        )
+
+    kwargs = provider._acompletion.call_args.kwargs
+    assert "temperature" not in kwargs
+    assert kwargs.get("top_p") == 0.9
+
+
 def test_litellm_router_forwards_extra_body():
     """The Router subclass forwards extra_body through to the shared LiteLLM base."""
     pytest.importorskip("litellm")

hindsight-api-slim/tests/test_llm_router_provider.py

@@ -314,6 +314,31 @@ async def test_no_cap_when_litellm_registry_has_no_data(self, two_step_config, m
         kwargs = mock_router.acompletion.await_args.kwargs
         assert kwargs["max_completion_tokens"] == 64000
 
+    @pytest.mark.asyncio
+    async def test_gpt5_deployment_omits_temperature(self, mock_router_response):
+        mock_router = MagicMock()
+        mock_router.acompletion = AsyncMock(return_value=mock_router_response)
+        config = {
+            "model_list": [
+                {
+                    "model_name": "default",
+                    "litellm_params": {"model": "azure/gpt-5.5", "api_key": "sk-primary"},
+                }
+            ]
+        }
+        provider = _make_router_provider(config, mock_router)
+        provider._router_omits_temperature = provider._config_has_temperature_rejecting_model(config)
+
+        await provider.call(
+            messages=[{"role": "user", "content": "hi"}],
+            temperature=0.1,
+            max_retries=0,
+        )
+
+        kwargs = mock_router.acompletion.await_args.kwargs
+        assert kwargs["model"] == "default"
+        assert "temperature" not in kwargs
+
     @pytest.mark.asyncio
     async def test_call_with_tools(self, two_step_config):
         response = MagicMock()