From 66918a9c0bf39a91982175aad89ad1ce4ecfbb2c Mon Sep 17 00:00:00 2001 From: Chibi Vikram Date: Wed, 29 Jul 2026 12:08:43 -0700 Subject: [PATCH 1/2] fix(platform): drop temperature for models that reject it LLM-as-judge evals on claude-sonnet-5 failed with HTTP 400 "`temperature` is deprecated for this model" while agent executions on the same model passed. LLM Gateway discovery publishes modelDetails.shouldSkipTemperature per model. uipath-langchain-python honors it, so agent runs omit the parameter. Both chat_completions paths here always sent it, so every judge call on such a model failed. In staging, all 137 normalized-route 400s over 24h carried x-uipath-agenthub-config: agentsevals; none came from agent execution. Resolve the flag from discovery (cached per base_url + agenthub config) and omit temperature when the model reports it. Unknown models and unreachable discovery keep the caller's parameters, so behaviour is unchanged wherever the flag is not set. Discovery is resolved through EndpointManager so it follows the same agenthub_/orchestrator_ routing as the completion it describes. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018nEN5r7dtNaEkN9y5ygXCT --- .../platform/chat/_llm_gateway_service.py | 21 +- .../platform/chat/_model_capabilities.py | 98 +++++++ .../platform/common/_endpoints_manager.py | 10 + .../services/test_llm_temperature_skip.py | 245 ++++++++++++++++++ .../tests/cli/eval/mocks/test_input_mocker.py | 7 + 5 files changed, 377 insertions(+), 4 deletions(-) create mode 100644 packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py create mode 100644 packages/uipath-platform/tests/services/test_llm_temperature_skip.py diff --git a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py index 00e373d60..65ec6223a 100644 --- a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py +++ b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py @@ -29,6 +29,7 @@ from ..common._endpoints_manager import EndpointManager from ..common._execution_context import UiPathExecutionContext from ..common._models import Endpoint +from ._model_capabilities import should_skip_temperature from .llm_gateway import ( ChatCompletion, SpecificToolChoice, @@ -175,6 +176,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -329,13 +331,18 @@ class Country(BaseModel): is_reasoning_model = model.lower().startswith(("o1", "o3", "o4")) + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) + request_body: dict[str, Any] = { "messages": messages, "max_tokens": max_tokens, } - # Reasoning models (o1, o3, o4) don't support temperature - if not is_reasoning_model: + if not skip_temperature: request_body["temperature"] = temperature # Handle response_format - convert BaseModel to schema if needed @@ -392,6 +399,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -558,13 +566,18 @@ class Country(BaseModel): is_claude_model = "claude" in model_lower is_reasoning_model = model_lower.startswith(("o1", "o3", "o4")) + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) + request_body: dict[str, Any] = { "messages": converted_messages, "max_tokens": max_tokens, } - # Reasoning models (o1, o3, o4) don't support temperature and sampling parameters - if not is_reasoning_model: + if not skip_temperature: request_body["temperature"] = temperature # Only add OpenAI-specific parameters for non-Claude and non-reasoning models diff --git a/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py new file mode 100644 index 000000000..4a435d117 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py @@ -0,0 +1,98 @@ +"""Per-model parameter support, resolved from the LLM Gateway discovery API. + +Newer models reject parameters that older ones accept: passing `temperature` to +a model that dropped it fails the whole request with HTTP 400. The gateway +publishes these constraints per model under `modelDetails`, so callers can strip +the parameter instead of guessing from the model name. +""" + +import logging +from typing import Any + +from ..common._base_service import BaseService +from ..common._endpoints_manager import EndpointManager +from ..common._models import Endpoint +from ..constants import HEADER_AGENTHUB_CONFIG + +logger = logging.getLogger(__name__) + +CacheKey = tuple[str, str | None] + +# One fetch per (base_url, agenthub_config); an empty mapping caches "couldn't read +# discovery". Deliberately unlocked: a racing duplicate GET is cheaper than an +# asyncio.Lock pinned to whichever event loop touched it first. +_model_details: dict[CacheKey, dict[str, dict[str, Any]]] = {} + + +async def should_skip_temperature( + service: BaseService, model: str, agenthub_config: str | None +) -> bool: + """Whether `model` rejects the `temperature` parameter. + + Args: + service: Service used to call discovery; supplies base URL and auth. + model: Model name as sent to the gateway. + agenthub_config: AgentHub config scoping which models are visible. + + Returns: + True only when discovery explicitly reports the model skips temperature. + Unknown models, unreachable discovery, and any unexpected failure return + False, leaving the caller's request untouched. + """ + try: + details = await _details_for(service, model, agenthub_config) + except Exception as e: + logger.warning( + "Could not resolve parameter support for model %s (%s); " + "sending model parameters unchanged", + model, + e, + ) + return False + + return bool(details.get("shouldSkipTemperature", False)) + + +async def _details_for( + service: BaseService, model: str, agenthub_config: str | None +) -> dict[str, Any]: + key = (service._config.base_url, agenthub_config) + + if key not in _model_details: + _model_details[key] = await _fetch_model_details(service, agenthub_config) + + return _model_details[key].get(model, {}) + + +async def _fetch_model_details( + service: BaseService, agenthub_config: str | None +) -> dict[str, dict[str, Any]]: + headers = {HEADER_AGENTHUB_CONFIG: agenthub_config} if agenthub_config else {} + endpoint = Endpoint("/" + EndpointManager.get_discovery_endpoint()) + + try: + response = await service.request_async("GET", endpoint, headers=headers) + models = response.json() + except Exception as e: + # Falling back to the caller's parameters is the pre-existing behaviour; + # failing the LLM call because discovery is down would be worse. + logger.warning( + "LLM Gateway discovery unavailable (%s); sending model parameters unchanged", + e, + ) + return {} + + if not isinstance(models, list): + logger.warning("Unexpected LLM Gateway discovery payload; expected a list") + return {} + + return { + model["modelName"]: model.get("modelDetails") or {} + for model in models + if isinstance(model, dict) and model.get("modelName") + } + + +def _reset_cache() -> None: + """Clear the discovery cache. For tests.""" + _model_details.clear() diff --git a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py index 044ab1393..8e3b436ac 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py +++ b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py @@ -21,6 +21,7 @@ class UiPathEndpoints(Enum): "agenthub_/llm/raw/vendor/{vendor}/model/{model}/completions" ) AH_CAPABILITIES_ENDPOINT = "agenthub_/llm/api/capabilities" + AH_DISCOVERY_ENDPOINT = "agenthub_/llm/api/discovery" OR_NORMALIZED_COMPLETION_ENDPOINT = "orchestrator_/llm/api/chat/completions" OR_PASSTHROUGH_COMPLETION_ENDPOINT = "orchestrator_/llm/openai/deployments/{model}/chat/completions?api-version={api_version}" @@ -29,6 +30,7 @@ class UiPathEndpoints(Enum): "orchestrator_/llm/raw/vendor/{vendor}/model/{model}/completions" ) OR_CAPABILITIES_ENDPOINT = "orchestrator_/llm/api/capabilities" + OR_DISCOVERY_ENDPOINT = "orchestrator_/llm/api/discovery" class EndpointManager: @@ -185,6 +187,14 @@ def get_normalized_endpoint(cls) -> str: UiPathEndpoints.OR_NORMALIZED_COMPLETION_ENDPOINT, ) + @classmethod + def get_discovery_endpoint(cls) -> str: + """Get the model discovery endpoint.""" + return cls._select_endpoint( + UiPathEndpoints.AH_DISCOVERY_ENDPOINT, + UiPathEndpoints.OR_DISCOVERY_ENDPOINT, + ) + @classmethod def get_embeddings_endpoint(cls) -> str: """Get the embeddings endpoint.""" diff --git a/packages/uipath-platform/tests/services/test_llm_temperature_skip.py b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py new file mode 100644 index 000000000..fd3aef196 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py @@ -0,0 +1,245 @@ +"""Temperature is dropped for models that reject it, per LLM Gateway discovery. + +Regression coverage for Sonnet 5 evals: the judge sent `temperature` on every +normalized chat completion, and the gateway answered 400 "`temperature` is +deprecated for this model." +""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.chat import UiPathLlmChatService, UiPathOpenAIService +from uipath.platform.chat._model_capabilities import _reset_cache + +SKIP_TEMPERATURE_MODEL = "anthropic.claude-sonnet-5" + +SKIP_TEMPERATURE_DISCOVERY = [ + { + "modelName": SKIP_TEMPERATURE_MODEL, + "modelDetails": {"shouldSkipTemperature": True}, + } +] + + +def _discovery_response(models: list[dict[str, Any]]) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = models + return response + + +def _completion_response(model: str) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + return response + + +def _responder(discovery: list[dict[str, Any]] | Exception, model: str = "test-model"): + """Answer by endpoint, so assertions don't depend on call ordering.""" + + async def respond(_method, endpoint, **_kwargs): + if "discovery" in str(endpoint): + if isinstance(discovery, Exception): + raise discovery + return _discovery_response(discovery) + return _completion_response(model) + + return respond + + +def _is_discovery(call) -> bool: + return "discovery" in str(call.args[1]) + + +def _discovery_calls(mock_request: MagicMock) -> list[Any]: + return [c for c in mock_request.call_args_list if _is_discovery(c)] + + +def _completion_body(mock_request: MagicMock) -> dict[str, Any]: + completions = [c for c in mock_request.call_args_list if not _is_discovery(c)] + assert completions, "no chat completion request was sent" + return completions[-1][1]["json"] + + +class TestSkipTemperatureFromDiscovery: + @pytest.fixture(autouse=True) + def clear_discovery_cache(self): + # The cache is process-wide; keep it from leaking in either direction. + _reset_cache() + yield + _reset_cache() + + @pytest.fixture + def config(self): + return UiPathApiConfig(base_url="https://example.com", secret="test_secret") + + @pytest.fixture + def execution_context(self): + return UiPathExecutionContext() + + @pytest.fixture + def llm_service(self, config, execution_context): + return UiPathLlmChatService(config=config, execution_context=execution_context) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_omits_temperature_when_discovery_sets_flag( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_flag_absent(self, mock_request, llm_service): + mock_request.side_effect = _responder( + [ + { + "modelName": "gpt-4o-2024-11-20", + "modelDetails": {"maxOutputTokens": 4096}, + } + ], + "gpt-4o-2024-11-20", + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4o-2024-11-20", + temperature=0.3, + ) + + assert _completion_body(mock_request)["temperature"] == 0.3 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_for_model_missing_from_discovery( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, "some-other-model" + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="some-other-model", + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_discovery_unavailable( + self, mock_request, llm_service + ): + """Discovery being down must not fail or silently alter the LLM call.""" + mock_request.side_effect = _responder( + RuntimeError("discovery unreachable"), SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_fetched_once_for_repeated_calls( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + for _ in range(2): + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert len(_discovery_calls(mock_request)) == 1 + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_scoped_by_agenthub_config( + self, mock_request, config, execution_context + ): + service = UiPathLlmChatService( + config=config, + execution_context=execution_context, + agenthub_config="agentsevals", + ) + mock_request.side_effect = _responder([], SKIP_TEMPERATURE_MODEL) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + ) + + discovery_headers = _discovery_calls(mock_request)[0][1]["headers"] + assert discovery_headers["x-uipath-agenthub-config"] == "agentsevals" + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathOpenAIService.request_async" + ) + @pytest.mark.asyncio + async def test_openai_compatible_path_omits_temperature( + self, mock_request, config, execution_context + ): + service = UiPathOpenAIService( + config=config, execution_context=execution_context + ) + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) diff --git a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py index a8a8a64ec..7abdf78b1 100644 --- a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py +++ b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py @@ -68,6 +68,13 @@ async def test_generate_llm_input_with_model_settings( json={}, ) + # Chat completions consults discovery to learn which parameters the model accepts. + httpx_mock.add_response( + url="https://example.com/llm/api/discovery", + status_code=200, + json=[{"modelName": "gpt-4o-mini-2024-07-18", "modelDetails": {}}], + ) + httpx_mock.add_response( url="https://example.com/llm/api/chat/completions" "?api-version=2024-08-01-preview", From 29fc56c45b4ce64c84a8e750634b7322146a541b Mon Sep 17 00:00:00 2001 From: Chibi Vikram Date: Wed, 29 Jul 2026 12:41:08 -0700 Subject: [PATCH 2/2] chore: release uipath-platform 0.2.14, uipath 2.13.19 Ship the temperature fix and make it unskippable downstream: raise uipath's lower bound on uipath-platform to 0.2.14 so no install of uipath can resolve a uipath-platform that still sends temperature to models that reject it. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_018nEN5r7dtNaEkN9y5ygXCT --- packages/uipath-platform/pyproject.toml | 2 +- packages/uipath-platform/uv.lock | 2 +- packages/uipath/pyproject.toml | 4 ++-- packages/uipath/uv.lock | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/uipath-platform/pyproject.toml b/packages/uipath-platform/pyproject.toml index bbf16c8ca..d973884de 100644 --- a/packages/uipath-platform/pyproject.toml +++ b/packages/uipath-platform/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "uipath-platform" -version = "0.2.13" +version = "0.2.14" description = "HTTP client library for programmatic access to UiPath Platform" readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" diff --git a/packages/uipath-platform/uv.lock b/packages/uipath-platform/uv.lock index 295f77671..80451fb6f 100644 --- a/packages/uipath-platform/uv.lock +++ b/packages/uipath-platform/uv.lock @@ -1095,7 +1095,7 @@ dev = [ [[package]] name = "uipath-platform" -version = "0.2.13" +version = "0.2.14" source = { editable = "." } dependencies = [ { name = "anyio" }, diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 6ba25b021..bb91f7c6f 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "uipath" -version = "2.13.18" +version = "2.13.19" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ "uipath-core>=0.5.30, <0.6.0", "uipath-runtime>=0.12.2, <0.13.0", - "uipath-platform>=0.2.4, <0.3.0", + "uipath-platform>=0.2.14, <0.3.0", "click>=8.3.1", "httpx>=0.28.1", "pyjwt>=2.10.1", diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index ab4fcb07b..fbd18e154 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.18" +version = "2.13.19" source = { editable = "." } dependencies = [ { name = "applicationinsights" }, @@ -2760,7 +2760,7 @@ wheels = [ [[package]] name = "uipath-platform" -version = "0.2.13" +version = "0.2.14" source = { editable = "../uipath-platform" } dependencies = [ { name = "anyio" },