diff --git a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py index 00e373d60..65ec6223a 100644 --- a/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py +++ b/packages/uipath-platform/src/uipath/platform/chat/_llm_gateway_service.py @@ -29,6 +29,7 @@ from ..common._endpoints_manager import EndpointManager from ..common._execution_context import UiPathExecutionContext from ..common._models import Endpoint +from ._model_capabilities import should_skip_temperature from .llm_gateway import ( ChatCompletion, SpecificToolChoice, @@ -175,6 +176,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -329,13 +331,18 @@ class Country(BaseModel): is_reasoning_model = model.lower().startswith(("o1", "o3", "o4")) + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) + request_body: dict[str, Any] = { "messages": messages, "max_tokens": max_tokens, } - # Reasoning models (o1, o3, o4) don't support temperature - if not is_reasoning_model: + if not skip_temperature: request_body["temperature"] = temperature # Handle response_format - convert BaseModel to schema if needed @@ -392,6 +399,7 @@ def __init__( action_id: Optional[str] = None, ) -> None: super().__init__(config=config, execution_context=execution_context) + self._agenthub_config = agenthub_config self._llm_headers = _build_llm_headers( requesting_product, requesting_feature, agenthub_config, action_id ) @@ -558,13 +566,18 @@ class Country(BaseModel): is_claude_model = "claude" in model_lower is_reasoning_model = model_lower.startswith(("o1", "o3", "o4")) + # Reasoning models (o1, o3, o4) don't support temperature; newer models + # reject it too, which only discovery knows about. + skip_temperature = is_reasoning_model or await should_skip_temperature( + self, model, self._agenthub_config + ) + request_body: dict[str, Any] = { "messages": converted_messages, "max_tokens": max_tokens, } - # Reasoning models (o1, o3, o4) don't support temperature and sampling parameters - if not is_reasoning_model: + if not skip_temperature: request_body["temperature"] = temperature # Only add OpenAI-specific parameters for non-Claude and non-reasoning models diff --git a/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py new file mode 100644 index 000000000..4a435d117 --- /dev/null +++ b/packages/uipath-platform/src/uipath/platform/chat/_model_capabilities.py @@ -0,0 +1,98 @@ +"""Per-model parameter support, resolved from the LLM Gateway discovery API. + +Newer models reject parameters that older ones accept: passing `temperature` to +a model that dropped it fails the whole request with HTTP 400. The gateway +publishes these constraints per model under `modelDetails`, so callers can strip +the parameter instead of guessing from the model name. +""" + +import logging +from typing import Any + +from ..common._base_service import BaseService +from ..common._endpoints_manager import EndpointManager +from ..common._models import Endpoint +from ..constants import HEADER_AGENTHUB_CONFIG + +logger = logging.getLogger(__name__) + +CacheKey = tuple[str, str | None] + +# One fetch per (base_url, agenthub_config); an empty mapping caches "couldn't read +# discovery". Deliberately unlocked: a racing duplicate GET is cheaper than an +# asyncio.Lock pinned to whichever event loop touched it first. +_model_details: dict[CacheKey, dict[str, dict[str, Any]]] = {} + + +async def should_skip_temperature( + service: BaseService, model: str, agenthub_config: str | None +) -> bool: + """Whether `model` rejects the `temperature` parameter. + + Args: + service: Service used to call discovery; supplies base URL and auth. + model: Model name as sent to the gateway. + agenthub_config: AgentHub config scoping which models are visible. + + Returns: + True only when discovery explicitly reports the model skips temperature. + Unknown models, unreachable discovery, and any unexpected failure return + False, leaving the caller's request untouched. + """ + try: + details = await _details_for(service, model, agenthub_config) + except Exception as e: + logger.warning( + "Could not resolve parameter support for model %s (%s); " + "sending model parameters unchanged", + model, + e, + ) + return False + + return bool(details.get("shouldSkipTemperature", False)) + + +async def _details_for( + service: BaseService, model: str, agenthub_config: str | None +) -> dict[str, Any]: + key = (service._config.base_url, agenthub_config) + + if key not in _model_details: + _model_details[key] = await _fetch_model_details(service, agenthub_config) + + return _model_details[key].get(model, {}) + + +async def _fetch_model_details( + service: BaseService, agenthub_config: str | None +) -> dict[str, dict[str, Any]]: + headers = {HEADER_AGENTHUB_CONFIG: agenthub_config} if agenthub_config else {} + endpoint = Endpoint("/" + EndpointManager.get_discovery_endpoint()) + + try: + response = await service.request_async("GET", endpoint, headers=headers) + models = response.json() + except Exception as e: + # Falling back to the caller's parameters is the pre-existing behaviour; + # failing the LLM call because discovery is down would be worse. + logger.warning( + "LLM Gateway discovery unavailable (%s); sending model parameters unchanged", + e, + ) + return {} + + if not isinstance(models, list): + logger.warning("Unexpected LLM Gateway discovery payload; expected a list") + return {} + + return { + model["modelName"]: model.get("modelDetails") or {} + for model in models + if isinstance(model, dict) and model.get("modelName") + } + + +def _reset_cache() -> None: + """Clear the discovery cache. For tests.""" + _model_details.clear() diff --git a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py index 044ab1393..8e3b436ac 100644 --- a/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py +++ b/packages/uipath-platform/src/uipath/platform/common/_endpoints_manager.py @@ -21,6 +21,7 @@ class UiPathEndpoints(Enum): "agenthub_/llm/raw/vendor/{vendor}/model/{model}/completions" ) AH_CAPABILITIES_ENDPOINT = "agenthub_/llm/api/capabilities" + AH_DISCOVERY_ENDPOINT = "agenthub_/llm/api/discovery" OR_NORMALIZED_COMPLETION_ENDPOINT = "orchestrator_/llm/api/chat/completions" OR_PASSTHROUGH_COMPLETION_ENDPOINT = "orchestrator_/llm/openai/deployments/{model}/chat/completions?api-version={api_version}" @@ -29,6 +30,7 @@ class UiPathEndpoints(Enum): "orchestrator_/llm/raw/vendor/{vendor}/model/{model}/completions" ) OR_CAPABILITIES_ENDPOINT = "orchestrator_/llm/api/capabilities" + OR_DISCOVERY_ENDPOINT = "orchestrator_/llm/api/discovery" class EndpointManager: @@ -185,6 +187,14 @@ def get_normalized_endpoint(cls) -> str: UiPathEndpoints.OR_NORMALIZED_COMPLETION_ENDPOINT, ) + @classmethod + def get_discovery_endpoint(cls) -> str: + """Get the model discovery endpoint.""" + return cls._select_endpoint( + UiPathEndpoints.AH_DISCOVERY_ENDPOINT, + UiPathEndpoints.OR_DISCOVERY_ENDPOINT, + ) + @classmethod def get_embeddings_endpoint(cls) -> str: """Get the embeddings endpoint.""" diff --git a/packages/uipath-platform/tests/services/test_llm_temperature_skip.py b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py new file mode 100644 index 000000000..fd3aef196 --- /dev/null +++ b/packages/uipath-platform/tests/services/test_llm_temperature_skip.py @@ -0,0 +1,245 @@ +"""Temperature is dropped for models that reject it, per LLM Gateway discovery. + +Regression coverage for Sonnet 5 evals: the judge sent `temperature` on every +normalized chat completion, and the gateway answered 400 "`temperature` is +deprecated for this model." +""" + +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + +from uipath.platform import UiPathApiConfig, UiPathExecutionContext +from uipath.platform.chat import UiPathLlmChatService, UiPathOpenAIService +from uipath.platform.chat._model_capabilities import _reset_cache + +SKIP_TEMPERATURE_MODEL = "anthropic.claude-sonnet-5" + +SKIP_TEMPERATURE_DISCOVERY = [ + { + "modelName": SKIP_TEMPERATURE_MODEL, + "modelDetails": {"shouldSkipTemperature": True}, + } +] + + +def _discovery_response(models: list[dict[str, Any]]) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = models + return response + + +def _completion_response(model: str) -> MagicMock: + response = MagicMock() + response.status_code = 200 + response.json.return_value = { + "id": "chatcmpl-test", + "object": "chat.completion", + "created": 1234567890, + "model": model, + "choices": [ + { + "index": 0, + "message": {"role": "assistant", "content": "Hello"}, + "finish_reason": "stop", + } + ], + "usage": {"prompt_tokens": 10, "completion_tokens": 5, "total_tokens": 15}, + } + return response + + +def _responder(discovery: list[dict[str, Any]] | Exception, model: str = "test-model"): + """Answer by endpoint, so assertions don't depend on call ordering.""" + + async def respond(_method, endpoint, **_kwargs): + if "discovery" in str(endpoint): + if isinstance(discovery, Exception): + raise discovery + return _discovery_response(discovery) + return _completion_response(model) + + return respond + + +def _is_discovery(call) -> bool: + return "discovery" in str(call.args[1]) + + +def _discovery_calls(mock_request: MagicMock) -> list[Any]: + return [c for c in mock_request.call_args_list if _is_discovery(c)] + + +def _completion_body(mock_request: MagicMock) -> dict[str, Any]: + completions = [c for c in mock_request.call_args_list if not _is_discovery(c)] + assert completions, "no chat completion request was sent" + return completions[-1][1]["json"] + + +class TestSkipTemperatureFromDiscovery: + @pytest.fixture(autouse=True) + def clear_discovery_cache(self): + # The cache is process-wide; keep it from leaking in either direction. + _reset_cache() + yield + _reset_cache() + + @pytest.fixture + def config(self): + return UiPathApiConfig(base_url="https://example.com", secret="test_secret") + + @pytest.fixture + def execution_context(self): + return UiPathExecutionContext() + + @pytest.fixture + def llm_service(self, config, execution_context): + return UiPathLlmChatService(config=config, execution_context=execution_context) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_omits_temperature_when_discovery_sets_flag( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_flag_absent(self, mock_request, llm_service): + mock_request.side_effect = _responder( + [ + { + "modelName": "gpt-4o-2024-11-20", + "modelDetails": {"maxOutputTokens": 4096}, + } + ], + "gpt-4o-2024-11-20", + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="gpt-4o-2024-11-20", + temperature=0.3, + ) + + assert _completion_body(mock_request)["temperature"] == 0.3 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_for_model_missing_from_discovery( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, "some-other-model" + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model="some-other-model", + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_keeps_temperature_when_discovery_unavailable( + self, mock_request, llm_service + ): + """Discovery being down must not fail or silently alter the LLM call.""" + mock_request.side_effect = _responder( + RuntimeError("discovery unreachable"), SKIP_TEMPERATURE_MODEL + ) + + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert _completion_body(mock_request)["temperature"] == 0 + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_fetched_once_for_repeated_calls( + self, mock_request, llm_service + ): + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + for _ in range(2): + await llm_service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert len(_discovery_calls(mock_request)) == 1 + assert "temperature" not in _completion_body(mock_request) + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathLlmChatService.request_async" + ) + @pytest.mark.asyncio + async def test_discovery_scoped_by_agenthub_config( + self, mock_request, config, execution_context + ): + service = UiPathLlmChatService( + config=config, + execution_context=execution_context, + agenthub_config="agentsevals", + ) + mock_request.side_effect = _responder([], SKIP_TEMPERATURE_MODEL) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + ) + + discovery_headers = _discovery_calls(mock_request)[0][1]["headers"] + assert discovery_headers["x-uipath-agenthub-config"] == "agentsevals" + + @patch( + "uipath.platform.chat._llm_gateway_service.UiPathOpenAIService.request_async" + ) + @pytest.mark.asyncio + async def test_openai_compatible_path_omits_temperature( + self, mock_request, config, execution_context + ): + service = UiPathOpenAIService( + config=config, execution_context=execution_context + ) + mock_request.side_effect = _responder( + SKIP_TEMPERATURE_DISCOVERY, SKIP_TEMPERATURE_MODEL + ) + + await service.chat_completions( + messages=[{"role": "user", "content": "Hello"}], + model=SKIP_TEMPERATURE_MODEL, + temperature=0, + ) + + assert "temperature" not in _completion_body(mock_request) diff --git a/packages/uipath/pyproject.toml b/packages/uipath/pyproject.toml index 6ba25b021..bb91f7c6f 100644 --- a/packages/uipath/pyproject.toml +++ b/packages/uipath/pyproject.toml @@ -1,13 +1,13 @@ [project] name = "uipath" -version = "2.13.18" +version = "2.13.19" description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools." readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.11" dependencies = [ "uipath-core>=0.5.30, <0.6.0", "uipath-runtime>=0.12.2, <0.13.0", - "uipath-platform>=0.2.4, <0.3.0", + "uipath-platform>=0.2.14, <0.3.0", "click>=8.3.1", "httpx>=0.28.1", "pyjwt>=2.10.1", diff --git a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py index a8a8a64ec..7abdf78b1 100644 --- a/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py +++ b/packages/uipath/tests/cli/eval/mocks/test_input_mocker.py @@ -68,6 +68,13 @@ async def test_generate_llm_input_with_model_settings( json={}, ) + # Chat completions consults discovery to learn which parameters the model accepts. + httpx_mock.add_response( + url="https://example.com/llm/api/discovery", + status_code=200, + json=[{"modelName": "gpt-4o-mini-2024-07-18", "modelDetails": {}}], + ) + httpx_mock.add_response( url="https://example.com/llm/api/chat/completions" "?api-version=2024-08-01-preview", diff --git a/packages/uipath/uv.lock b/packages/uipath/uv.lock index 9c6d369c2..fbd18e154 100644 --- a/packages/uipath/uv.lock +++ b/packages/uipath/uv.lock @@ -2599,7 +2599,7 @@ wheels = [ [[package]] name = "uipath" -version = "2.13.18" +version = "2.13.19" source = { editable = "." } dependencies = [ { name = "applicationinsights" },