fix(eval): stop stripping -community-agents model suffix before calling LLM Gateway (#1803)

Co-authored-by: Claude Sonnet 5 <noreply@anthropic.com>

Author: Chibionos

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.13.9"
+version = "2.13.10"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/eval/evaluators/legacy_context_precision_evaluator.py

@@ -18,7 +18,7 @@
     LegacyEvaluatorConfig,
     track_evaluation_metrics,
 )
-from .legacy_evaluator_utils import clean_model_name, serialize_object
+from .legacy_evaluator_utils import serialize_object
 
 
 class LegacyContextPrecisionEvaluatorConfig(LegacyEvaluatorConfig):
@@ -326,8 +326,7 @@ async def _get_structured_llm_response(
             ToolParametersDefinition,
         )
 
-        # Remove community-agents suffix from llm model name
-        model = clean_model_name(self.model)
+        model = self.model
 
         # Create tool definition for context precision evaluation
         # Note: We pass the array schema as a raw dict because ToolPropertyDefinition

packages/uipath/src/uipath/eval/evaluators/legacy_evaluator_utils.py

@@ -3,22 +3,6 @@
 import json
 from typing import Any, Optional
 
-from uipath.platform.constants import COMMUNITY_agents_SUFFIX
-
-
-def clean_model_name(model: str) -> str:
-    """Remove community-agents suffix from model name.
-
-    Args:
-        model: Model name that may have the community suffix
-
-    Returns:
-        Model name without the community suffix
-    """
-    if model.endswith(COMMUNITY_agents_SUFFIX):
-        return model.replace(COMMUNITY_agents_SUFFIX, "")
-    return model
-
 
 def serialize_object(
     content: Any,

packages/uipath/src/uipath/eval/evaluators/legacy_faithfulness_evaluator.py

@@ -18,7 +18,6 @@
     track_evaluation_metrics,
 )
 from .legacy_evaluator_utils import (
-    clean_model_name,
     serialize_object,
 )
 
@@ -513,8 +512,7 @@ async def _get_structured_llm_response(
             ToolParametersDefinition,
         )
 
-        # Remove community-agents suffix from llm model name
-        model = clean_model_name(self.model)
+        model = self.model
 
         # Create a dynamic tool definition based on the schema
         tool = ToolDefinition(

packages/uipath/src/uipath/eval/evaluators/legacy_llm_as_judge_evaluator.py

@@ -8,7 +8,6 @@
 from uipath.platform import UiPath
 from uipath.platform.chat import UiPathLlmChatService
 from uipath.platform.chat.llm_gateway import RequiredToolChoice
-from uipath.platform.constants import COMMUNITY_agents_SUFFIX
 
 from .._execution_context import eval_set_run_id_context
 from .._helpers.helpers import is_empty_value
@@ -193,10 +192,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
         Returns:
             LLMResponse with score and justification
         """
-        # remove community-agents suffix from llm model name
         model = self.model
-        if model.endswith(COMMUNITY_agents_SUFFIX):
-            model = model.replace(COMMUNITY_agents_SUFFIX, "")
 
         # Create evaluation tool for function calling (works across all models)
         evaluation_tool = create_evaluation_tool()

packages/uipath/src/uipath/eval/evaluators/legacy_trajectory_evaluator.py

@@ -9,7 +9,6 @@
 from uipath.platform import UiPath
 from uipath.platform.chat import UiPathLlmChatService
 from uipath.platform.chat.llm_gateway import RequiredToolChoice
-from uipath.platform.constants import COMMUNITY_agents_SUFFIX
 
 from .._execution_context import eval_set_run_id_context
 from .._helpers.evaluators_helpers import trace_to_str
@@ -163,8 +162,6 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
         assert self.llm, "LLM should be initialized before calling this method."
 
         model = self.model
-        if model.endswith(COMMUNITY_agents_SUFFIX):
-            model = model.replace(COMMUNITY_agents_SUFFIX, "")
 
         # Create evaluation tool for function calling (works across all models)
         evaluation_tool = create_evaluation_tool()

packages/uipath/src/uipath/eval/evaluators/llm_as_judge_evaluator.py

@@ -11,7 +11,6 @@
 
 from uipath.platform import UiPath
 from uipath.platform.chat import UiPathLlmChatService
-from uipath.platform.constants import COMMUNITY_agents_SUFFIX
 
 from .._execution_context import eval_set_run_id_context
 from ..models import (
@@ -211,10 +210,7 @@ async def _get_llm_response(self, evaluation_prompt: str) -> LLMResponse:
             ToolPropertyDefinition,
         )
 
-        # Remove community-agents suffix from llm model name
         model = self.evaluator_config.model
-        if model.endswith(COMMUNITY_agents_SUFFIX):
-            model = model.replace(COMMUNITY_agents_SUFFIX, "")
 
         # Define function/tool for structured output (works for ALL models via Normalized API)
         evaluation_tool = ToolDefinition(

packages/uipath/tests/evaluators/test_llm_judge_model_suffix.py

@@ -0,0 +1,219 @@
+"""Regression tests: LLM-judge evaluators must send the model name to the LLM
+Gateway exactly as configured, including a "-community-agents" suffix.
+
+Community/EU tenants' LLM Gateway routing rules are keyed on the suffixed
+model id -- the same id AgentHub sends when it runs the agent itself.
+Stripping the suffix before calling the Gateway causes a 417 "No llm routing
+rule found for product agentsplaygroundfallback in EU using model ..." for
+every Community/EU evaluation run, even though the identical model id works
+fine for the agent.
+"""
+
+import uuid
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+import pytest
+
+from uipath.eval.evaluators.base_legacy_evaluator import LegacyEvaluationCriteria
+from uipath.eval.evaluators.legacy_context_precision_evaluator import (
+    LegacyContextPrecisionEvaluator,
+    LegacyContextPrecisionEvaluatorConfig,
+)
+from uipath.eval.evaluators.legacy_faithfulness_evaluator import (
+    LegacyFaithfulnessEvaluator,
+    LegacyFaithfulnessEvaluatorConfig,
+)
+from uipath.eval.evaluators.legacy_llm_as_judge_evaluator import (
+    LegacyLlmAsAJudgeEvaluator,
+    LegacyLlmAsAJudgeEvaluatorConfig,
+)
+from uipath.eval.evaluators.legacy_trajectory_evaluator import (
+    LegacyTrajectoryEvaluator,
+    LegacyTrajectoryEvaluatorConfig,
+)
+from uipath.eval.evaluators.llm_judge_output_evaluator import LLMJudgeOutputEvaluator
+from uipath.eval.evaluators.llm_judge_trajectory_evaluator import (
+    LLMJudgeTrajectoryEvaluator,
+)
+from uipath.eval.models.models import LegacyEvaluatorCategory, LegacyEvaluatorType
+
+COMMUNITY_MODEL = "gpt-5.4-2026-03-05-community-agents"
+
+
+def _fake_tool_call_response(score: float = 90, justification: str = "ok"):
+    tool_call = SimpleNamespace(
+        arguments={"score": score, "justification": justification}
+    )
+    message = SimpleNamespace(tool_calls=[tool_call])
+    choice = SimpleNamespace(message=message)
+    return SimpleNamespace(choices=[choice])
+
+
+def _legacy_context_precision_evaluator() -> LegacyContextPrecisionEvaluator:
+    return LegacyContextPrecisionEvaluator(
+        id="context-precision",
+        config_type=LegacyContextPrecisionEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.LlmAsAJudge,
+        type=LegacyEvaluatorType.ContextPrecision,
+        name="Context Precision",
+        description="Evaluates context chunk relevance",
+        createdAt="2025-01-01T00:00:00Z",
+        updatedAt="2025-01-01T00:00:00Z",
+        targetOutputKey="*",
+        model=COMMUNITY_MODEL,
+    )
+
+
+class TestLegacyContextPrecisionEvaluatorSendsConfiguredModel:
+    @pytest.mark.asyncio
+    async def test_get_structured_llm_response_sends_full_model_name(self):
+        evaluator = _legacy_context_precision_evaluator()
+        mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm = AsyncMock(chat_completions=mock_chat_completions)
+
+        await evaluator._get_structured_llm_response("some evaluation prompt")
+
+        sent_model = mock_chat_completions.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL
+
+
+def _legacy_faithfulness_evaluator() -> LegacyFaithfulnessEvaluator:
+    return LegacyFaithfulnessEvaluator(
+        id="faithfulness",
+        config_type=LegacyFaithfulnessEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.LlmAsAJudge,
+        type=LegacyEvaluatorType.Faithfulness,
+        name="Faithfulness",
+        description="Evaluates faithfulness of claims against context",
+        createdAt="2025-01-01T00:00:00Z",
+        updatedAt="2025-01-01T00:00:00Z",
+        targetOutputKey="*",
+        model=COMMUNITY_MODEL,
+    )
+
+
+class TestLegacyFaithfulnessEvaluatorSendsConfiguredModel:
+    @pytest.mark.asyncio
+    async def test_get_structured_llm_response_sends_full_model_name(self):
+        evaluator = _legacy_faithfulness_evaluator()
+        mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm = AsyncMock(chat_completions=mock_chat_completions)
+
+        await evaluator._get_structured_llm_response(
+            "some evaluation prompt", "submit_result", {"type": "object"}
+        )
+
+        sent_model = mock_chat_completions.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL
+
+
+class TestLLMJudgeOutputEvaluatorSendsConfiguredModel:
+    """Covers LLMJudgeMixin._get_llm_response -- the code path hit by
+    'uipath-llm-judge-output-semantic-similarity' in production."""
+
+    @pytest.mark.asyncio
+    async def test_get_llm_response_sends_full_model_name_to_gateway(self):
+        config = {
+            "name": "TestEvaluator",
+            "prompt": "Evaluate {{ActualOutput}} against {{ExpectedOutput}}",
+            "model": COMMUNITY_MODEL,
+        }
+        with patch("uipath.platform.UiPath"):
+            evaluator = LLMJudgeOutputEvaluator.model_validate(
+                {"evaluatorConfig": config, "id": str(uuid.uuid4())}
+            )
+        mock_llm_service = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm_service = mock_llm_service
+
+        await evaluator._get_llm_response("some evaluation prompt")
+
+        sent_model = mock_llm_service.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL
+
+
+class TestLLMJudgeTrajectoryEvaluatorSendsConfiguredModel:
+    """Covers the same LLMJudgeMixin._get_llm_response via the trajectory
+    evaluator -- this is the exact evaluator type from the reported
+    production trace ('uipath-llm-judge-trajectory-similarity')."""
+
+    @pytest.mark.asyncio
+    async def test_get_llm_response_sends_full_model_name_to_gateway(self):
+        config = {
+            "name": "TestEvaluator",
+            "prompt": "Judge {{AgentRunHistory}} against {{ExpectedAgentBehavior}}",
+            "model": COMMUNITY_MODEL,
+        }
+        with patch("uipath.platform.UiPath"):
+            evaluator = LLMJudgeTrajectoryEvaluator.model_validate(
+                {"evaluatorConfig": config, "id": str(uuid.uuid4())}
+            )
+        mock_llm_service = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm_service = mock_llm_service
+
+        await evaluator._get_llm_response("some evaluation prompt")
+
+        sent_model = mock_llm_service.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL
+
+
+def _legacy_trajectory_evaluator() -> LegacyTrajectoryEvaluator:
+    return LegacyTrajectoryEvaluator(
+        id=str(uuid.uuid4()),
+        name="Legacy trajectory",
+        config_type=LegacyTrajectoryEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.Trajectory,
+        type=LegacyEvaluatorType.Trajectory,
+        prompt="History:\n{{AgentRunHistory}}\nExpected:\n{{ExpectedAgentBehavior}}",
+        model=COMMUNITY_MODEL,
+        createdAt="2026-05-14T00:00:00Z",
+        updatedAt="2026-05-14T00:00:00Z",
+    )
+
+
+class TestLegacyTrajectoryEvaluatorSendsConfiguredModel:
+    @pytest.mark.asyncio
+    async def test_get_llm_response_sends_full_model_name_to_gateway(self):
+        evaluator = _legacy_trajectory_evaluator()
+        mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm = AsyncMock(chat_completions=mock_chat_completions)
+
+        await evaluator._get_llm_response("some evaluation prompt")
+
+        sent_model = mock_chat_completions.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL
+
+
+def _legacy_llm_as_judge_evaluator() -> LegacyLlmAsAJudgeEvaluator:
+    return LegacyLlmAsAJudgeEvaluator(
+        id=str(uuid.uuid4()),
+        name="Legacy LLM judge",
+        config_type=LegacyLlmAsAJudgeEvaluatorConfig,
+        evaluation_criteria_type=LegacyEvaluationCriteria,
+        justification_type=str,
+        category=LegacyEvaluatorCategory.LlmAsAJudge,
+        type=LegacyEvaluatorType.Factuality,
+        prompt="Compare {{ActualOutput}} to {{ExpectedOutput}}",
+        model=COMMUNITY_MODEL,
+        createdAt="2026-05-14T00:00:00Z",
+        updatedAt="2026-05-14T00:00:00Z",
+    )
+
+
+class TestLegacyLlmAsAJudgeEvaluatorSendsConfiguredModel:
+    @pytest.mark.asyncio
+    async def test_get_llm_response_sends_full_model_name_to_gateway(self):
+        evaluator = _legacy_llm_as_judge_evaluator()
+        mock_chat_completions = AsyncMock(return_value=_fake_tool_call_response())
+        evaluator.llm = AsyncMock(chat_completions=mock_chat_completions)
+
+        await evaluator._get_llm_response("some evaluation prompt")
+
+        sent_model = mock_chat_completions.call_args.kwargs["model"]
+        assert sent_model == COMMUNITY_MODEL