fix(eval): reject out-of-range scores from LLM judge tool calls (#1819)

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: mjnovice

packages/uipath/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "uipath"
-version = "2.13.13"
+version = "2.13.14"
 description = "Python SDK and CLI for UiPath Platform, enabling programmatic interaction with automation services, process management, and deployment tools."
 readme = { file = "README.md", content-type = "text/markdown" }
 requires-python = ">=3.11"

packages/uipath/src/uipath/eval/evaluators/legacy_llm_helpers.py

@@ -1,6 +1,7 @@
 """Helper functions for legacy LLM evaluators using function calling."""
 
 import logging
+import math
 from typing import Any
 
 from uipath.platform.chat.llm_gateway import (
@@ -89,9 +90,32 @@ def extract_tool_call_response(response: Any, model: str) -> LLMResponse:
             logger.debug(f"Arguments: {arguments}")
             raise ValueError(error_msg)
 
-        score = float(arguments["score"])
+        try:
+            score = float(arguments["score"])
+        except (ValueError, TypeError) as e:
+            error_msg = (
+                f"Non-numeric score {arguments['score']!r} in tool call arguments "
+                f"from model {model}: expected a number between 0 and 100"
+            )
+            logger.error(f"❌ {error_msg}")
+            logger.debug(f"Arguments: {arguments}")
+            raise ValueError(error_msg) from e
+
         justification = str(arguments["justification"])
 
+        # Models occasionally emit corrupted numeric tool arguments despite the
+        # 0-100 range stated in the tool schema (e.g. gemini-2.5-flash returning
+        # 989898 or 950). Unvalidated, such a value poisons every run-level
+        # aggregate downstream, so reject it and let the evaluation surface as
+        # an error instead of recording a fabricated score.
+        if not math.isfinite(score) or score < 0.0 or score > 100.0:
+            error_msg = (
+                f"Invalid score {score!r} in tool call arguments from model {model}: "
+                f"expected a number between 0 and 100"
+            )
+            logger.error(f"❌ {error_msg}")
+            raise ValueError(error_msg)
+
         logger.debug(
             f"✅ Extracted score: {score}, justification length: {len(justification)} chars"
         )

packages/uipath/tests/evaluators/test_legacy_llm_helpers.py

@@ -0,0 +1,84 @@
+"""Tests for legacy LLM helper functions (submit_evaluation tool-call parsing)."""
+
+from types import SimpleNamespace
+from typing import Any
+
+import pytest
+
+from uipath.eval.evaluators.legacy_llm_helpers import extract_tool_call_response
+
+
+def _make_response(arguments: dict[str, Any]) -> Any:
+    """Build a minimal chat-completions response carrying a submit_evaluation tool call."""
+    tool_call = SimpleNamespace(arguments=arguments)
+    message = SimpleNamespace(tool_calls=[tool_call])
+    choice = SimpleNamespace(message=message)
+    return SimpleNamespace(choices=[choice])
+
+
+class TestExtractToolCallResponse:
+    """Test extract_tool_call_response score validation."""
+
+    def test_valid_score_passes_through(self) -> None:
+        response = _make_response({"score": 88, "justification": "ok"})
+
+        result = extract_tool_call_response(response, "gemini-2.5-flash")
+
+        assert result.score == 88.0
+        assert result.justification == "ok"
+
+    def test_out_of_range_score_is_rejected(self) -> None:
+        # Real payload observed in production: gemini-2.5-flash returned
+        # score=989898 in its submit_evaluation tool call while the justification
+        # said the outputs "match perfectly". Unvalidated, this single value blew
+        # a 64-item run-level average up to 15559.13%. The evaluation must surface
+        # as an error rather than record a fabricated score.
+        response = _make_response(
+            {"score": 989898, "justification": "matches perfectly"}
+        )
+
+        with pytest.raises(ValueError, match="Invalid score 989898"):
+            extract_tool_call_response(response, "gemini-2.5-flash")
+
+    def test_out_of_range_950_is_rejected(self) -> None:
+        # Second production occurrence from the same eval run: score=950
+        # (the model most likely intended 95).
+        response = _make_response({"score": 950, "justification": "equivalent"})
+
+        with pytest.raises(ValueError, match="Invalid score 950"):
+            extract_tool_call_response(response, "gemini-2.5-flash")
+
+    def test_negative_score_is_rejected(self) -> None:
+        response = _make_response({"score": -5, "justification": "bad"})
+
+        with pytest.raises(ValueError, match="Invalid score -5"):
+            extract_tool_call_response(response, "gpt-4o")
+
+    @pytest.mark.parametrize("boundary", [0, 100])
+    def test_boundary_scores_accepted(self, boundary: int) -> None:
+        response = _make_response({"score": boundary, "justification": "j"})
+
+        result = extract_tool_call_response(response, "m")
+
+        assert result.score == float(boundary)
+        assert result.justification == "j"
+
+    @pytest.mark.parametrize("value", [float("nan"), float("inf"), float("-inf")])
+    def test_non_finite_score_is_rejected(self, value: float) -> None:
+        response = _make_response({"score": value, "justification": "j"})
+
+        with pytest.raises(ValueError, match="Invalid score"):
+            extract_tool_call_response(response, "m")
+
+    def test_missing_score_raises(self) -> None:
+        response = _make_response({"justification": "j"})
+
+        with pytest.raises(ValueError, match="Missing 'score'"):
+            extract_tool_call_response(response, "m")
+
+    @pytest.mark.parametrize("value", ["not-a-number", None, [95]])
+    def test_non_numeric_score_is_rejected(self, value: Any) -> None:
+        response = _make_response({"score": value, "justification": "j"})
+
+        with pytest.raises(ValueError, match="Non-numeric score"):
+            extract_tool_call_response(response, "m")