Skip to content

Commit 62062cc

Browse files
rakesh-uipathclaude
andcommitted
fix(uipath): restore line_by_line support in ExactMatchEvaluator
ExactMatchEvaluator was a pure re-export from uipath_eval which lacks line_by_line_evaluator support in its OutputEvaluatorConfig. Restore by defining a local ExactMatchEvaluator that extends uipath's OutputEvaluator (which has line_by_line logic in validate_and_evaluate_criteria). Also fix import sort in output_evaluator.py. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent adf1fca commit 62062cc

2 files changed

Lines changed: 69 additions & 8 deletions

File tree

packages/uipath/src/uipath/eval/evaluators/exact_match_evaluator.py

Lines changed: 66 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,72 @@
1-
"""Re-exported from uipath-eval."""
1+
"""Exact match evaluator for agent outputs."""
22

3-
from uipath_eval.evaluators.exact_match_evaluator import (
4-
ExactMatchEvaluator,
5-
ExactMatchEvaluatorConfig,
3+
from uipath_eval.evaluators.base_evaluator import BaseEvaluatorJustification
4+
from uipath_eval.models import EvaluatorType
5+
6+
from ..models import AgentExecution, EvaluationResult, NumericEvaluationResult
7+
from .output_evaluator import (
8+
OutputEvaluationCriteria,
9+
OutputEvaluator,
10+
OutputEvaluatorConfig,
611
)
712

13+
14+
class ExactMatchEvaluatorConfig(OutputEvaluatorConfig[OutputEvaluationCriteria]):
15+
"""Configuration for the exact match evaluator."""
16+
17+
name: str = "ExactMatchEvaluator"
18+
case_sensitive: bool = False
19+
negated: bool = False
20+
21+
22+
class ExactMatchEvaluator(
23+
OutputEvaluator[
24+
OutputEvaluationCriteria,
25+
ExactMatchEvaluatorConfig,
26+
BaseEvaluatorJustification,
27+
]
28+
):
29+
"""Evaluator that performs exact structural matching between expected and actual outputs."""
30+
31+
@classmethod
32+
def get_evaluator_id(cls) -> str:
33+
"""Get the evaluator id."""
34+
return EvaluatorType.EXACT_MATCH.value
35+
36+
async def evaluate(
37+
self,
38+
agent_execution: AgentExecution,
39+
evaluation_criteria: OutputEvaluationCriteria,
40+
) -> EvaluationResult:
41+
"""Evaluate whether actual output exactly matches expected output."""
42+
actual_output = self._get_actual_output(agent_execution)
43+
expected_output = self._get_expected_output(evaluation_criteria)
44+
45+
if isinstance(actual_output, str) or isinstance(expected_output, str):
46+
actual_str = str(actual_output)
47+
expected_str = str(expected_output)
48+
if not self.evaluator_config.case_sensitive:
49+
actual_str = actual_str.lower()
50+
expected_str = expected_str.lower()
51+
is_exact_match = actual_str == expected_str
52+
else:
53+
is_exact_match = actual_output == expected_output
54+
55+
if self.evaluator_config.negated:
56+
is_exact_match = not is_exact_match
57+
58+
validated_justification = self.validate_justification(
59+
{
60+
"expected": str(expected_output),
61+
"actual": str(actual_output),
62+
}
63+
)
64+
return NumericEvaluationResult(
65+
score=float(is_exact_match),
66+
details=validated_justification,
67+
)
68+
69+
870
__all__ = [
971
"ExactMatchEvaluator",
1072
"ExactMatchEvaluatorConfig",

packages/uipath/src/uipath/eval/evaluators/output_evaluator.py

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,9 @@
55
from typing import TYPE_CHECKING, Any, TypeVar, Union
66

77
from pydantic import BaseModel, Field
8+
from uipath_eval.evaluators.output_evaluator import (
9+
OutputEvaluationCriteria as OutputEvaluationCriteria,
10+
)
811

912
from .._helpers.output_path import resolve_output_path
1013
from ..models import AgentExecution
@@ -14,10 +17,6 @@
1417
extract_attachment_id,
1518
is_job_attachment_uri,
1619
)
17-
from uipath_eval.evaluators.output_evaluator import (
18-
OutputEvaluationCriteria as OutputEvaluationCriteria,
19-
)
20-
2120
from .base_evaluator import (
2221
BaseEvaluationCriteria,
2322
BaseEvaluator,

0 commit comments

Comments
 (0)