Skip to content

Commit 94a80ac

Browse files
ajay-kesavanclaude
ajay-kesavan
and
claude
committed
chore(eval): resync evaluator type schemas with Python source
Refreshes the 11 evaluator schemas in evaluators_types/ that drifted from their Python config classes since the last regeneration. Pure output of `python -m uipath.eval.evaluators_types.generate_types`, no hand edits. Notable additions surfaced by the regenerator: - Output evaluators (Contains, ExactMatch, JsonSimilarity, LLMJudge*) gain line_by_line_evaluator and line_delimiter, added to OutputEvaluatorConfig in #1481. - All evaluators gain a populated justificationSchema where previously empty. - ToolCall evaluators gain target_output_key documentation. Split out from #1663 (which now ships only the two new classification schemas) so this snapshot refresh can be reviewed independently. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 010a8b7 commit 94a80ac

11 files changed

Lines changed: 251 additions & 21 deletions

‎packages/uipath/src/uipath/eval/evaluators_types/ContainsEvaluator.json‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,18 @@
4747
"title": "Target Output Key",
4848
"type": "string"
4949
},
50+
"line_by_line_evaluator": {
51+
"default": false,
52+
"description": "If True, split output by delimiter and evaluate each line separately",
53+
"title": "Line By Line Evaluator",
54+
"type": "boolean"
55+
},
56+
"line_delimiter": {
57+
"default": "\n",
58+
"description": "Delimiter to split output when line_by_line_evaluator is True",
59+
"title": "Line Delimiter",
60+
"type": "string"
61+
},
5062
"case_sensitive": {
5163
"default": false,
5264
"title": "Case Sensitive",
@@ -75,5 +87,23 @@
7587
"title": "ContainsEvaluationCriteria",
7688
"type": "object"
7789
},
78-
"justificationSchema": {}
90+
"justificationSchema": {
91+
"description": "Base class for all evaluator justifications.",
92+
"properties": {
93+
"expected": {
94+
"title": "Expected",
95+
"type": "string"
96+
},
97+
"actual": {
98+
"title": "Actual",
99+
"type": "string"
100+
}
101+
},
102+
"required": [
103+
"expected",
104+
"actual"
105+
],
106+
"title": "BaseEvaluatorJustification",
107+
"type": "object"
108+
}
79109
}

‎packages/uipath/src/uipath/eval/evaluators_types/ExactMatchEvaluator.json‎

Lines changed: 31 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,18 @@
5555
"title": "Target Output Key",
5656
"type": "string"
5757
},
58+
"line_by_line_evaluator": {
59+
"default": false,
60+
"description": "If True, split output by delimiter and evaluate each line separately",
61+
"title": "Line By Line Evaluator",
62+
"type": "boolean"
63+
},
64+
"line_delimiter": {
65+
"default": "\n",
66+
"description": "Delimiter to split output when line_by_line_evaluator is True",
67+
"title": "Line Delimiter",
68+
"type": "string"
69+
},
5870
"case_sensitive": {
5971
"default": false,
6072
"title": "Case Sensitive",
@@ -91,5 +103,23 @@
91103
"title": "OutputEvaluationCriteria",
92104
"type": "object"
93105
},
94-
"justificationSchema": {}
106+
"justificationSchema": {
107+
"description": "Base class for all evaluator justifications.",
108+
"properties": {
109+
"expected": {
110+
"title": "Expected",
111+
"type": "string"
112+
},
113+
"actual": {
114+
"title": "Actual",
115+
"type": "string"
116+
}
117+
},
118+
"required": [
119+
"expected",
120+
"actual"
121+
],
122+
"title": "BaseEvaluatorJustification",
123+
"type": "object"
124+
}
95125
}

‎packages/uipath/src/uipath/eval/evaluators_types/JsonSimilarityEvaluator.json‎

Lines changed: 39 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,18 @@
5454
"description": "Key to extract output from agent execution",
5555
"title": "Target Output Key",
5656
"type": "string"
57+
},
58+
"line_by_line_evaluator": {
59+
"default": false,
60+
"description": "If True, split output by delimiter and evaluate each line separately",
61+
"title": "Line By Line Evaluator",
62+
"type": "boolean"
63+
},
64+
"line_delimiter": {
65+
"default": "\n",
66+
"description": "Delimiter to split output when line_by_line_evaluator is True",
67+
"title": "Line Delimiter",
68+
"type": "string"
5769
}
5870
},
5971
"title": "JsonSimilarityEvaluatorConfig",
@@ -82,6 +94,32 @@
8294
"type": "object"
8395
},
8496
"justificationSchema": {
85-
"type": "string"
97+
"description": "Justification for the JSON similarity evaluator.",
98+
"properties": {
99+
"expected": {
100+
"title": "Expected",
101+
"type": "string"
102+
},
103+
"actual": {
104+
"title": "Actual",
105+
"type": "string"
106+
},
107+
"matched_leaves": {
108+
"title": "Matched Leaves",
109+
"type": "number"
110+
},
111+
"total_leaves": {
112+
"title": "Total Leaves",
113+
"type": "number"
114+
}
115+
},
116+
"required": [
117+
"expected",
118+
"actual",
119+
"matched_leaves",
120+
"total_leaves"
121+
],
122+
"title": "JsonSimilarityJustification",
123+
"type": "object"
86124
}
87125
}

‎packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeOutputEvaluator.json‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,18 @@
8181
"description": "Key to extract output from agent execution",
8282
"title": "Target Output Key",
8383
"type": "string"
84+
},
85+
"line_by_line_evaluator": {
86+
"default": false,
87+
"description": "If True, split output by delimiter and evaluate each line separately",
88+
"title": "Line By Line Evaluator",
89+
"type": "boolean"
90+
},
91+
"line_delimiter": {
92+
"default": "\n",
93+
"description": "Delimiter to split output when line_by_line_evaluator is True",
94+
"title": "Line Delimiter",
95+
"type": "string"
8496
}
8597
},
8698
"title": "LLMJudgeOutputEvaluatorConfig",
@@ -109,6 +121,27 @@
109121
"type": "object"
110122
},
111123
"justificationSchema": {
112-
"type": "string"
124+
"description": "Justification for LLM judge evaluators.",
125+
"properties": {
126+
"expected": {
127+
"title": "Expected",
128+
"type": "string"
129+
},
130+
"actual": {
131+
"title": "Actual",
132+
"type": "string"
133+
},
134+
"justification": {
135+
"title": "Justification",
136+
"type": "string"
137+
}
138+
},
139+
"required": [
140+
"expected",
141+
"actual",
142+
"justification"
143+
],
144+
"title": "LLMJudgeJustification",
145+
"type": "object"
113146
}
114147
}

‎packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeStrictJSONSimilarityOutputEvaluator.json‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,18 @@
8181
"description": "Key to extract output from agent execution",
8282
"title": "Target Output Key",
8383
"type": "string"
84+
},
85+
"line_by_line_evaluator": {
86+
"default": false,
87+
"description": "If True, split output by delimiter and evaluate each line separately",
88+
"title": "Line By Line Evaluator",
89+
"type": "boolean"
90+
},
91+
"line_delimiter": {
92+
"default": "\n",
93+
"description": "Delimiter to split output when line_by_line_evaluator is True",
94+
"title": "Line Delimiter",
95+
"type": "string"
8496
}
8597
},
8698
"title": "LLMJudgeStrictJSONSimilarityOutputEvaluatorConfig",
@@ -109,6 +121,27 @@
109121
"type": "object"
110122
},
111123
"justificationSchema": {
112-
"type": "string"
124+
"description": "Justification for LLM judge evaluators.",
125+
"properties": {
126+
"expected": {
127+
"title": "Expected",
128+
"type": "string"
129+
},
130+
"actual": {
131+
"title": "Actual",
132+
"type": "string"
133+
},
134+
"justification": {
135+
"title": "Justification",
136+
"type": "string"
137+
}
138+
},
139+
"required": [
140+
"expected",
141+
"actual",
142+
"justification"
143+
],
144+
"title": "LLMJudgeJustification",
145+
"type": "object"
113146
}
114147
}

‎packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectoryEvaluator.json‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,27 @@
8787
"type": "object"
8888
},
8989
"justificationSchema": {
90-
"type": "string"
90+
"description": "Justification for LLM judge evaluators.",
91+
"properties": {
92+
"expected": {
93+
"title": "Expected",
94+
"type": "string"
95+
},
96+
"actual": {
97+
"title": "Actual",
98+
"type": "string"
99+
},
100+
"justification": {
101+
"title": "Justification",
102+
"type": "string"
103+
}
104+
},
105+
"required": [
106+
"expected",
107+
"actual",
108+
"justification"
109+
],
110+
"title": "LLMJudgeJustification",
111+
"type": "object"
91112
}
92113
}

‎packages/uipath/src/uipath/eval/evaluators_types/LLMJudgeTrajectorySimulationEvaluator.json‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,27 @@
8787
"type": "object"
8888
},
8989
"justificationSchema": {
90-
"type": "string"
90+
"description": "Justification for LLM judge evaluators.",
91+
"properties": {
92+
"expected": {
93+
"title": "Expected",
94+
"type": "string"
95+
},
96+
"actual": {
97+
"title": "Actual",
98+
"type": "string"
99+
},
100+
"justification": {
101+
"title": "Justification",
102+
"type": "string"
103+
}
104+
},
105+
"required": [
106+
"expected",
107+
"actual",
108+
"justification"
109+
],
110+
"title": "LLMJudgeJustification",
111+
"type": "object"
91112
}
92113
}

‎packages/uipath/src/uipath/eval/evaluators_types/ToolCallArgsEvaluator.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,14 @@
120120
"justificationSchema": {
121121
"description": "Justification for the tool call args evaluator.",
122122
"properties": {
123+
"expected": {
124+
"title": "Expected",
125+
"type": "string"
126+
},
127+
"actual": {
128+
"title": "Actual",
129+
"type": "string"
130+
},
123131
"explained_tool_calls_args": {
124132
"additionalProperties": {
125133
"type": "string"
@@ -129,6 +137,8 @@
129137
}
130138
},
131139
"required": [
140+
"expected",
141+
"actual",
132142
"explained_tool_calls_args"
133143
],
134144
"title": "ToolCallArgsEvaluatorJustification",

‎packages/uipath/src/uipath/eval/evaluators_types/ToolCallCountEvaluator.json‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,14 @@
9393
"justificationSchema": {
9494
"description": "Justification for the tool call count evaluator.",
9595
"properties": {
96+
"expected": {
97+
"title": "Expected",
98+
"type": "string"
99+
},
100+
"actual": {
101+
"title": "Actual",
102+
"type": "string"
103+
},
96104
"explained_tool_calls_count": {
97105
"additionalProperties": {
98106
"type": "string"
@@ -102,6 +110,8 @@
102110
}
103111
},
104112
"required": [
113+
"expected",
114+
"actual",
105115
"explained_tool_calls_count"
106116
],
107117
"title": "ToolCallCountEvaluatorJustification",

‎packages/uipath/src/uipath/eval/evaluators_types/ToolCallOrderEvaluator.json‎

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -73,19 +73,13 @@
7373
"justificationSchema": {
7474
"description": "Justification for the tool call order evaluator.",
7575
"properties": {
76-
"actual_tool_calls_order": {
77-
"items": {
78-
"type": "string"
79-
},
80-
"title": "Actual Tool Calls Order",
81-
"type": "array"
76+
"expected": {
77+
"title": "Expected",
78+
"type": "string"
8279
},
83-
"expected_tool_calls_order": {
84-
"items": {
85-
"type": "string"
86-
},
87-
"title": "Expected Tool Calls Order",
88-
"type": "array"
80+
"actual": {
81+
"title": "Actual",
82+
"type": "string"
8983
},
9084
"lcs": {
9185
"items": {
@@ -96,8 +90,8 @@
9690
}
9791
},
9892
"required": [
99-
"actual_tool_calls_order",
100-
"expected_tool_calls_order",
93+
"expected",
94+
"actual",
10195
"lcs"
10296
],
10397
"title": "ToolCallOrderEvaluatorJustification",

0 commit comments

Comments
 (0)