Skip to content

Commit eccf238

Browse files
posaninagendraposaninagendra
authored
Fixed remaining evaluators (#5098)
1 parent a73ad67 commit eccf238

14 files changed

Lines changed: 155 additions & 41 deletions

File tree

‎assets/evaluators/builtin/customer_satisfaction/evaluator/_customer_satisfaction.py‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1270,7 +1270,7 @@ def _build_result(
12701270
"sample_input": p.get("sample_input", ""),
12711271
"sample_output": p.get("sample_output", ""),
12721272
}
1273-
return {
1273+
result_payload = {
12741274
self._result_key: score,
12751275
f"{self._result_key}_score": score,
12761276
f"{self._result_key}_result": result,
@@ -1280,6 +1280,9 @@ def _build_result(
12801280
f"{self._result_key}_status": status,
12811281
f"{self._result_key}_properties": {**properties, **metadata},
12821282
}
1283+
# Add top-level token metadata fields for backward compatibility.
1284+
result_payload.update({f"{self._result_key}_{key}": value for key, value in metadata.items()})
1285+
return result_payload
12831286

12841287
def _parse_prompty_output(self, prompty_output_dict: Dict) -> Dict[str, Any]:
12851288
"""Parse the prompty output into a standardized result dictionary.

‎assets/evaluators/builtin/fluency/evaluator/_fluency.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -712,7 +712,8 @@ def _return_not_applicable_result(
712712
:return: A dictionary containing the result of the evaluation.
713713
:rtype: Dict[str, Union[str, float, None]]
714714
"""
715-
return {
715+
token_metadata = self._get_token_metadata({})
716+
result = {
716717
f"{self._result_key}": None,
717718
f"{self._result_key}_score": None,
718719
f"{self._result_key}_passed": None,
@@ -722,6 +723,9 @@ def _return_not_applicable_result(
722723
f"{self._result_key}_threshold": threshold,
723724
f"{self._result_key}_properties": None,
724725
}
726+
# Add top-level token metadata fields for backward compatibility.
727+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
728+
return result
725729

726730
async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
727731
"""Do a relevance evaluation.
@@ -782,8 +786,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
782786
score = float(match.group())
783787
score = float(score) if score is not None else math.nan
784788
score_result = self._get_binary_result(score)
785-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
786-
return {
789+
token_metadata = self._get_token_metadata(prompty_output_dict)
790+
llm_properties.update(token_metadata)
791+
result = {
787792
self._result_key: score,
788793
f"{self._result_key}_score": score,
789794
f"{self._result_key}_passed": score_result == "pass",
@@ -793,6 +798,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
793798
f"{self._result_key}_threshold": self._threshold,
794799
f"{self._result_key}_properties": llm_properties,
795800
}
801+
# Add top-level token metadata fields for backward compatibility.
802+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
803+
return result
796804
raise EvaluationException(
797805
message="Evaluator returned invalid output.",
798806
blame=ErrorBlame.SYSTEM_ERROR,

‎assets/evaluators/builtin/groundedness/evaluator/_groundedness.py‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1253,7 +1253,8 @@ def _build_result(
12531253
"""Build a standardized groundedness result dictionary."""
12541254
p = prompty_output_dict if isinstance(prompty_output_dict, dict) else {}
12551255
properties = dict(properties) if isinstance(properties, dict) else {}
1256-
properties.update(self._get_token_metadata(p))
1256+
token_metadata = self._get_token_metadata(p)
1257+
properties.update(token_metadata)
12571258
parsed_result: Dict[str, Union[str, int, float, Dict, None]] = {
12581259
self._result_key: score,
12591260
f"{self._result_key}_score": score,
@@ -1264,6 +1265,8 @@ def _build_result(
12641265
}
12651266
if status is not None:
12661267
parsed_result[f"{self._result_key}_status"] = status
1268+
# Add top-level token metadata fields for backward compatibility.
1269+
parsed_result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
12671270
return parsed_result
12681271

12691272
def _return_not_applicable_result(
@@ -1278,7 +1281,8 @@ def _return_not_applicable_result(
12781281
:return: A dictionary containing the result of the evaluation.
12791282
:rtype: Dict[str, Union[str, float, None]]
12801283
"""
1281-
return {
1284+
token_metadata = self._get_token_metadata({})
1285+
result = {
12821286
f"{self._result_key}": None,
12831287
f"{self._result_key}_score": None,
12841288
f"{self._result_key}_passed": None,
@@ -1288,6 +1292,9 @@ def _return_not_applicable_result(
12881292
f"{self._result_key}_threshold": threshold,
12891293
f"{self._result_key}_properties": None,
12901294
}
1295+
# Add top-level token metadata fields for backward compatibility.
1296+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
1297+
return result
12911298

12921299
async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
12931300
"""Do a relevance evaluation.
@@ -1348,8 +1355,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
13481355
score = float(match.group())
13491356
score = float(score) if score is not None else math.nan
13501357
score_result = self._get_binary_result(score)
1351-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
1352-
return {
1358+
token_metadata = self._get_token_metadata(prompty_output_dict)
1359+
llm_properties.update(token_metadata)
1360+
result = {
13531361
self._result_key: score,
13541362
f"{self._result_key}_score": score,
13551363
f"{self._result_key}_passed": score_result == "pass",
@@ -1359,6 +1367,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
13591367
f"{self._result_key}_threshold": self._threshold,
13601368
f"{self._result_key}_properties": llm_properties,
13611369
}
1370+
# Add top-level token metadata fields for backward compatibility.
1371+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
1372+
return result
13621373
raise EvaluationException(
13631374
message="Evaluator returned invalid output.",
13641375
blame=ErrorBlame.SYSTEM_ERROR,

‎assets/evaluators/builtin/intent_resolution/evaluator/_intent_resolution.py‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -827,7 +827,8 @@ def _return_not_applicable_result(
827827
:return: A dictionary containing the result of the evaluation.
828828
:rtype: Dict[str, Union[str, float, None]]
829829
"""
830-
return {
830+
token_metadata = self._get_token_metadata({})
831+
result = {
831832
f"{self._result_key}": None,
832833
f"{self._result_key}_score": None,
833834
f"{self._result_key}_passed": None,
@@ -837,6 +838,9 @@ def _return_not_applicable_result(
837838
f"{self._result_key}_threshold": threshold,
838839
f"{self._result_key}_properties": None,
839840
}
841+
# Add top-level token metadata fields for backward compatibility.
842+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
843+
return result
840844

841845
@staticmethod
842846
def _get_token_metadata(prompty_output: Dict) -> Dict:
@@ -995,7 +999,8 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
995999
score = float(score)
9961000
score_result = "pass" if score >= self._threshold else "fail"
9971001
llm_properties = llm_output.get("properties", {}) or {}
998-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
1002+
token_metadata = self._get_token_metadata(prompty_output_dict)
1003+
llm_properties.update(token_metadata)
9991004

10001005
response_dict = {
10011006
self._result_key: score,
@@ -1007,6 +1012,8 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
10071012
f"{self._result_key}_threshold": self._threshold,
10081013
f"{self._result_key}_properties": llm_properties,
10091014
}
1015+
# Add top-level token metadata fields for backward compatibility.
1016+
response_dict.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
10101017
return response_dict
10111018
raise EvaluationException(
10121019
message="Evaluator returned invalid output.",

‎assets/evaluators/builtin/relevance/evaluator/_relevance.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -720,7 +720,8 @@ def _return_not_applicable_result(
720720
:return: A dictionary containing the result of the evaluation.
721721
:rtype: Dict[str, Union[str, float, None]]
722722
"""
723-
return {
723+
token_metadata = self._get_token_metadata({})
724+
result = {
724725
f"{self._result_key}": None,
725726
f"{self._result_key}_score": None,
726727
f"{self._result_key}_passed": None,
@@ -730,6 +731,9 @@ def _return_not_applicable_result(
730731
f"{self._result_key}_threshold": threshold,
731732
f"{self._result_key}_properties": None,
732733
}
734+
# Add top-level token metadata fields for backward compatibility.
735+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
736+
return result
733737

734738
@staticmethod
735739
def _get_token_metadata(prompty_output: Dict) -> Dict:
@@ -872,8 +876,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
872876
reason = llm_output.get("reason", "")
873877
llm_properties = llm_output.get("properties", {}) or {}
874878
score_result = self._get_binary_result(score)
875-
llm_properties.update(self._get_token_metadata(result))
876-
return {
879+
token_metadata = self._get_token_metadata(result)
880+
llm_properties.update(token_metadata)
881+
response_dict = {
877882
self._result_key: score,
878883
f"{self._result_key}_score": score,
879884
f"{self._result_key}_passed": score_result == "pass",
@@ -883,6 +888,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
883888
f"{self._result_key}_threshold": self._threshold,
884889
f"{self._result_key}_properties": llm_properties,
885890
}
891+
# Add top-level token metadata fields for backward compatibility.
892+
response_dict.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
893+
return response_dict
886894

887895
raise EvaluationException(
888896
message="Evaluator returned invalid output.",

‎assets/evaluators/builtin/response_completeness/evaluator/_response_completeness.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -238,7 +238,8 @@ def _return_not_applicable_result(
238238
:return: A dictionary containing the result of the evaluation.
239239
:rtype: Dict[str, Union[str, float, None]]
240240
"""
241-
return {
241+
token_metadata = self._get_token_metadata({})
242+
result = {
242243
f"{self._result_key}": None,
243244
f"{self._result_key}_score": None,
244245
f"{self._result_key}_passed": None,
@@ -248,6 +249,9 @@ def _return_not_applicable_result(
248249
f"{self._result_key}_threshold": threshold,
249250
f"{self._result_key}_properties": None,
250251
}
252+
# Add top-level token metadata fields for backward compatibility.
253+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
254+
return result
251255

252256
@staticmethod
253257
def _get_token_metadata(prompty_output: Dict) -> Dict:
@@ -310,9 +314,10 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
310314
llm_properties = llm_output.get("properties", {}) or {}
311315
score_result = self._get_binary_result(score)
312316

313-
llm_properties.update(self._get_token_metadata(result if isinstance(result, dict) else {}))
317+
token_metadata = self._get_token_metadata(result if isinstance(result, dict) else {})
318+
llm_properties.update(token_metadata)
314319

315-
return {
320+
response_dict = {
316321
self._result_key: score,
317322
f"{self._result_key}_score": score,
318323
f"{self._result_key}_passed": score_result == "pass",
@@ -322,6 +327,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
322327
f"{self._result_key}_threshold": self._threshold,
323328
f"{self._result_key}_properties": llm_properties,
324329
}
330+
# Add top-level token metadata fields for backward compatibility.
331+
response_dict.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
332+
return response_dict
325333

326334
raise EvaluationException(
327335
message="Evaluator returned invalid output.",

‎assets/evaluators/builtin/retrieval/evaluator/_retrieval.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -245,7 +245,8 @@ def _return_not_applicable_result(
245245
:return: A dictionary containing the result of the evaluation.
246246
:rtype: Dict[str, Union[str, float, None]]
247247
"""
248-
return {
248+
token_metadata = self._get_token_metadata({})
249+
result = {
249250
f"{self._result_key}": None,
250251
f"{self._result_key}_score": None,
251252
f"{self._result_key}_passed": None,
@@ -255,6 +256,9 @@ def _return_not_applicable_result(
255256
f"{self._result_key}_threshold": threshold,
256257
f"{self._result_key}_properties": None,
257258
}
259+
# Add top-level token metadata fields for backward compatibility.
260+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
261+
return result
258262

259263
async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
260264
"""Do a relevance evaluation.
@@ -315,8 +319,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
315319
score = float(match.group())
316320
score = float(score) if score is not None else math.nan
317321
score_result = self._get_binary_result(score)
318-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
319-
return {
322+
token_metadata = self._get_token_metadata(prompty_output_dict)
323+
llm_properties.update(token_metadata)
324+
result = {
320325
self._result_key: score,
321326
f"{self._result_key}_score": score,
322327
f"{self._result_key}_passed": score_result == "pass",
@@ -326,6 +331,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
326331
f"{self._result_key}_threshold": self._threshold,
327332
f"{self._result_key}_properties": llm_properties,
328333
}
334+
# Add top-level token metadata fields for backward compatibility.
335+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
336+
return result
329337
raise EvaluationException(
330338
message="Evaluator returned invalid output.",
331339
blame=ErrorBlame.SYSTEM_ERROR,

‎assets/evaluators/builtin/similarity/evaluator/_similarity.py‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,8 @@ def _return_not_applicable_result(
270270
:return: A dictionary containing the result of the evaluation.
271271
:rtype: Dict[str, Union[str, float, None]]
272272
"""
273-
return {
273+
token_metadata = self._get_token_metadata({})
274+
result = {
274275
f"{self._result_key}": None,
275276
f"{self._result_key}_score": None,
276277
f"{self._result_key}_passed": None,
@@ -280,6 +281,9 @@ def _return_not_applicable_result(
280281
f"{self._result_key}_threshold": threshold,
281282
f"{self._result_key}_properties": None,
282283
}
284+
# Add top-level token metadata fields for backward compatibility.
285+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
286+
return result
283287

284288
async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
285289
"""Do a relevance evaluation.
@@ -340,8 +344,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
340344
score = float(match.group())
341345
score = float(score) if score is not None else math.nan
342346
score_result = self._get_binary_result(score)
343-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
344-
return {
347+
token_metadata = self._get_token_metadata(prompty_output_dict)
348+
llm_properties.update(token_metadata)
349+
result = {
345350
self._result_key: score,
346351
f"{self._result_key}_score": score,
347352
f"{self._result_key}_passed": score_result == "pass",
@@ -351,6 +356,9 @@ async def _the_super_do_eval(self, eval_input: Dict) -> Dict[str, Union[float, s
351356
f"{self._result_key}_threshold": self._threshold,
352357
f"{self._result_key}_properties": llm_properties,
353358
}
359+
# Add top-level token metadata fields for backward compatibility.
360+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
361+
return result
354362
raise EvaluationException(
355363
message="Evaluator returned invalid output.",
356364
blame=ErrorBlame.SYSTEM_ERROR,

‎assets/evaluators/builtin/task_adherence/evaluator/_task_adherence.py‎

Lines changed: 17 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1099,15 +1099,19 @@ def _build_result(
10991099
p = prompty_output_dict if isinstance(prompty_output_dict, dict) else {}
11001100
resolved_threshold = threshold if threshold is not None else self._threshold
11011101
properties = dict(properties) if isinstance(properties, dict) else {}
1102-
properties.update(self._get_token_metadata(p))
1103-
return {
1102+
token_metadata = self._get_token_metadata(p)
1103+
properties.update(token_metadata)
1104+
result_payload = {
11041105
self._result_key: score,
11051106
f"{self._result_key}_score": score,
11061107
f"{self._result_key}_result": result,
11071108
f"{self._result_key}_threshold": resolved_threshold,
11081109
f"{self._result_key}_reason": reason,
11091110
f"{self._result_key}_properties": properties,
11101111
}
1112+
# Add top-level token metadata fields for backward compatibility.
1113+
result_payload.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
1114+
return result_payload
11111115

11121116
def _return_not_applicable_result(
11131117
self, error_message: str, threshold: Union[int, float]
@@ -1121,7 +1125,8 @@ def _return_not_applicable_result(
11211125
:return: A dictionary containing the result of the evaluation.
11221126
:rtype: Dict[str, Union[str, float, None]]
11231127
"""
1124-
return {
1128+
token_metadata = self._get_token_metadata({})
1129+
result = {
11251130
f"{self._result_key}": None,
11261131
f"{self._result_key}_score": None,
11271132
f"{self._result_key}_passed": None,
@@ -1131,6 +1136,9 @@ def _return_not_applicable_result(
11311136
f"{self._result_key}_threshold": threshold,
11321137
f"{self._result_key}_properties": None,
11331138
}
1139+
# Add top-level token metadata fields for backward compatibility.
1140+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
1141+
return result
11341142

11351143
@staticmethod
11361144
def _get_token_metadata(prompty_output: Dict) -> Dict:
@@ -1371,9 +1379,10 @@ def _parse_prompty_output(self, prompty_output_dict: Dict[str, Any]) -> Dict[str
13711379
score = float(llm_output.get("score", 0.0))
13721380
score_result = "pass" if score >= 1.0 else "fail"
13731381
llm_properties = llm_output.get("properties", {}) or {}
1374-
llm_properties.update(self._get_token_metadata(prompty_output_dict))
1382+
token_metadata = self._get_token_metadata(prompty_output_dict)
1383+
llm_properties.update(token_metadata)
13751384

1376-
return {
1385+
result = {
13771386
self._result_key: score,
13781387
f"{self._result_key}_score": score,
13791388
f"{self._result_key}_passed": score_result == "pass",
@@ -1383,3 +1392,6 @@ def _parse_prompty_output(self, prompty_output_dict: Dict[str, Any]) -> Dict[str
13831392
f"{self._result_key}_threshold": self._threshold,
13841393
f"{self._result_key}_properties": llm_properties,
13851394
}
1395+
# Add top-level token metadata fields for backward compatibility.
1396+
result.update({f"{self._result_key}_{key}": value for key, value in token_metadata.items()})
1397+
return result

0 commit comments

Comments
 (0)