Skip to content

Commit def776b

Browse files
salma-elshafeysalma-elshafey
authored
Throw exception if LLM output is not parseable (#45158)
* Throw exception if LLM output is not parseable * Fix task completion test
1 parent f40b764 commit def776b

12 files changed

Lines changed: 58 additions & 71 deletions

File tree

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -183,12 +183,12 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
183183
}
184184

185185
binary_result = self._get_binary_result(score)
186-
return {
187-
self._result_key: float(score),
188-
f"gpt_{self._result_key}": float(score),
189-
f"{self._result_key}_result": binary_result,
190-
f"{self._result_key}_threshold": self._threshold,
191-
}
186+
raise EvaluationException(
187+
message="Evaluator returned invalid output.",
188+
blame=ErrorBlame.SYSTEM_ERROR,
189+
category=ErrorCategory.FAILED_EXECUTION,
190+
target=ErrorTarget.EVALUATE,
191+
)
192192

193193
@staticmethod
194194
def _get_built_in_tool_definition(tool_name: str):

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_intent_resolution/_intent_resolution.py‎

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -204,14 +204,10 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
204204
f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
205205
}
206206
return response_dict
207-
# If llm_output is not a dictionary, return NaN for the score. This should never happen
208-
if logger:
209-
logger.warning("LLM output is not a dictionary, returning NaN for the score.")
210-
211-
binary_result = self._get_binary_result(score)
212-
return {
213-
self._result_key: float(score),
214-
f"gpt_{self._result_key}": float(score),
215-
f"{self._result_key}_result": binary_result,
216-
f"{self._result_key}_threshold": self._threshold,
217-
}
207+
# If llm_output is not a dictionary, raise exception
208+
raise EvaluationException(
209+
message="Evaluator returned invalid output.",
210+
blame=ErrorBlame.SYSTEM_ERROR,
211+
category=ErrorCategory.FAILED_EXECUTION,
212+
target=ErrorTarget.EVALUATE,
213+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_relevance/_relevance.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -220,12 +220,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
220220
f"{self._result_key}_sample_output": result.get("sample_output", ""),
221221
}
222222

223-
if logger:
224-
logger.warning("LLM output is not a dictionary, returning NaN for the score.")
225-
226-
binary_result = self._get_binary_result(score)
227-
return {
228-
self._result_key: float(score),
229-
f"{self._result_key}_result": binary_result,
230-
f"{self._result_key}_threshold": self._threshold,
231-
}
223+
raise EvaluationException(
224+
message="Evaluator returned invalid output.",
225+
blame=ErrorBlame.SYSTEM_ERROR,
226+
category=ErrorCategory.FAILED_EXECUTION,
227+
target=ErrorTarget.EVALUATE,
228+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_response_completeness/_response_completeness.py‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -191,12 +191,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
191191
f"{self._result_key}_sample_output": result.get("sample_output", ""),
192192
}
193193

194-
if logger:
195-
logger.warning("LLM output is not a dictionary, returning NaN for the score.")
196-
197-
binary_result = self._get_binary_result(score)
198-
return {
199-
self._result_key: float(score),
200-
f"{self._result_key}_result": binary_result,
201-
f"{self._result_key}_threshold": self._threshold,
202-
}
194+
raise EvaluationException(
195+
message="Evaluator returned invalid output.",
196+
blame=ErrorBlame.SYSTEM_ERROR,
197+
category=ErrorCategory.FAILED_EXECUTION,
198+
target=ErrorTarget.EVALUATE,
199+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_adherence/_task_adherence.py‎

Lines changed: 6 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -241,7 +241,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str, bool]]
241241
f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
242242
}
243243

244-
if logger:
245-
logger.warning("LLM output is not a dictionary, returning 0 for the success.")
246-
247-
return {self._result_key: 0}
244+
raise EvaluationException(
245+
message="Evaluator returned invalid output.",
246+
blame=ErrorBlame.SYSTEM_ERROR,
247+
category=ErrorCategory.FAILED_EXECUTION,
248+
target=ErrorTarget.EVALUATE,
249+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_task_completion/_task_completion.py‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
194194
f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
195195
f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
196196
}
197-
if logger:
198-
logger.warning("LLM output is not a dictionary, returning 0 for the success.")
199-
return {self._result_key: 0}
197+
raise EvaluationException(
198+
message="Evaluator returned invalid output.",
199+
blame=ErrorBlame.SYSTEM_ERROR,
200+
category=ErrorCategory.FAILED_EXECUTION,
201+
target=ErrorTarget.EVALUATE,
202+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_accuracy/_tool_call_accuracy.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -262,10 +262,10 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
262262

263263
else:
264264
raise EvaluationException(
265-
message="Tool call accuracy evaluator returned invalid output.",
265+
message="Evaluator returned invalid output.",
266266
blame=ErrorBlame.SYSTEM_ERROR,
267267
category=ErrorCategory.FAILED_EXECUTION,
268-
target=ErrorTarget.TOOL_CALL_ACCURACY_EVALUATOR,
268+
target=ErrorTarget.EVALUATE,
269269
)
270270

271271
async def _real_call(self, **kwargs):

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_call_success/_tool_call_success.py‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -207,16 +207,12 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[str, float]]: # t
207207
f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
208208
f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
209209
}
210-
if logger:
211-
logger.warning("LLM output is not a dictionary, returning NaN for the score.")
212-
213-
score = math.nan
214-
binary_result = self._get_binary_result(score)
215-
return {
216-
self._result_key: float(score),
217-
f"{self._result_key}_result": binary_result,
218-
f"{self._result_key}_threshold": self._threshold,
219-
}
210+
raise EvaluationException(
211+
message="Evaluator returned invalid output.",
212+
blame=ErrorBlame.SYSTEM_ERROR,
213+
category=ErrorCategory.FAILED_EXECUTION,
214+
target=ErrorTarget.EVALUATE,
215+
)
220216

221217

222218
def _filter_to_used_tools(tool_definitions, msgs_list, logger=None):

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_input_accuracy/_tool_input_accuracy.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -217,10 +217,10 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:
217217

218218
else:
219219
raise EvaluationException(
220-
message="Tool input accuracy evaluator returned invalid output.",
220+
message="Evaluator returned invalid output.",
221221
blame=ErrorBlame.SYSTEM_ERROR,
222222
category=ErrorCategory.FAILED_EXECUTION,
223-
target=ErrorTarget.TOOL_INPUT_ACCURACY_EVALUATOR,
223+
target=ErrorTarget.EVALUATE,
224224
)
225225

226226
async def _real_call(self, **kwargs):

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_tool_output_utilization/_tool_output_utilization.py‎

Lines changed: 6 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,9 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]: # t
232232
f"{self._result_key}_sample_input": prompty_output_dict.get("sample_input", ""),
233233
f"{self._result_key}_sample_output": prompty_output_dict.get("sample_output", ""),
234234
}
235-
if logger:
236-
logger.warning("LLM output is not a dictionary, returning NaN for the score.")
237-
238-
score = math.nan
239-
binary_result = self._get_binary_result(score)
240-
return {
241-
self._result_key: float(score),
242-
f"{self._result_key}_result": binary_result,
243-
f"{self._result_key}_threshold": self._threshold,
244-
}
235+
raise EvaluationException(
236+
message="Evaluator returned invalid output.",
237+
blame=ErrorBlame.SYSTEM_ERROR,
238+
category=ErrorCategory.FAILED_EXECUTION,
239+
target=ErrorTarget.EVALUATE,
240+
)

0 commit comments

Comments
 (0)