Add status to prompty

Author: m7md7sien

assets/evaluators/builtin/tool_call_accuracy/evaluator/_tool_call_accuracy.py

@@ -974,6 +974,12 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # t
         llm_output = prompty_output_dict.get("llm_output", prompty_output_dict)
 
         if isinstance(llm_output, dict):
+            # Handle skipped status from LLM
+            llm_status = llm_output.get("status", "completed")
+            if llm_status == "skipped":
+                reason = llm_output.get("reasoning", "")
+                return self._not_applicable_result(reason, self.threshold)
+
             score = llm_output.get(self._LLM_SCORE_KEY, None)
             if not score or not check_score_is_valid(
                 score,
@@ -1004,12 +1010,13 @@ async def _do_eval(self, eval_input: Dict) -> Dict[str, Union[float, str]]:  # t
                 "sample_output": prompty_output_dict.get("sample_output", ""),
             })
             response_dict = {
-                f"{self.result_key}_score": score,
-                f"{self.result_key}_result": score_result,
-                f"{self.result_key}_reasoning": reason,
-                f"{self.result_key}_status": "completed",
-                f"{self.result_key}_threshold": self._threshold,
-                f"{self.result_key}_properties": llm_properties,
+                f"{self._result_key}_score": score,
+                f"{self._result_key}_result": score_result,
+                f"{self._result_key}_passed": score_result == "pass",
+                f"{self._result_key}_reasoning": reason,
+                f"{self._result_key}_status": "completed",
+                f"{self._result_key}_threshold": self._threshold,
+                f"{self._result_key}_properties": llm_properties,
             }
             return response_dict
 
@@ -1054,23 +1061,24 @@ async def _real_call(self, **kwargs):
 
     def _not_applicable_result(
         self, error_message: str, threshold: Union[int, float]
-    ) -> Dict[str, Union[str, float, Dict]]:
+    ) -> Dict[str, Union[str, float, Dict, None]]:
         """Return a result indicating that the tool call is not applicable for evaluation.
 
         :param error_message: The error message indicating why the evaluation is not applicable.
         :type error_message: str
         :param threshold: The threshold value for the evaluation.
         :type threshold: Union[int, float]
         :return: A dictionary containing the result of the evaluation.
-        :rtype: Dict[str, Union[str, float]]
+        :rtype: Dict[str, Union[str, float, None]]
         """
         return {
-            f"{self.result_key}_score": None,
-            f"{self.result_key}_result": "not_applicable",
-            f"{self.result_key}_reasoning": f"Not applicable: {error_message}",
-            f"{self.result_key}_status": "skipped",
-            f"{self.result_key}_threshold": threshold,
-            f"{self.result_key}_properties": {},
+            f"{self._result_key}_score": None,
+            f"{self._result_key}_result": "not_applicable",
+            f"{self._result_key}_passed": None,
+            f"{self._result_key}_reasoning": f"Not applicable: {error_message}",
+            f"{self._result_key}_status": "skipped",
+            f"{self._result_key}_threshold": threshold,
+            f"{self._result_key}_properties": {},
         }
 
     def _extract_needed_tool_definitions(self, tool_calls, tool_definitions):

assets/evaluators/builtin/tool_call_accuracy/evaluator/tool_call_accuracy.prompty

@@ -54,6 +54,16 @@ Evaluate based on these factors:
 
 **Tool Assessment**: Focus solely on appropriate use of available tools, not on capabilities beyond what tools can provide.
 
+## Status: Skipped
+Before performing any evaluation, check for the following conditions. If ANY are true, return `status: "skipped"` immediately without scoring:
+1. **No tool calls to evaluate**: The TOOL CALLS TO BE EVALUATED section is empty (tool calls appearing only in the CONVERSATION section do not count).
+2. **Missing tool definitions**: Any tool call in TOOL CALLS TO BE EVALUATED references a tool that is not present in the TOOL DEFINITIONS.
+
+When skipped, return:
+```json
+{"score": null, "reasoning": "<explain why evaluation was skipped>", "status": "skipped", "properties": null}
+```
+
 
 # Ratings
 ## [Tool Call Accuracy: 1] (Irrelevant)
@@ -140,8 +150,11 @@ TOOL DEFINITIONS: {{tool_definitions}}
 # Tasks
 ## Please provide your evaluation for the assistant RESPONSE in relation to the user QUERY and tool definitions based on the Definitions and examples above.
 Your output should consist only of a JSON object that has the following keys:
-  - score: an integer value between 1 and 5 that represents the level of tool call success, based on the level definitions mentioned before. You need to be very precise when deciding on this level. Ensure you are correctly following the rating system based on the description of each level.
-  - reasoning: a string that explains your thought process to decide on the tool call accuracy level, based on the Chain of Thought structure. Start this string with 'Let's think step by step:'.
+  - score: an integer value between 1 and 5 that represents the level of tool call success, based on the level definitions mentioned before. You need to be very precise when deciding on this level. Ensure you are correctly following the rating system based on the description of each level. Set to null when status is "skipped".
+  - status: a string indicating the evaluation status. Must be one of:
+      - "completed": tool calls were present, tool definitions were available, and evaluation was performed.
+      - "skipped": evaluation was not performed because there were no tool calls to evaluate, or tool definitions were missing for the tool calls. When skipped, set score to null and properties to null.
+  - reasoning: a string that explains your thought process to decide on the tool call accuracy level, based on the Chain of Thought structure. Start this string with 'Let's think step by step:'. When status is "skipped", explain why the evaluation was skipped.
   - properties: a dictionary that contains the following keys:
         - tool_calls_made_by_agent: total number of tool calls made by the agent
         - correct_tool_calls_made_by_agent: total number of correct tool calls made by the agent

assets/evaluators/tests/common/base_evaluator_runner.py

@@ -156,6 +156,8 @@ def _extract_and_print_result(self, results: Dict[str, Any], test_label: str) ->
             Dictionary with standardized result fields.
         """
         score = results.get(self.result_key)
+        if score is None:
+            score = results.get(f"{self.result_key}_score")
 
         if f"{self.result_key}_error_message" not in results and score != "not applicable":
             for field in self.expected_result_fields:
@@ -246,15 +248,23 @@ def assert_pass(self, result_data: Dict[str, Any]):
         self._assert_pass_result(result_data)
 
     def assert_not_applicable(self, result_data: Dict[str, Any]):
-        """Assert a not-applicable result (intermediate response).
+        """Assert a not-applicable result (intermediate response or skipped evaluation).
 
         Args:
             result_data: Dictionary containing evaluation result data.
 
         Raises:
             AssertionError: If the result is not a valid not-applicable result.
         """
-        self._assert_pass_result(result_data)
+        label_key = "label"
+        score_key = "score"
+        if result_data[label_key] == "not_applicable":
+            assert result_data[label_key] == "not_applicable", \
+                f"Expected 'not_applicable' but got '{result_data[label_key]}'"
+            assert result_data[score_key] is None, \
+                f"Expected score to be None for not-applicable result but got '{result_data[score_key]}'"
+        else:
+            self._assert_pass_result(result_data)
         assert "Not applicable" in result_data.get("reason", ""), \
             f"Expected reason to contain 'Not applicable' but got '{result_data.get('reason')}'"
 

assets/evaluators/tests/test_evaluators_quality/test_tool_call_accuracy_evaluator_quality.py

@@ -4,6 +4,7 @@
 """Quality tests for Tool Call Accuracy Evaluator with real flow execution."""
 
 import pytest
+from typing import List
 from ..common.base_quality_evaluator_runner import BaseQualityEvaluatorRunner, ExpectedResult
 from ...builtin.tool_call_accuracy.evaluator._tool_call_accuracy import ToolCallAccuracyEvaluator
 
@@ -16,6 +17,18 @@ class TestToolCallAccuracyEvaluatorQuality(BaseQualityEvaluatorRunner):
     Tests actual LLM evaluation with real flow execution (no mocking).
     """
 
+    @property
+    def expected_result_fields(self) -> List[str]:
+        """Get the expected result fields for tools evaluators."""
+        return [
+            f"{self.result_key}_score",
+            f"{self.result_key}_reasoning",
+            f"{self.result_key}_status",
+            f"{self.result_key}_threshold",
+            f"{self.result_key}_result",
+            f"{self.result_key}_properties",
+        ]
+
     evaluator_type = ToolCallAccuracyEvaluator
 
     def test_pass_single_call(self) -> None: