test: Enable tool call evaluator

Author: clareliguori

e2e_tests/python/main.py

@@ -1,6 +1,7 @@
 import json
 import logging
 import os
+import sys
 from typing import Any
 
 from botocore.config import Config
@@ -150,6 +151,10 @@ def task_fn(case: Case) -> TaskOutput:
     print(f"Reasons: {report.reasons}")
     print(f"Tools called: {called_tools}")
 
+    # Exit with non-zero code if test fails
+    if not report.test_passes:
+        sys.exit(1)
+
 
 if __name__ == "__main__":
     main()

e2e_tests/python/requirements.txt

@@ -1,5 +1,5 @@
 strands-agents==1.15.0
-strands-agents-evals==0.1.0
+strands-agents-evals==0.1.1
 mcp==1.23.1
 uvicorn==0.38.0
 boto3==1.42.3

e2e_tests/python/tool_call_evaluator.py

@@ -26,8 +26,7 @@ def evaluate(self, evaluation_case: EvaluationData[InputT, OutputT]) -> list[Eva
 
         for event in evaluation_case.actual_trajectory:
             if isinstance(event, dict) and 'name' in event:
-                result = event.get('tool_result', '')
-                if isinstance(result, str) and result.startswith('Error:'):
+                if event.get('is_error', False):
                     tool_errors.append(event['name'])
                 else:
                     tool_successes.add(event['name'])