feat: Adding new set validator (#543)

Issue description:

Need a validator that checks if specific tools were called during test
execution without requiring a specific order, unlike the existing
ToolCallValidator which enforces sequential order. This enables more
vague prompts to be tested such as `My payments-latency SLO is
breaching. Can you investigate the root cause?` and test only whether a
specific tool(s) is being called or not. Introducing this for the new
tests being authored (will be part of a separate PR) for the new Change
Indicators tool introduced as part of
awslabs/mcp#1944

Description of changes:

Added ToolCallSetValidator class to mcp-testing/evals/core/validator.py:
- Validates that all expected tools are called, regardless of order
- Supports filtering out file-related tools via ignore_file_tools
parameter
- Returns detailed validation results including missing tools, extra
tools called, and the complete list of called tools
- Provides clear pass/fail criteria with reasoning for test results

Rollback procedure:

Yes, this commit can be safely reverted. It only adds a new validator
class without modifying existing functionality. No migration or cleanup
steps are required.

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Co-authored-by: Omkar Masur <omasur@amazon.com>

Author: oam-mit

mcp-testing/evals/core/validator.py

@@ -316,6 +316,84 @@ async def validate(
             }
 
 
+class ToolPresenceValidator(Validator):
+    """Validator that checks if specific tools were called, regardless of order and other tools being called."""
+
+    def __init__(self, expected_tools: List[str], ignore_file_tools: bool = False):
+        """Initialize tool call set validator.
+
+        Args:
+            expected_tools: List of tool names that must be called (order doesn't matter)
+            ignore_file_tools: If True, filter out file-related tools before validation
+        """
+        self.expected_tools = set(expected_tools)
+        self.ignore_file_tools = ignore_file_tools
+
+    def get_name(self) -> str:
+        """Return validator name."""
+        return 'Tool Call Set'
+
+    async def validate(
+        self,
+        captured_data: Dict[str, Any],
+    ) -> ValidationResult:
+        """Validate that all expected tools were called."""
+        logger.info('Validating tool calls (order-independent)...')
+
+        tool_calls = captured_data.get(TOOL_CALLS, [])
+        called_tools = [call['name'] for call in tool_calls]
+
+        if self.ignore_file_tools:
+            called_tools = [tool for tool in called_tools if tool not in PERMITTED_FILE_TOOLS]
+
+        called_tools_set = set(called_tools)
+        missing_tools = self.expected_tools - called_tools_set
+        extra_tools = called_tools_set - self.expected_tools
+
+        if not missing_tools:
+            reasoning = f'All expected tools called: {sorted(self.expected_tools)}'
+            if extra_tools:
+                reasoning += f' (also called: {sorted(extra_tools)})'
+            
+            return {
+                'validator_name': self.get_name(),
+                'overall_pass': True,
+                'criteria_results': [
+                    {
+                        'criterion': 'All expected tools called',
+                        'status': 'PASS',
+                        'reasoning': reasoning,
+                    }
+                ],
+                'raw_validation_output': {
+                    'expected_tools': sorted(self.expected_tools),
+                    'called_tools': called_tools,
+                    'missing_tools': [],
+                    'extra_tools': sorted(extra_tools),
+                    'ignore_file_tools': self.ignore_file_tools,
+                },
+            }
+        else:
+            return {
+                'validator_name': self.get_name(),
+                'overall_pass': False,
+                'criteria_results': [
+                    {
+                        'criterion': 'All expected tools called',
+                        'status': 'FAIL',
+                        'reasoning': f'Missing tools: {sorted(missing_tools)}. Called: {called_tools}',
+                    }
+                ],
+                'raw_validation_output': {
+                    'expected_tools': sorted(self.expected_tools),
+                    'called_tools': called_tools,
+                    'missing_tools': sorted(missing_tools),
+                    'extra_tools': sorted(extra_tools),
+                    'ignore_file_tools': self.ignore_file_tools,
+                },
+            }
+
+
 class BuildValidator(Validator):
     """Validator that runs build commands and checks exit code."""