Merge pull request #23 from microsoft/test-optional

Make functional testing optional and require confirmation

Author: tylerpayne

README.md

@@ -28,7 +28,7 @@ Use `--constraints [CODE ...]` to customize output.
 
 ### 🛠️ Functional testing
 
-MCP servers are intended to be used by LLM agents, so we test them with an LLM agent. Using your specified LLM, the interviewer generates a test plan based on the MCP server's capabilities and then executes that plan (e.g. by calling tools), collecting statistics about observed tool behavior.
+MCP servers are intended to be used by LLM agents, so we can optionally test them with an LLM agent. When enabled with the `--test` flag, the interviewer uses your specified LLM to generate a test plan based on the MCP server's capabilities and then executes that plan (e.g. by calling tools), collecting statistics about observed tool behavior.
 
 ### 🧪 LLM evaluation
 
@@ -68,6 +68,8 @@ Use `--reports [CODE ...]` to customize output.
 
 ⚠️ ***mcp-interviewer arbitrarily executes the provided MCP server command in a child process. Whenever possible, run your server in a container like in the examples below to isolate the server from your host system.***
 
+🚨 ***mcp-interviewer actually invokes the server's tools, DO NOT use mcp-interviewer with admin privileges etc***
+
 ```bash
 # Command to run npx safely inside a Docker container
 NPX_CONTAINER="docker run -i --rm node:lts npx"
@@ -102,21 +104,30 @@ Which will generate a report like [this](./mcp-interview.md).
 
 ### CLI
 
+**Key Flags:**
+- `--test`: Enable functional testing (disabled by default for faster execution)
+- `--judge`: Enable experimental LLM evaluation of tools and tests
+- `--reports [CODE ...]`: Customize which report sections to include
+- `--constraints [CODE ...]`: Customize which constraints to check
+
 ```bash
 # Docker command to run uvx inside a container
 UVX_CONTAINER="docker run -i --rm ghcr.io/astral-sh/uv:python3.12-alpine uvx"
 
-# Constraint checking, functional testing, default report generation
+# Basic constraint checking and server inspection (no functional testing)
 mcp-interviewer --model gpt-4o "$UVX_CONTAINER mcp-server-fetch"
 
+# Constraint checking with functional testing and default report generation
+mcp-interviewer --model gpt-4o --test "$UVX_CONTAINER mcp-server-fetch"
+
 # Constraint checking, functional testing, LLM evaluation, default report generation
-mcp-interviewer --model gpt-4o --judge "$UVX_CONTAINER mcp-server-fetch"
+mcp-interviewer --model gpt-4o --test --judge "$UVX_CONTAINER mcp-server-fetch"
 
-# Constraint checking, functional testing, custom report generation
-mcp-interviewer --model gpt-4o --reports SI TS FT CV "$UVX_CONTAINER mcp-server-fetch"
+# Constraint checking with functional testing and custom report generation
+mcp-interviewer --model gpt-4o --test --reports SI TS FT CV "$UVX_CONTAINER mcp-server-fetch"
 
-# Custom constraint checking, functional testing, report generation
-mcp-interviewer --model gpt-4o --select OTC ONL "$UVX_CONTAINER mcp-server-fetch"
+# Custom constraint checking with functional testing and report generation
+mcp-interviewer --model gpt-4o --test --constraints OTC ONL "$UVX_CONTAINER mcp-server-fetch"
 
 # Test remote servers
 mcp-interviewer --model gpt-4o "https://my-mcp-server.com/sse"
@@ -173,8 +184,8 @@ params = StdioServerParameters(
     args=["run", "-i", "--rm", "node:lts", "npx", "-y", "@modelcontextprotocol/server-everything"]
 )
 
-interviewer = MCPInterviewer(client, "gpt-4o")
-interview = await interviewer.score_server(params)
+interviewer = MCPInterviewer(client, "gpt-4o", should_run_functional_test=True)
+interview = await interviewer.interview_server(params)
 ```
 
 ## Limitations

src/mcp_interviewer/cli.py

@@ -1,4 +1,5 @@
 import argparse
+import sys
 
 
 def cli():
@@ -72,6 +73,16 @@ def cli():
         nargs="+",
         help="Specify which constraint violations to check (all enabled by default). Can use full names (e.g., openai-tool-count, openai-name-length) or shorthand codes (e.g., OTC, ONL, ONP, OTL, OA)",
     )
+    parser.add_argument(
+        "--test",
+        action="store_true",
+        help="Enable functional testing of the server",
+    )
+    parser.add_argument(
+        "--accept-risk",
+        action="store_true",
+        help="Bypass user confirmation of functional test risk.",
+    )
 
     args = parser.parse_args()
 
@@ -130,6 +141,22 @@ def cli():
 
         params = StdioServerParameters(command=params_command, args=params_args)
 
+    # Handle the --judge flag which enables experimental judging operations (disabled by default)
+    should_judge_tool = args.judge or args.judge_tools
+    should_judge_functional_test = args.judge or args.judge_test
+
+    if args.test:
+        print(
+            "🚨 MCP Interviewer will make tool call requests to your MCP server. Depending on the server's capabilities this can lead to irreversible outcomes (e.g. deleting files)."
+        )
+        accept_risk = args.accept_risk
+        while not accept_risk:
+            input_str = input("Do you accept this risk? y|[n]: ").strip().lower()
+            if not input_str or input_str == "n":
+                sys.exit(1)
+            else:
+                accept_risk = input_str == "y"
+
     import importlib
 
     module, client = args.client.rsplit(".")
@@ -155,17 +182,14 @@ def cli():
 
     from .main import main
 
-    # Handle the --judge flag which enables experimental judging operations (disabled by default)
-    should_judge_tool = args.judge or args.judge_tools
-    should_judge_functional_test = args.judge or args.judge_test
-
     main(
         client,
         args.model,
         params,
         out_dir=args.out_dir,
         should_judge_tool=should_judge_tool,
         should_judge_functional_test=should_judge_functional_test,
+        should_run_functional_test=args.test,
         custom_reports=args.reports,
         no_collapse=args.no_collapse,
         selected_constraints=args.constraints,

src/mcp_interviewer/constraints/base.py

@@ -198,6 +198,8 @@ def test(
         Yields:
             ConstraintViolation: Violations from all tool results
         """
+        if server.functional_test_scorecard is None:
+            return
         for step in server.functional_test_scorecard.steps:
             if isinstance(step.tool_output, CallToolResult):
                 yield from self.test_tool_result(step.tool_output)

src/mcp_interviewer/interviewer/_interviewer.py

@@ -61,6 +61,7 @@ def __init__(
         self,
         client: Client,
         model: str,
+        should_run_functional_test: bool = False,
         should_judge_tool: bool = False,
         should_judge_functional_test: bool = False,
     ):
@@ -74,6 +75,7 @@ def __init__(
         """
         self._client = client
         self._model = model
+        self._should_run_functional_test = should_run_functional_test
         self._should_judge_tool = should_judge_tool
         self._should_judge_functional_test = should_judge_functional_test
         self._request_counters = create_request_counters()
@@ -381,22 +383,29 @@ async def interview_server(self, params: ServerParameters) -> ServerScoreCard:
                         tool_scorecards = []
 
                     # Phase 3: Functional Testing
-                    logger.info("=" * 60)
-                    logger.info("PHASE 3: Functional Testing")
-                    logger.info("=" * 60)
-                    functional_test = await self.generate_functional_test(server)
-
-                    (
-                        functional_test_output,
-                        functional_test_step_outputs,
-                    ) = await self.execute_functional_test(session, functional_test)
-
-                    # Judge functional test
-                    functional_test_scorecard = await self.judge_functional_test(
-                        functional_test,
-                        functional_test_output,
-                        functional_test_step_outputs,
-                    )
+                    if self._should_run_functional_test:
+                        logger.info("=" * 60)
+                        logger.info("PHASE 3: Functional Testing")
+                        logger.info("=" * 60)
+
+                        functional_test = await self.generate_functional_test(server)
+
+                        (
+                            functional_test_output,
+                            functional_test_step_outputs,
+                        ) = await self.execute_functional_test(session, functional_test)
+
+                        # Judge functional test
+                        functional_test_scorecard = await self.judge_functional_test(
+                            functional_test,
+                            functional_test_output,
+                            functional_test_step_outputs,
+                        )
+                    else:
+                        logger.info("=" * 60)
+                        logger.info("PHASE 3: Functional Testing - SKIPPED")
+                        logger.info("=" * 60)
+                        functional_test_scorecard = None
 
                     # Create final scorecard
                     logger.info("Creating final server scorecard")

src/mcp_interviewer/main.py

@@ -22,6 +22,7 @@ async def amain(
     out_dir=Path("."),
     should_judge_tool: bool = False,
     should_judge_functional_test: bool = False,
+    should_run_functional_test: bool = False,
     custom_reports: list[str] | None = None,
     no_collapse: bool = False,
     selected_constraints: list[str] | None = None,
@@ -39,12 +40,17 @@ async def amain(
         out_dir: Directory to save output files (default: current directory)
         should_judge_tool: Whether to perform expensive experimental LLM judging of tools (default: False)
         should_judge_functional_test: Whether to perform expensive experimental LLM judging of functional tests (default: False)
+        should_run_functional_test: Whether to run functional tests (default: False)
         custom_reports: List of specific report names to include
         no_collapse: If True, don't use collapsible sections in the report (default: False)
         selected_constraints: List of constraint names or codes to check (all if None)
     """
     interviewer = MCPInterviewer(
-        client, model, should_judge_tool, should_judge_functional_test
+        client,
+        model,
+        should_run_functional_test,
+        should_judge_tool,
+        should_judge_functional_test,
     )
     interview = await interviewer.interview_server(params)
 
@@ -105,6 +111,7 @@ def main(
     out_dir=Path("."),
     should_judge_tool: bool = False,
     should_judge_functional_test: bool = False,
+    should_run_functional_test: bool = False,
     custom_reports: list[str] | None = None,
     no_collapse: bool = False,
     selected_constraints: list[str] | None = None,
@@ -118,6 +125,7 @@ def main(
         out_dir: Directory to save output files (default: current directory)
         should_judge_tool: Whether to perform expensive experimental LLM judging of tools (default: False)
         should_judge_functional_test: Whether to perform expensive experimental LLM judging of functional tests (default: False)
+        should_run_functional_test: Whether to run functional tests (default: False)
         custom_reports: List of specific report names to include
         no_collapse: If True, don't use collapsible sections in the report (default: False)
         selected_constraints: List of constraint names or codes to check (all if None)
@@ -130,6 +138,7 @@ def main(
             out_dir,
             should_judge_tool,
             should_judge_functional_test,
+            should_run_functional_test,
             custom_reports,
             no_collapse,
             selected_constraints,

src/mcp_interviewer/models.py

@@ -206,4 +206,4 @@ class Server(BaseModel):
 class ServerScoreCard(Server):
     model: str
     tool_scorecards: list[ToolScoreCard]
-    functional_test_scorecard: FunctionalTestScoreCard
+    functional_test_scorecard: FunctionalTestScoreCard | None

src/mcp_interviewer/reports/functional_test/failed_tests.py

@@ -1,6 +1,6 @@
 """Failed tests report generation."""
 
-from ...models import ServerScoreCard
+from ...models import FunctionalTestScoreCard, ServerScoreCard
 from ..base import BaseReport
 from .test_step import TestStepReport
 
@@ -20,34 +20,40 @@ def __init__(self, scorecard: ServerScoreCard, detailed: bool = False):
         """Initialize and build the failed tests report."""
         super().__init__(scorecard)
         self.detailed = detailed
-        if self._has_failed_tests():
-            self._build()
+        self._build(scorecard.functional_test_scorecard)
 
-    def _has_failed_tests(self) -> bool:
+    def _has_failed_tests(
+        self, functional_test_scorecard: FunctionalTestScoreCard
+    ) -> bool:
         """Check if there are any failed tests."""
-        if not self._scorecard.functional_test_scorecard:
-            return False
-
-        for step in self._scorecard.functional_test_scorecard.steps:
+        for step in functional_test_scorecard.steps:
             # Check if any evaluation criteria failed
             for field_name in step.model_fields_set:
                 field_value = getattr(step, field_name)
                 if hasattr(field_value, "score") and field_value.score == "fail":
                     return True
         return False
 
-    def _build(self):
+    def _build(self, functional_test_scorecard: FunctionalTestScoreCard | None):
         """Build the failed tests section."""
+        if functional_test_scorecard is None:
+            return
+
+        if self._has_failed_tests(functional_test_scorecard):
+            return
+
         if self.detailed:
-            self._add_detailed_failed_test_steps()
+            self._add_detailed_failed_test_steps(functional_test_scorecard)
         else:
-            self._add_failed_test_steps()
+            self._add_failed_test_steps(functional_test_scorecard)
 
-    def _add_failed_test_steps(self) -> "FailedTestsReport":
+    def _add_failed_test_steps(
+        self, functional_test_scorecard: FunctionalTestScoreCard
+    ) -> "FailedTestsReport":
         """Add a summary of failed test steps."""
         self.add_title("Failed Test Steps (🤖)", 2)
 
-        for i, step in enumerate(self._scorecard.functional_test_scorecard.steps):
+        for i, step in enumerate(functional_test_scorecard.steps):
             has_failure = False
             failures = []
 
@@ -66,11 +72,13 @@ def _add_failed_test_steps(self) -> "FailedTestsReport":
         self.add_blank_line()
         return self
 
-    def _add_detailed_failed_test_steps(self) -> "FailedTestsReport":
+    def _add_detailed_failed_test_steps(
+        self, functional_test_scorecard: FunctionalTestScoreCard
+    ) -> "FailedTestsReport":
         """Add detailed information about failed test steps."""
         self.add_title("Failed Test Steps (🤖)", 2)
 
-        for i, step in enumerate(self._scorecard.functional_test_scorecard.steps):
+        for i, step in enumerate(functional_test_scorecard.steps):
             # Use TestStepReport with show_only_failures=True
             # This will only build the report if the step has failures
             step_report = TestStepReport(

src/mcp_interviewer/reports/functional_test/score_summary.py

@@ -107,7 +107,11 @@ def _build(self):
                     ]
                 )
 
-                if self.detailed and self._scorecard.functional_test_scorecard.steps:
+                if (
+                    self.detailed
+                    and self._scorecard.functional_test_scorecard
+                    and self._scorecard.functional_test_scorecard.steps
+                ):
                     # Show individual test steps
                     for i, step in enumerate(
                         self._scorecard.functional_test_scorecard.steps

src/mcp_interviewer/reports/functional_test/test.py

@@ -31,11 +31,13 @@ def __init__(self, scorecard: ServerScoreCard, include_evaluations: bool = True)
 
     def _build(self):
         """Build the functional test results section."""
+        test = self._scorecard.functional_test_scorecard
+        if test is None:
+            return
+
         # Check if scoring was disabled
         self.start_collapsible("Functional Test Results", 2)
 
-        test = self._scorecard.functional_test_scorecard
-
         # Test plan
         if self.include_evaluations and test.plan:
             self.add_text("**Test Plan (🤖):**")

src/mcp_interviewer/reports/statistics/tool_call_statistics.py

@@ -62,13 +62,17 @@ def add_stats_table_row(
 
     def _build(self):
         """Build the tool output analysis section."""
+        functional_test_scorecard = self._scorecard.functional_test_scorecard
+        if functional_test_scorecard is None:
+            return
+
         tokenizer = encoding_for_model("gpt-4o")
 
         # Never collapse this section - always show the statistics
         self.add_title("Tool Call Statistics", 2)
         self.add_table_header(["Metric", "Total", "Average", "Min", "Max"])
 
-        steps = self._scorecard.functional_test_scorecard.steps
+        steps = functional_test_scorecard.steps
 
         # Analyze tool calls
         total_attempted = len(steps)