test(evals): inject tool failures in workflow evals to exercise report-problem

Author: RobertCrupa

evals/shared/types.ts

@@ -43,6 +43,12 @@ export type WorkflowTestCase = {
     maxTurns?: number;
     /** Tools to enable for this test (optional, e.g., ["actors", "docs", "apify/rag-web-browser"]) */
     tools?: string[];
+    /**
+     * Tool names the harness force-fails with a synthetic INTERNAL_ERROR carrying the real
+     * report-problem nudge (optional). Lets an eval deterministically throw a nudge-eligible error
+     * that the live server + API cannot reproduce on demand. See mcp_client.ts.
+     */
+    failTools?: string[];
 } & BaseTestCase;
 
 /**

evals/workflows/README.md

@@ -461,6 +461,7 @@ File: `test-cases.json`
 **Optional:**
 - `maxTurns` - Override default (10)
 - `tools` - List of tools to enable for this test (e.g., `["actors", "docs", "apify/rag-web-browser"]`). If omitted, all default tools are enabled. Passed to MCP server as `--tools` argument.
+- `failTools` - Tool names the harness force-fails with a synthetic `INTERNAL_ERROR` result carrying the real `report-problem` nudge, instead of calling the server (e.g. `["call-actor"]`). Use it to deterministically throw a nudge-eligible error that the live server + API cannot reproduce on demand, e.g. to test that the agent proactively calls `report-problem` after a failure. See `mcp_client.ts`.
 
 ## Performance
 

evals/workflows/mcp_client.ts

@@ -6,6 +6,7 @@
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
 
+import { REPORT_PROBLEM_NUDGE } from '../../src/tools/report_problem/report_problem.js';
 import type { McpTool, McpToolCall, McpToolResult } from './types.js';
 
 export class McpClient {
@@ -14,13 +15,16 @@ export class McpClient {
     private tools: McpTool[] = [];
     private instructions: string | null = null;
     private toolTimeoutMs: number;
+    private failTools: Set<string>;
 
     /**
      * Create MCP client
      * @param toolTimeoutSeconds - Timeout for tool calls in seconds (default: 60)
+     * @param failTools - Tool names to force-fail with a synthetic INTERNAL_ERROR (see callTool)
      */
-    constructor(toolTimeoutSeconds = 60) {
+    constructor(toolTimeoutSeconds = 60, failTools: string[] = []) {
         this.toolTimeoutMs = toolTimeoutSeconds * 1000;
+        this.failTools = new Set(failTools);
     }
 
     /**
@@ -112,6 +116,18 @@ export class McpClient {
             throw new Error('MCP client is not started');
         }
 
+        // Deterministic failure injection: force-fail listed tools with a synthetic INTERNAL_ERROR
+        // carrying the real server nudge, so the agent sees exactly what production appends on a genuine
+        // failure. The live server + API cannot reproduce an INTERNAL_ERROR on demand, so evals that
+        // exercise error-driven behavior (e.g. report-problem) rely on this.
+        if (this.failTools.has(toolCall.name)) {
+            return {
+                toolName: toolCall.name,
+                success: false,
+                error: `The ${toolCall.name} tool failed with an internal error.\n\n${REPORT_PROBLEM_NUDGE}`,
+            };
+        }
+
         try {
             const response = await this.client.callTool(
                 {

evals/workflows/run_workflow_evals.ts

@@ -77,7 +77,7 @@ async function runSingleTest(
     logWithPrefix(testId, `[${index + 1}/${total}] Running...`);
 
     // Create FRESH MCP instance per test for isolation
-    const mcpClient = new McpClient(argv.toolTimeout);
+    const mcpClient = new McpClient(argv.toolTimeout, testCase.failTools);
     const startTime = Date.now();
     let result: EvaluationResult;
 

evals/workflows/test_cases.json

@@ -197,6 +197,15 @@
       "reference": "The agent must find a Reddit scraper and call-actor it for the r/technology subreddit, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several posts, each with a title and an upvote count.",
       "tools": ["actors", "get-dataset-items"],
       "maxTurns": 18
+    },
+    {
+      "id": "report-problem-on-tool-error",
+      "category": "report-problem",
+      "query": "Use the apify/rag-web-browser Actor to scrape https://example.com and tell me the page title.",
+      "reference": "While running the Actor, the call-actor tool fails with an internal error, so the task cannot be completed. The user did NOT ask for any problem to be reported. This tests whether the agent proactively reports the blocker after the error. PASS only if the agent calls the report-problem tool with a non-empty message describing what it was trying to do and what went wrong. FAIL if report-problem is not called (e.g., the agent only tells the user it could not complete the task).",
+      "tools": ["actors", "report-problem"],
+      "failTools": ["call-actor"],
+      "maxTurns": 8
     }
   ]
 }