feat: Add report-problem tool for agents to report issues

README.md

@@ -243,6 +243,7 @@ Here is an overview list of all the tools provided by the Apify MCP Server.
 Legend for the **Enabled by default** column:
 - ✅ — in the default tool set.
 - ⚡ — auto-injected when `call-actor`, `add-actor`, an Actor tool, or `get-actor-run` is present (which is true in the default configuration).
+- ✅¹ — served by default, but only when telemetry is enabled and the client is not an Anthropic surface (Claude.ai / Claude Desktop / Claude Code). To disable, pass an explicit `tools=` list that omits it.
 
 | Tool name | Category | Description | Enabled by default |
 | :--- | :--- | :--- | :---: |
@@ -256,6 +257,7 @@ Legend for the **Enabled by default** column:
 | `search-apify-docs` | docs | Search the Apify documentation for relevant pages. | ✅ |
 | `fetch-apify-docs` | docs | Fetch the full content of an Apify documentation page by its URL. | ✅ |
 | [`apify--rag-web-browser`](https://apify.com/apify/rag-web-browser) | Actor (see [tool configuration](#tools-configuration)) | An Actor tool to browse the web. | ✅ |
+| `report-problem` | dev | Report a problem with an Apify tool or Actor to the Apify team. | ✅¹ |
 | `get-actor-run-list` | runs | Get a list of an Actor's runs, filterable by status. |  |
 | `get-actor-log` | runs | Retrieve the logs for a specific Actor run. |  |
 | `get-dataset` | storage | Get metadata about a specific dataset. |  |
@@ -292,6 +294,8 @@ When no query parameters are provided, the MCP server loads the following `tools
 
 If the tools parameter is specified, only the listed tools or categories will be enabled – no default tools will be included.
 
+`report-problem` is served by default (subject to the gating in the footnote above) but lives in the `dev` category, so an explicit `tools=dev` selects it too. To disable it, pass an explicit `tools=` list that omits it (e.g. `tools=actors,docs`).
+
 > **Easy configuration:**
 >
 > Use the [UI configurator](https://mcp.apify.com/) to configure your server, then copy the configuration to your client.

evals/shared/types.ts

@@ -43,6 +43,12 @@ export type WorkflowTestCase = {
     maxTurns?: number;
     /** Tools to enable for this test (optional, e.g., ["actors", "docs", "apify/rag-web-browser"]) */
     tools?: string[];
+    /**
+     * Tool names the harness force-fails with a synthetic INTERNAL_ERROR carrying the real
+     * report-problem nudge (optional). Lets an eval deterministically throw a nudge-eligible error
+     * that the live server + API cannot reproduce on demand. See mcp_client.ts.
+     */
+    failTools?: string[];
 } & BaseTestCase;
 
 /**

evals/workflows/README.md

@@ -461,6 +461,7 @@ File: `test-cases.json`
 **Optional:**
 - `maxTurns` - Override default (10)
 - `tools` - List of tools to enable for this test (e.g., `["actors", "docs", "apify/rag-web-browser"]`). If omitted, all default tools are enabled. Passed to MCP server as `--tools` argument.
+- `failTools` - Tool names the harness force-fails with a synthetic `INTERNAL_ERROR` result carrying the real `report-problem` nudge, instead of calling the server (e.g. `["call-actor"]`). Use it to deterministically throw a nudge-eligible error that the live server + API cannot reproduce on demand, e.g. to test that the agent proactively calls `report-problem` after a failure. See `mcp_client.ts`.
 
 ## Performance
 

evals/workflows/mcp_client.ts

@@ -6,6 +6,7 @@
 import { Client } from '@modelcontextprotocol/sdk/client/index.js';
 import { StdioClientTransport } from '@modelcontextprotocol/sdk/client/stdio.js';
 
+import { REPORT_PROBLEM_NUDGE } from '../../src/tools/report_problem/report_problem.js';
 import type { McpTool, McpToolCall, McpToolResult } from './types.js';
 
 export class McpClient {
@@ -14,13 +15,16 @@ export class McpClient {
     private tools: McpTool[] = [];
     private instructions: string | null = null;
     private toolTimeoutMs: number;
+    private failTools: Set<string>;
 
     /**
      * Create MCP client
      * @param toolTimeoutSeconds - Timeout for tool calls in seconds (default: 60)
+     * @param failTools - Tool names to force-fail with a synthetic INTERNAL_ERROR (see callTool)
      */
-    constructor(toolTimeoutSeconds = 60) {
+    constructor(toolTimeoutSeconds = 60, failTools: string[] = []) {
         this.toolTimeoutMs = toolTimeoutSeconds * 1000;
+        this.failTools = new Set(failTools);
     }
 
     /**
@@ -112,6 +116,18 @@ export class McpClient {
             throw new Error('MCP client is not started');
         }
 
+        // Deterministic failure injection: force-fail listed tools with a synthetic INTERNAL_ERROR
+        // carrying the real server nudge, so the agent sees exactly what production appends on a genuine
+        // failure. The live server + API cannot reproduce an INTERNAL_ERROR on demand, so evals that
+        // exercise error-driven behavior (e.g. report-problem) rely on this.
+        if (this.failTools.has(toolCall.name)) {
+            return {
+                toolName: toolCall.name,
+                success: false,
+                error: `The ${toolCall.name} tool failed with an internal error.\n\n${REPORT_PROBLEM_NUDGE}`,
+            };
+        }
+
         try {
             const response = await this.client.callTool(
                 {

evals/workflows/run_workflow_evals.ts

@@ -77,7 +77,7 @@ async function runSingleTest(
     logWithPrefix(testId, `[${index + 1}/${total}] Running...`);
 
     // Create FRESH MCP instance per test for isolation
-    const mcpClient = new McpClient(argv.toolTimeout);
+    const mcpClient = new McpClient(argv.toolTimeout, testCase.failTools);
     const startTime = Date.now();
     let result: EvaluationResult;
 

evals/workflows/test_cases.json

@@ -197,6 +197,15 @@
       "reference": "The agent must find a Reddit scraper and call-actor it for the r/technology subreddit, then call get-dataset-items with the datasetId from that run. PASS only if get-dataset-items was called with a datasetId argument and the final answer lists several posts, each with a title and an upvote count.",
       "tools": ["actors", "get-dataset-items"],
       "maxTurns": 18
+    },
+    {
+      "id": "report-problem-on-tool-error",
+      "category": "report-problem",
+      "query": "Use the apify/rag-web-browser Actor to scrape https://example.com and tell me the page title.",
+      "reference": "While running the Actor, the call-actor tool fails with an internal error, so the task cannot be completed. The user did NOT ask for any problem to be reported. This tests whether the agent proactively reports the blocker after the error. PASS only if the agent calls the report-problem tool with a non-empty message describing what it was trying to do and what went wrong. FAIL if report-problem is not called (e.g., the agent only tells the user it could not complete the task).",
+      "tools": ["actors", "report-problem"],
+      "failTools": ["call-actor"],
+      "maxTurns": 8
     }
   ]
 }

src/const.ts

@@ -54,9 +54,20 @@ export const HELPER_TOOLS = {
     STORE_SEARCH_WIDGET: 'search-actors-widget',
     DOCS_SEARCH: 'search-apify-docs',
     DOCS_FETCH: 'fetch-apify-docs',
+    PROBLEM_REPORT: 'report-problem',
 } as const;
 export type HelperToolName = (typeof HELPER_TOOLS)[keyof typeof HELPER_TOOLS];
 
+/**
+ * Client-name substrings (lowercased, matched against `clientInfo.name`) that `report-problem` is
+ * hidden from. Applied once per connection in the compose step, where the client is known.
+ * `report-problem` is hidden from Anthropic surfaces (Claude.ai / Claude Desktop / Claude Code /
+ * `local-agent-mode-apify`) pending the directory review. Substring matching covers new client builds
+ * without a maintained allowlist; over-matching only hides an optional tool, which is the safe failure
+ * mode.
+ */
+export const REPORT_PROBLEM_BLOCKED_CLIENTS: string[] = ['claude', 'anthropic', 'local-agent-mode-apify'];
+
 export const RAG_WEB_BROWSER = 'apify/rag-web-browser';
 export const RAG_WEB_BROWSER_WHITELISTED_FIELDS = ['query', 'maxResults', 'outputFormats'];
 export const RAG_WEB_BROWSER_ADDITIONAL_DESC = `Use this tool when user wants to GET or RETRIEVE actual data immediately (one-time data retrieval).