feat: add invalid input as a nudge case for report tool

Author: RobertCrupa

src/tools/report_problem/report_problem.ts

@@ -26,23 +26,33 @@ const ACKNOWLEDGEMENT = `Problem reported. Thank you — this helps the Apify te
 export const REPORT_PROBLEM_NUDGE = `If you cannot resolve this yourself, report it to the Apify team by calling ${HELPER_TOOLS.PROBLEM_REPORT} (describe what you were doing and this error) before telling the user you could not complete the task.`;
 
 /**
- * Failure categories that represent expected, user-resolvable control-flow states (fix input, pay,
- * approve, fix token) rather than genuine Apify defects. The nudge is suppressed for these so it does
- * not flood the channel with paywalls/approvals or steer the agent to give up instead of surfacing the
- * payment/approval URL. `INTERNAL_ERROR` and an absent/unknown category still get the nudge.
+ * Softer nudge for INVALID_INPUT failures (payment/standby short-circuits, actor input rejected at
+ * runtime, etc.). Most of these are the agent's own mistake, so this doesn't push it to report before
+ * giving up — but some are a genuine schema/Actor mismatch the agent would otherwise never think to
+ * report, so it still surfaces the tool's existence.
+ */
+export const REPORT_PROBLEM_INVALID_INPUT_NUDGE = `If this looks like a bug rather than something fixable by adjusting your request, report it by calling ${HELPER_TOOLS.PROBLEM_REPORT} (describe what you were doing and this error).`;
+
+/**
+ * Failure categories that represent expected, user-resolvable control-flow states (pay, approve, fix
+ * token) rather than potential Apify defects. The nudge is suppressed for these so it does not flood
+ * the channel with paywalls/approvals or steer the agent to give up instead of surfacing the
+ * payment/approval URL. `INVALID_INPUT` gets the softer {@link REPORT_PROBLEM_INVALID_INPUT_NUDGE}
+ * instead of being suppressed outright. `INTERNAL_ERROR` and an absent/unknown category still get the
+ * full {@link REPORT_PROBLEM_NUDGE}.
  */
 const NON_NUDGE_FAILURE_CATEGORIES = new Set<string>([
-    FAILURE_CATEGORY.INVALID_INPUT,
     FAILURE_CATEGORY.PERMISSION_APPROVAL_REQUIRED,
     FAILURE_CATEGORY.AUTH,
 ]);
 
 /**
- * Append {@link REPORT_PROBLEM_NUDGE} to a failed tool result's text content, returning a shallow
- * copy with a new `content` array (never mutates the input). Returns the original unchanged unless the
- * result is an error with a text `content[]`, `report-problem` is actually served (`available`), the
- * failing tool is not `report-problem` itself, and `failureCategory` is not an expected,
- * user-resolvable state (see {@link NON_NUDGE_FAILURE_CATEGORIES}).
+ * Append a report-problem nudge to a failed tool result's text content, returning a shallow copy with
+ * a new `content` array (never mutates the input). Returns the original unchanged unless the result is
+ * an error with a text `content[]`, `report-problem` is actually served (`available`), the failing tool
+ * is not `report-problem` itself, and `failureCategory` is not an expected, user-resolvable state (see
+ * {@link NON_NUDGE_FAILURE_CATEGORIES}). Uses {@link REPORT_PROBLEM_INVALID_INPUT_NUDGE} for
+ * `INVALID_INPUT` and {@link REPORT_PROBLEM_NUDGE} otherwise.
  */
 export function appendReportProblemNudge<T>(
     result: T,
@@ -53,7 +63,11 @@ export function appendReportProblemNudge<T>(
     if (opts.failureCategory !== undefined && NON_NUDGE_FAILURE_CATEGORIES.has(opts.failureCategory)) return result;
     const r = result as { isError?: unknown; content?: unknown };
     if (r?.isError !== true || !Array.isArray(r.content)) return result;
-    return { ...r, content: [...r.content, { type: 'text', text: REPORT_PROBLEM_NUDGE }] } as T;
+    const nudge =
+        opts.failureCategory === FAILURE_CATEGORY.INVALID_INPUT
+            ? REPORT_PROBLEM_INVALID_INPUT_NUDGE
+            : REPORT_PROBLEM_NUDGE;
+    return { ...r, content: [...r.content, { type: 'text', text: nudge }] } as T;
 }
 
 export const reportProblemArgsSchema = z.object({

tests/integration/suite.ts

@@ -27,6 +27,19 @@ import { assertStatusMessagePropagated, captureInflightActorRunId, waitForRunAbo
 
 const AUTO_INJECTED_TOOL_NAMES = AUTO_INJECTED_TOOLS.map((t) => t.name);
 
+// report-problem is served only when telemetry is enabled — its sole function is forwarding
+// submissions via telemetry (see isToolServable in mcp/server.ts). This suite runs with telemetry
+// off (the helper default, so runs emit no Sentry/Segment data), so report-problem is absent from
+// the served default set here. Its served/hidden/acknowledge behavior is covered by the unit tests
+// (tests/unit/mcp.server.report_problem_gating.test.ts, tests/unit/tools.report_problem.test.ts).
+// These helpers give the default set as this suite actually sees it, with report-problem excluded.
+function servedDefaultTools(): ToolEntry[] {
+    return getDefaultTools('default').filter((tool) => tool.name !== HELPER_TOOLS.PROBLEM_REPORT);
+}
+function servedDefaultToolNames(): string[] {
+    return getDefaultToolNames().filter((name) => name !== HELPER_TOOLS.PROBLEM_REPORT);
+}
+
 // Helper to find tool by name, resolving categories for the given mode on each call.
 // This ensures we always validate against the correct mode-specific tool definition
 // (e.g. outputSchema may diverge between modes in the future).
@@ -240,10 +253,10 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
             it('should list all default tools and Actors', async () => {
                 client = await createClientFn();
                 const tools = await client.listTools();
-                expect(tools.tools.length).toEqual(getDefaultTools('default').length + defaults.actors.length + 4);
+                expect(tools.tools.length).toEqual(servedDefaultTools().length + defaults.actors.length + 4);
 
                 const names = getToolNames(tools);
-                expectToolNamesToContain(names, getDefaultToolNames());
+                expectToolNamesToContain(names, servedDefaultToolNames());
                 expectToolNamesToContain(names, DEFAULT_ACTOR_NAMES);
                 // get-actor-run + storage/abort helpers are auto-injected alongside call-actor.
                 expect(names).toContain(HELPER_TOOLS.ACTOR_RUNS_GET);
@@ -258,21 +271,17 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
 
                 // Should be equivalent to tools=actors,docs,apify/rag-web-browser
                 // Note: UI tools (search-actors-widget, fetch-actor-details-widget) are only available in apps mode
+                // report-problem is telemetry-gated and telemetry is off in this suite, so it is not listed.
                 const expectedActorsTools = ['fetch-actor-details', 'search-actors', 'call-actor'];
                 const expectedDocsTools = ['search-apify-docs', 'fetch-apify-docs'];
-                const expectedReportProblemTools = ['report-problem'];
                 const expectedActors = [actorNameToToolName('apify/rag-web-browser')];
 
-                const expectedTotal = expectedActorsTools.concat(
-                    expectedDocsTools,
-                    expectedReportProblemTools,
-                    expectedActors,
-                );
+                const expectedTotal = expectedActorsTools.concat(expectedDocsTools, expectedActors);
                 expect(names).toHaveLength(expectedTotal.length + 4);
 
                 expectToolNamesToContain(names, expectedActorsTools);
                 expectToolNamesToContain(names, expectedDocsTools);
-                expectToolNamesToContain(names, expectedReportProblemTools);
+                expect(names).not.toContain(HELPER_TOOLS.PROBLEM_REPORT);
                 expectToolNamesToContain(names, expectedActors);
                 expectToolNamesToContain(names, AUTO_INJECTED_TOOL_NAMES);
                 // get-actor-run should be automatically included when call-actor is present
@@ -282,27 +291,14 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
             });
 
             describe('report-problem', () => {
-                it('is listed by default and acknowledges a submission', async () => {
+                // report-problem is served only when telemetry is enabled; this suite runs with telemetry
+                // off, so end-to-end it must be absent. That gating is what we can verify here without
+                // emitting telemetry. The served path (listed for non-Anthropic clients, hidden from
+                // Anthropic clients, acknowledges a submission) is covered by the unit tests
+                // tests/unit/mcp.server.report_problem_gating.test.ts and tests/unit/tools.report_problem.test.ts.
+                it('is not served when telemetry is disabled', async () => {
                     client = await createClientFn();
                     const names = getToolNames(await client.listTools());
-                    expect(names).toContain(HELPER_TOOLS.PROBLEM_REPORT);
-
-                    const result = await client.callTool({
-                        name: HELPER_TOOLS.PROBLEM_REPORT,
-                        arguments: {
-                            message: 'fetch-actor-details did not explain the input schema clearly.',
-                            relatedTools: ['fetch-actor-details'],
-                        },
-                    });
-
-                    expect(result.isError).not.toBe(true);
-                    const content = result.content as { type: string; text: string }[];
-                    expect(content[0].text).toContain('Problem reported');
-                });
-
-                it('is hidden from Anthropic clients', async () => {
-                    client = await createClientFn({ clientName: 'claude-ai' });
-                    const names = getToolNames(await client.listTools());
                     expect(names).not.toContain(HELPER_TOOLS.PROBLEM_REPORT);
                 });
             });
@@ -336,9 +332,9 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
             it('should list all default tools and Actors when enableAddingActors is false', async () => {
                 client = await createClientFn({ enableAddingActors: false });
                 const names = getToolNames(await client.listTools());
-                expect(names.length).toEqual(getDefaultTools('default').length + defaults.actors.length + 4);
+                expect(names.length).toEqual(servedDefaultTools().length + defaults.actors.length + 4);
 
-                expectToolNamesToContain(names, getDefaultToolNames());
+                expectToolNamesToContain(names, servedDefaultToolNames());
                 expectToolNamesToContain(names, DEFAULT_ACTOR_NAMES);
                 expectToolNamesToContain(names, AUTO_INJECTED_TOOL_NAMES);
                 // get-actor-run should be automatically included when call-actor is present
@@ -1755,7 +1751,9 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
                 validateStructuredOutputForTool(result, HELPER_TOOLS.DOCS_FETCH, 'default');
             });
 
-            it.for(Object.keys(getCategoryTools('default')))(
+            // report-problem is telemetry-gated (off in this suite), so it can't load standalone here;
+            // its gating is covered by unit tests. Exclude it from the category sweep.
+            it.for(Object.keys(getCategoryTools('default')).filter((category) => category !== 'report-problem'))(
                 'should load correct tools for %s category',
                 async (category) => {
                     client = await createClientFn({
@@ -1929,9 +1927,9 @@ export function createIntegrationTestsSuite(options: IntegrationTestsSuiteOption
                     // Test with enableAddingActors = false via env var
                     client = await createClientFn({ enableAddingActors: false, useEnv: true });
                     const names = getToolNames(await client.listTools());
-                    expect(names.length).toEqual(getDefaultTools('default').length + defaults.actors.length + 4);
+                    expect(names.length).toEqual(servedDefaultTools().length + defaults.actors.length + 4);
 
-                    expectToolNamesToContain(names, getDefaultToolNames());
+                    expectToolNamesToContain(names, servedDefaultToolNames());
                     expectToolNamesToContain(names, DEFAULT_ACTOR_NAMES);
                     expectToolNamesToContain(names, AUTO_INJECTED_TOOL_NAMES);
                     // get-actor-run should be automatically included when call-actor is present

tests/unit/tools.report_problem.test.ts

@@ -2,6 +2,7 @@ import { describe, expect, it } from 'vitest';
 
 import { FAILURE_CATEGORY, HELPER_TOOLS } from '../../src/const.js';
 import {
+    REPORT_PROBLEM_INVALID_INPUT_NUDGE,
     REPORT_PROBLEM_NUDGE,
     appendReportProblemNudge,
     reportProblem,
@@ -98,7 +99,6 @@ describe('reportProblem', () => {
         });
 
         it.each([
-            ['INVALID_INPUT (payment/standby)', FAILURE_CATEGORY.INVALID_INPUT],
             ['PERMISSION_APPROVAL_REQUIRED', FAILURE_CATEGORY.PERMISSION_APPROVAL_REQUIRED],
             ['AUTH', FAILURE_CATEGORY.AUTH],
         ])('does not append for the user-resolvable category %s', (_label, failureCategory) => {
@@ -110,6 +110,16 @@ describe('reportProblem', () => {
             expect(nudgeCount(result)).toBe(0);
         });
 
+        it('appends the softer nudge for INVALID_INPUT (payment/standby, or a genuine input bug)', () => {
+            const result = appendReportProblemNudge(errorResult(), {
+                failingToolName: HELPER_TOOLS.ACTOR_CALL,
+                available: true,
+                failureCategory: FAILURE_CATEGORY.INVALID_INPUT,
+            });
+            expect(result.content.filter((c) => c.text === REPORT_PROBLEM_INVALID_INPUT_NUDGE)).toHaveLength(1);
+            expect(nudgeCount(result)).toBe(0);
+        });
+
         it('appends for a genuine INTERNAL_ERROR failure', () => {
             const result = appendReportProblemNudge(errorResult(), {
                 failingToolName: HELPER_TOOLS.ACTOR_CALL,