fix(agent): corrected read-before-write prompt, install_skill capture-and-continue (#816)

Co-authored-by: Edwin Lim <edwin@posthog.com>

Author: gewenyu99

src/lib/agent/__tests__/output-signals.test.ts

@@ -129,4 +129,32 @@ describe('AgentOutputSignals', () => {
       expect(signals.usedManagedLogin()).toBe(true);
     });
   });
+
+  describe('skillInstallFailure', () => {
+    it('returns the detail after the marker', () => {
+      const signals = new AgentOutputSignals();
+      signals.push('some prose');
+      signals.push(
+        '[SKILL-INSTALL-FAILED] integration-nextjs-app-router — download timed out',
+      );
+
+      expect(signals.skillInstallFailure()).toBe(
+        'integration-nextjs-app-router — download timed out',
+      );
+    });
+
+    it("reports '' for a bare marker so the failure is never lost", () => {
+      const signals = new AgentOutputSignals();
+      signals.push('[SKILL-INSTALL-FAILED]');
+
+      expect(signals.skillInstallFailure()).toBe('');
+    });
+
+    it('is undefined when the skill installed fine', () => {
+      const signals = new AgentOutputSignals();
+      signals.push('[STATUS] Installing skill');
+
+      expect(signals.skillInstallFailure()).toBeUndefined();
+    });
+  });
 });

src/lib/agent/agent-interface.ts

@@ -858,6 +858,15 @@ export async function runAgent(
       analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark });
     }
 
+    // A failed install_skill is non-fatal — the agent continues best-effort
+    // without the skill — but every such run must be measurable.
+    const skillFailure = signals.skillInstallFailure();
+    if (skillFailure !== undefined) {
+      analytics.wizardCapture('agent continued without skill', {
+        detail: skillFailure,
+      });
+    }
+
     // Token usage comes from the SDK result message and is per agent run —
     // for the orchestrator that means per task, the secondary cost to watch.
     const usage = lastResultMessage?.usage as

src/lib/agent/output-signals.ts

@@ -25,6 +25,7 @@ const OUTPUT_SIGNALS = {
   API_ERROR: 'API Error:',
   MCP_MISSING: AgentSignals.ERROR_MCP_MISSING,
   RESOURCE_MISSING: AgentSignals.ERROR_RESOURCE_MISSING,
+  SKILL_INSTALL_FAILED: AgentSignals.SKILL_INSTALL_FAILED,
   WIZARD_REMARK: AgentSignals.WIZARD_REMARK,
 } as const;
 
@@ -89,6 +90,20 @@ export class AgentOutputSignals {
     return m ? m.join('\n') : undefined;
   }
 
+  /**
+   * Text after the `[SKILL-INSTALL-FAILED]` marker (skill id + reason),
+   * trimmed — or undefined when the skill installed fine. A marker with no
+   * trailing text still reports as '' so the failure is never lost.
+   */
+  skillInstallFailure(): string | undefined {
+    const marker = OUTPUT_SIGNALS.SKILL_INSTALL_FAILED;
+    for (const line of this.lines) {
+      const idx = line.indexOf(marker);
+      if (idx !== -1) return line.slice(idx + marker.length).trim();
+    }
+    return undefined;
+  }
+
   /** Text after the single `[WIZARD-REMARK]` marker, trimmed, or undefined. */
   remark(): string | undefined {
     const re = new RegExp(

src/lib/agent/runner/harness/pi/index.ts

@@ -566,6 +566,15 @@ export const piBackend: AgentHarness = {
         analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark });
       }
 
+      // A failed install_skill is non-fatal — the agent continues best-effort
+      // without the skill — but every such run must be measurable.
+      const skillFailure = signals.skillInstallFailure();
+      if (skillFailure !== undefined) {
+        analytics.wizardCapture('agent continued without skill', {
+          detail: skillFailure,
+        });
+      }
+
       // The skill plans events into .posthog-events.json then asks to remove it
       // on completion; pi's `rm` is fence-blocked, so the agent can't — clean it
       // up host-side rather than leave a stale (often empty) artifact (#15).

src/lib/agent/signals.ts

@@ -11,6 +11,14 @@ export const AgentSignals = {
   ERROR_MCP_MISSING: '[ERROR-MCP-MISSING]',
   /** Signal emitted when the agent cannot access the setup resource */
   ERROR_RESOURCE_MISSING: '[ERROR-RESOURCE-MISSING]',
+  /**
+   * Signal emitted when `install_skill` failed and the agent is continuing
+   * the integration WITHOUT the skill (best-effort from its own knowledge).
+   * Format: "[SKILL-INSTALL-FAILED] <skill id — reason>". Non-fatal by
+   * design: a freestyled integration beats an outright failure, but the run
+   * must be measurable — the runner captures it as an analytics event.
+   */
+  SKILL_INSTALL_FAILED: '[SKILL-INSTALL-FAILED]',
   /**
    * Signal emitted when the agent cannot complete the program and is
    * aborting intentionally (distinct from errors). Format: "[ABORT] <reason>".

src/lib/programs/posthog-integration/index.ts

@@ -188,6 +188,11 @@ STEP 1: Call load_skill_menu (from the wizard-tools MCP server) to see available
 
 STEP 2: Call install_skill (from the wizard-tools MCP server) with the chosen skill ID (e.g., "integration-nextjs-app-router").
    Do NOT run any shell commands to install skills.
+   If install_skill fails, emit on its own line: ${
+     AgentSignals.SKILL_INSTALL_FAILED
+   } <skill id — one-line reason>. Then CONTINUE and SKIP to STEP 5 the integration without the skill, following these steps and your knowledge of ${
+          config.metadata.name
+        } and PostHog's official docs, and note in the setup report that the skill could not be installed.
 
 STEP 3: Load the installed skill's SKILL.md file to understand what references are available.
 
@@ -200,7 +205,7 @@ STEP 5: Set up environment variables for PostHog using the wizard-tools MCP serv
    }, which you'll find in example code. The tool will also ensure .gitignore coverage. Don't assume the presence of keys means the value is up to date. Write the correct value each time.
    - Reference these environment variables in the code files you create instead of hardcoding the public token and host.
 
-Important: Use the detect_package_manager tool (from the wizard-tools MCP server) to determine which package manager the project uses, then run its install command to add the SDK. Do not manually search for lockfiles or config files. You must read a file immediately before attempting to write it, even if you have previously read it; failure to do so will cause a tool failure.
+Important: Use the detect_package_manager tool (from the wizard-tools MCP server) to determine which package manager the project uses, then run its install command to add the SDK. Do not manually search for lockfiles or config files. If a file already EXISTS, read it immediately before you edit or overwrite it — writing from a stale read causes a tool failure. Creating a brand-new file needs no prior read: never read a path that does not exist yet; just write it.
 
 
 `;