fix(switchboard): disable the pi-harness flag on the headless (cloud) path (#832)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: gewenyu99

src/lib/agent/runner/__tests__/switchboard.test.ts

@@ -1,4 +1,4 @@
-import { describe, it, expect } from 'vitest';
+import { describe, it, expect, vi } from 'vitest';
 import { PROGRAM_REGISTRY } from '@lib/programs/program-registry';
 import {
   DEFAULT_AGENT_MODEL,
@@ -23,6 +23,17 @@ import {
 } from '@lib/agent/runner/switchboard';
 import { modelCapabilities } from '@lib/agent/runner/switchboard/models';
 
+// RUN_SURFACE is read live in flagRunnerOverride; a getter lets a test flip it.
+const envState = vi.hoisted(() => ({
+  runSurface: 'local' as 'cloud' | 'local',
+}));
+vi.mock('@env', async (importOriginal) => ({
+  ...(await importOriginal<typeof import('@env')>()),
+  get RUN_SURFACE() {
+    return envState.runSurface;
+  },
+}));
+
 const PROGRAM_IDS = PROGRAM_REGISTRY.map((c) => c.id);
 
 describe('switchboard PROGRAM_BINDINGS', () => {
@@ -163,6 +174,30 @@ describe('switchboard resolveHarness — pi flag is gated to posthog-integration
     expect(pick).toEqual({ harness: Harness.pi, model: GPT5_4_MODEL });
   });
 
+  it('disables the pi flag on the cloud run surface, even for posthog-integration', () => {
+    envState.runSurface = 'cloud';
+    try {
+      const pick = resolveHarness({
+        program: 'posthog-integration',
+        flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+      });
+      expect(pick).toEqual({
+        harness: Harness.anthropic,
+        model: DEFAULT_AGENT_MODEL,
+      });
+    } finally {
+      envState.runSurface = 'local';
+    }
+  });
+
+  it('keeps the pi flag on the local run surface for posthog-integration', () => {
+    const pick = resolveHarness({
+      program: 'posthog-integration',
+      flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+    });
+    expect(pick.harness).toBe(Harness.pi);
+  });
+
   it('ignores the pi flag for self-driving — stays on the anthropic default', () => {
     const pick = resolveHarness({
       program: 'self-driving',
@@ -319,6 +354,20 @@ describe('switchboard wizard-pi-effort flag', () => {
       }).thinkingLevel,
     ).toBeUndefined();
   });
+
+  it('is inert on the cloud surface even with the pi flag on', () => {
+    envState.runSurface = 'cloud';
+    try {
+      expect(
+        modelCapabilities(GPT5_4_MODEL, {
+          [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true',
+          [WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
+        }).thinkingLevel,
+      ).toBe('low');
+    } finally {
+      envState.runSurface = 'local';
+    }
+  });
 });
 
 describe('switchboard resolveSequence — orchestrator stays flag-gated', () => {

src/lib/agent/runner/switchboard/harness.ts

@@ -2,7 +2,7 @@
  * Harness axis: registry, middleware, resolver. Mirrors `sequence.ts`.
  */
 
-import { IS_PRODUCTION_BUILD } from '@env';
+import { IS_PRODUCTION_BUILD, RUN_SURFACE } from '@env';
 import {
   DEFAULT_AGENT_MODEL,
   GPT5_4_MODEL,
@@ -60,6 +60,8 @@ const PI_FLAG_PROGRAMS = new Set<ProgramId>(['posthog-integration']);
 const flagRunnerOverride: Middleware<HarnessPick> = (ctx, next) => {
   const pick = next();
   if (ctx.flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return pick;
+  // The pi experiment is disabled on the cloud (headless) run surface.
+  if (RUN_SURFACE === 'cloud') return pick;
   if (!PI_FLAG_PROGRAMS.has(ctx.program)) return pick;
   if (ctx.trace) Object.assign(ctx.trace, { harness: 'flag', model: 'flag' });
   const variant = ctx.flags[WIZARD_PI_MODEL_FLAG_KEY] ?? '';

src/lib/agent/runner/switchboard/models.ts

@@ -20,6 +20,7 @@ import {
   WIZARD_PI_EFFORT_FLAG_KEY,
   WIZARD_USE_PI_HARNESS_FLAG_KEY,
 } from '@lib/constants';
+import { RUN_SURFACE } from '@env';
 
 /** Reasoning effort. pi maps it to `reasoning_effort` for openai-completions. */
 export type ThinkingLevel =
@@ -78,8 +79,12 @@ export function modelCapabilities(
   flags: Record<string, string> = {},
 ): ModelCapabilities {
   const caps = MODEL_CAPABILITIES[modelId] ?? defaultCaps(modelId);
-  // The wizard-pi-effort override applies only when the pi-harness flag selected the run; otherwise the model's own table effort stands.
-  if (flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return caps;
+  // The wizard-pi-effort override applies only to a pi run — inert on the cloud surface or without the pi flag.
+  if (
+    RUN_SURFACE === 'cloud' ||
+    flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true'
+  )
+    return caps;
   const effort = flags[WIZARD_PI_EFFORT_FLAG_KEY] as ThinkingLevel;
   if (caps.reasoning && EFFORT_FLAG_VARIANTS.includes(effort)) {
     return { ...caps, thinkingLevel: effort };