fix(switchboard): gate the pi harness flag to posthog-integration only (#824)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: gewenyu99

src/lib/agent/runner/__tests__/switchboard.test.ts

@@ -154,6 +154,52 @@ describe('switchboard resolveHarness — CLI precedence', () => {
   });
 });
 
+describe('switchboard resolveHarness — pi flag is gated to posthog-integration', () => {
+  it('honours the pi flag for posthog-integration', () => {
+    const pick = resolveHarness({
+      program: 'posthog-integration',
+      flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+    });
+    expect(pick).toEqual({ harness: Harness.pi, model: GPT5_4_MODEL });
+  });
+
+  it('ignores the pi flag for self-driving — stays on the anthropic default', () => {
+    const pick = resolveHarness({
+      program: 'self-driving',
+      flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+    });
+    expect(pick).toEqual({
+      harness: Harness.anthropic,
+      model: DEFAULT_AGENT_MODEL,
+    });
+  });
+
+  it('ignores the pi flag for every non-posthog-integration program', () => {
+    for (const program of PROGRAM_IDS) {
+      if (program === 'posthog-integration') continue;
+      expect(
+        resolveHarness({
+          program,
+          flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+        }).harness,
+      ).toBe(Harness.anthropic);
+    }
+  });
+
+  it('leaves the sequence unclamped for a gated program (pi never selected)', () => {
+    const ctx: SwitchboardCtx = {
+      program: 'self-driving',
+      flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' },
+    };
+    resolveBinding(ctx);
+    expect(ctx.trace).toEqual({
+      harness: 'binding',
+      model: 'binding',
+      sequence: 'binding',
+    });
+  });
+});
+
 describe('switchboard decision trace', () => {
   it('stamps binding sources when nothing overrides', () => {
     const ctx: SwitchboardCtx = { program: 'posthog-integration', flags: {} };
@@ -237,17 +283,38 @@ describe('switchboard modelCapabilities', () => {
 });
 
 describe('switchboard wizard-pi-effort flag', () => {
+  const PI_ON = { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' };
+
   it('overrides effort for reasoning models; ignores invalid; skips non-reasoning', () => {
     expect(
-      modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' })
-        .thinkingLevel,
+      modelCapabilities(GPT5_4_MODEL, {
+        ...PI_ON,
+        [WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
+      }).thinkingLevel,
     ).toBe('high');
     expect(
-      modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'banana' })
-        .thinkingLevel,
+      modelCapabilities(GPT5_4_MODEL, {
+        ...PI_ON,
+        [WIZARD_PI_EFFORT_FLAG_KEY]: 'banana',
+      }).thinkingLevel,
     ).toBe('low');
     expect(
       modelCapabilities('openai/gpt-4o', {
+        ...PI_ON,
+        [WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
+      }).thinkingLevel,
+    ).toBeUndefined();
+  });
+
+  it('is inert unless the pi-harness flag is on — effort cannot ride a non-pi pick', () => {
+    // Effort set but the pi flag off: the override is ignored, so the model's
+    // own table effort stands (gpt-5.4 → low) and anthropic keeps no effort.
+    expect(
+      modelCapabilities(GPT5_4_MODEL, { [WIZARD_PI_EFFORT_FLAG_KEY]: 'high' })
+        .thinkingLevel,
+    ).toBe('low');
+    expect(
+      modelCapabilities(DEFAULT_AGENT_MODEL, {
         [WIZARD_PI_EFFORT_FLAG_KEY]: 'high',
       }).thinkingLevel,
     ).toBeUndefined();

src/lib/agent/runner/switchboard/harness.ts

@@ -14,6 +14,7 @@ import {
   WIZARD_USE_PI_HARNESS_FLAG_KEY,
 } from '@lib/constants';
 import { logToFile } from '@utils/debug';
+import type { ProgramId } from '@lib/programs/program-registry';
 import { anthropicBackend } from '../harness/anthropic';
 import { piBackend } from '../harness/pi';
 import type { AgentHarness } from '../harness/types';
@@ -48,13 +49,18 @@ const PI_MODEL_FLAG_VARIANTS: Record<string, string> = {
   'sonnet-5': SONNET_5_MODEL,
 };
 
+/** Programs the wizard-use-pi-harness flag may switch to pi; off this set the flag is a no-op and the binding default stands. */
+const PI_FLAG_PROGRAMS = new Set<ProgramId>(['posthog-integration']);
+
 /**
  * `wizard-use-pi-harness` on → pi, paired with the `wizard-pi-model` variant
- * (unknown/missing variant → gpt-5.4). Otherwise binding default.
+ * (unknown/missing variant → gpt-5.4). Off `PI_FLAG_PROGRAMS`, the flag is
+ * ignored and the binding default stands.
  */
 const flagRunnerOverride: Middleware<HarnessPick> = (ctx, next) => {
   const pick = next();
   if (ctx.flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return pick;
+  if (!PI_FLAG_PROGRAMS.has(ctx.program)) return pick;
   if (ctx.trace) Object.assign(ctx.trace, { harness: 'flag', model: 'flag' });
   const variant = ctx.flags[WIZARD_PI_MODEL_FLAG_KEY] ?? '';
   return {

src/lib/agent/runner/switchboard/models.ts

@@ -18,6 +18,7 @@ import {
   GPT5_4_MODEL,
   GPT5_MINI_MODEL,
   WIZARD_PI_EFFORT_FLAG_KEY,
+  WIZARD_USE_PI_HARNESS_FLAG_KEY,
 } from '@lib/constants';
 
 /** Reasoning effort. pi maps it to `reasoning_effort` for openai-completions. */
@@ -77,8 +78,8 @@ export function modelCapabilities(
   flags: Record<string, string> = {},
 ): ModelCapabilities {
   const caps = MODEL_CAPABILITIES[modelId] ?? defaultCaps(modelId);
-  // `wizard-pi-effort` overrides the table for reasoning models only; an
-  // unknown variant is ignored.
+  // The wizard-pi-effort override applies only when the pi-harness flag selected the run; otherwise the model's own table effort stands.
+  if (flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return caps;
   const effort = flags[WIZARD_PI_EFFORT_FLAG_KEY] as ThinkingLevel;
   if (caps.reasoning && EFFORT_FLAG_VARIANTS.includes(effort)) {
     return { ...caps, thinkingLevel: effort };