diff --git a/src/lib/agent/runner/__tests__/switchboard.test.ts b/src/lib/agent/runner/__tests__/switchboard.test.ts index 5ec48b13..7a9f324b 100644 --- a/src/lib/agent/runner/__tests__/switchboard.test.ts +++ b/src/lib/agent/runner/__tests__/switchboard.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect } from 'vitest'; +import { describe, it, expect, vi } from 'vitest'; import { PROGRAM_REGISTRY } from '@lib/programs/program-registry'; import { DEFAULT_AGENT_MODEL, @@ -23,6 +23,17 @@ import { } from '@lib/agent/runner/switchboard'; import { modelCapabilities } from '@lib/agent/runner/switchboard/models'; +// RUN_SURFACE is read live in flagRunnerOverride; a getter lets a test flip it. +const envState = vi.hoisted(() => ({ + runSurface: 'local' as 'cloud' | 'local', +})); +vi.mock('@env', async (importOriginal) => ({ + ...(await importOriginal()), + get RUN_SURFACE() { + return envState.runSurface; + }, +})); + const PROGRAM_IDS = PROGRAM_REGISTRY.map((c) => c.id); describe('switchboard PROGRAM_BINDINGS', () => { @@ -163,6 +174,30 @@ describe('switchboard resolveHarness — pi flag is gated to posthog-integration expect(pick).toEqual({ harness: Harness.pi, model: GPT5_4_MODEL }); }); + it('disables the pi flag on the cloud run surface, even for posthog-integration', () => { + envState.runSurface = 'cloud'; + try { + const pick = resolveHarness({ + program: 'posthog-integration', + flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' }, + }); + expect(pick).toEqual({ + harness: Harness.anthropic, + model: DEFAULT_AGENT_MODEL, + }); + } finally { + envState.runSurface = 'local'; + } + }); + + it('keeps the pi flag on the local run surface for posthog-integration', () => { + const pick = resolveHarness({ + program: 'posthog-integration', + flags: { [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true' }, + }); + expect(pick.harness).toBe(Harness.pi); + }); + it('ignores the pi flag for self-driving — stays on the anthropic default', () => { const pick = resolveHarness({ program: 'self-driving', @@ -319,6 +354,20 @@ describe('switchboard wizard-pi-effort flag', () => { }).thinkingLevel, ).toBeUndefined(); }); + + it('is inert on the cloud surface even with the pi flag on', () => { + envState.runSurface = 'cloud'; + try { + expect( + modelCapabilities(GPT5_4_MODEL, { + [WIZARD_USE_PI_HARNESS_FLAG_KEY]: 'true', + [WIZARD_PI_EFFORT_FLAG_KEY]: 'high', + }).thinkingLevel, + ).toBe('low'); + } finally { + envState.runSurface = 'local'; + } + }); }); describe('switchboard resolveSequence — orchestrator stays flag-gated', () => { diff --git a/src/lib/agent/runner/switchboard/harness.ts b/src/lib/agent/runner/switchboard/harness.ts index 09eaa073..f9406153 100644 --- a/src/lib/agent/runner/switchboard/harness.ts +++ b/src/lib/agent/runner/switchboard/harness.ts @@ -2,7 +2,7 @@ * Harness axis: registry, middleware, resolver. Mirrors `sequence.ts`. */ -import { IS_PRODUCTION_BUILD } from '@env'; +import { IS_PRODUCTION_BUILD, RUN_SURFACE } from '@env'; import { DEFAULT_AGENT_MODEL, GPT5_4_MODEL, @@ -60,6 +60,8 @@ const PI_FLAG_PROGRAMS = new Set(['posthog-integration']); const flagRunnerOverride: Middleware = (ctx, next) => { const pick = next(); if (ctx.flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return pick; + // The pi experiment is disabled on the cloud (headless) run surface. + if (RUN_SURFACE === 'cloud') return pick; if (!PI_FLAG_PROGRAMS.has(ctx.program)) return pick; if (ctx.trace) Object.assign(ctx.trace, { harness: 'flag', model: 'flag' }); const variant = ctx.flags[WIZARD_PI_MODEL_FLAG_KEY] ?? ''; diff --git a/src/lib/agent/runner/switchboard/models.ts b/src/lib/agent/runner/switchboard/models.ts index cfd57838..5c97a84d 100644 --- a/src/lib/agent/runner/switchboard/models.ts +++ b/src/lib/agent/runner/switchboard/models.ts @@ -20,6 +20,7 @@ import { WIZARD_PI_EFFORT_FLAG_KEY, WIZARD_USE_PI_HARNESS_FLAG_KEY, } from '@lib/constants'; +import { RUN_SURFACE } from '@env'; /** Reasoning effort. pi maps it to `reasoning_effort` for openai-completions. */ export type ThinkingLevel = @@ -78,8 +79,12 @@ export function modelCapabilities( flags: Record = {}, ): ModelCapabilities { const caps = MODEL_CAPABILITIES[modelId] ?? defaultCaps(modelId); - // The wizard-pi-effort override applies only when the pi-harness flag selected the run; otherwise the model's own table effort stands. - if (flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true') return caps; + // The wizard-pi-effort override applies only to a pi run — inert on the cloud surface or without the pi flag. + if ( + RUN_SURFACE === 'cloud' || + flags[WIZARD_USE_PI_HARNESS_FLAG_KEY] !== 'true' + ) + return caps; const effort = flags[WIZARD_PI_EFFORT_FLAG_KEY] as ThinkingLevel; if (caps.reasoning && EFFORT_FLAG_VARIANTS.includes(effort)) { return { ...caps, thinkingLevel: effort };