feat(ai-observability): agent-driven AI Observability program (#922)

Co-authored-by: Vincent (Wen Yu) Ge <vincent@posthog.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

Author: 3 people

bin.ts

@@ -37,6 +37,7 @@ import { Wizard } from './src/wizard';
 import { basicIntegrationCommand } from './src/commands/basic-integration';
 import { mcpCommand } from './src/commands/mcp';
 import { mcpAnalyticsCommand } from './src/commands/mcp-analytics';
+import { aiObservabilityCommand } from './src/commands/ai-observability';
 import { auditCommand } from './src/commands/audit';
 import { doctorCommand } from './src/commands/doctor';
 import { migrateCommand } from './src/commands/migrate';
@@ -67,6 +68,7 @@ function resolveInstallDir(): string {
 Wizard.use(basicIntegrationCommand)
   .use(mcpCommand)
   .use(mcpAnalyticsCommand)
+  .use(aiObservabilityCommand)
   .use(cliCommand)
   .use(auditCommand)
   .use(doctorCommand)

e2e-harness/__tests__/__snapshots__/e2e-flow-snapshot.test.ts.snap

@@ -1,5 +1,45 @@
 // Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
 
+exports[`e2e flow snapshot — ai-observability > walks intro → health → auth → run → outro → skills 1`] = `
+{
+  "profile": {
+    "ask": "first",
+    "healthCheck": "dismiss",
+    "mcp": "skip",
+    "setup": "first",
+    "skills": "delete",
+    "slack": "skip",
+  },
+  "program": "ai-observability",
+  "trace": [
+    {
+      "action": "confirm_setup",
+      "screen": "ai-observability-intro",
+    },
+    {
+      "action": "dismiss_outage",
+      "screen": "health-check",
+    },
+    {
+      "action": "(external)",
+      "screen": "auth",
+    },
+    {
+      "action": "(external)",
+      "screen": "run",
+    },
+    {
+      "action": "dismiss_outro",
+      "screen": "outro",
+    },
+    {
+      "action": "keep_skills",
+      "screen": "keep-skills",
+    },
+  ],
+}
+`;
+
 exports[`e2e flow snapshot — posthog-integration > Next.js (with a setup question) walks a stable path 1`] = `
 {
   "profile": {

e2e-harness/__tests__/e2e-flow-snapshot.test.ts

@@ -34,16 +34,17 @@ import { profileFor } from '../profiles';
  */
 function traceFlow(
   integration: Integration,
+  program: (typeof Program)[keyof typeof Program] = Program.PostHogIntegration,
 ): Array<{ screen: string; action: string }> {
-  const store = new WizardStore(Program.PostHogIntegration);
+  const store = new WizardStore(program);
   setUI(new InkUI(store));
   const session = buildSession({ installDir: '/tmp/e2e-snap', ci: true });
   session.integration = integration;
   session.frameworkConfig = FRAMEWORK_REGISTRY[integration];
   store.session = session;
 
   const driver = new WizardCiDriver(store);
-  const profile = profileFor(Program.PostHogIntegration);
+  const profile = profileFor(program);
 
   const trace: Array<{ screen: string; action: string }> = [];
   for (let guard = 0; guard < 40; guard++) {
@@ -95,3 +96,13 @@ describe('e2e flow snapshot — posthog-integration', () => {
     }).toMatchSnapshot();
   });
 });
+
+describe('e2e flow snapshot — ai-observability', () => {
+  it('walks intro → health → auth → run → outro → skills', () => {
+    expect({
+      program: 'ai-observability',
+      profile: profileFor(Program.AiObservability),
+      trace: traceFlow(Integration.javascriptNode, Program.AiObservability),
+    }).toMatchSnapshot();
+  });
+});

e2e-harness/action-registry.ts

@@ -73,6 +73,9 @@ export const NO_ACTION_SCREENS: ReadonlySet<ScreenName> = new Set<ScreenName>([
   ScreenId.SourceMapsDetect,
   ScreenId.SourceMapsOutro,
   ScreenId.AuditOutro,
+  ScreenId.SelfDrivingIntegrationCheck,
+  ScreenId.SelfDrivingIntegrationDetect,
+  ScreenId.SelfDrivingHandoff,
   Overlay.ManagedSettings,
   Overlay.AuthError,
   Overlay.SessionTimeout,
@@ -96,6 +99,7 @@ export const ACTION_REGISTRY: Partial<Record<ScreenName, DriverAction[]>> = {
   [ScreenId.SourceMapsIntro]: [confirmSetupAction],
   [ScreenId.MigrationIntro]: [confirmSetupAction],
   [ScreenId.AgentSkillIntro]: [confirmSetupAction],
+  [ScreenId.AiObservabilityIntro]: [confirmSetupAction],
   [ScreenId.AuditIntro]: [confirmSetupAction],
   [ScreenId.DoctorIntro]: [confirmSetupAction],
   [ScreenId.WarehouseIntro]: [confirmSetupAction],

e2e-harness/e2e-profile.ts

@@ -91,6 +91,7 @@ export function decideE2eAction(
     case ScreenId.RevenueIntro:
     case ScreenId.MigrationIntro:
     case ScreenId.AgentSkillIntro:
+    case ScreenId.AiObservabilityIntro:
     case ScreenId.AuditIntro:
     case ScreenId.SourceMapsIntro:
     case ScreenId.DoctorIntro:

e2e-harness/profiles.ts

@@ -16,15 +16,19 @@ import {
   type WizardE2eVariation,
 } from './e2e-profile.js';
 import posthogIntegrationE2e from '@lib/programs/posthog-integration/test/e2e.json';
+import aiObservabilityE2e from '@lib/programs/ai-observability/test/e2e.json';
 
 const PROFILES: Partial<Record<ProgramId, WizardE2eProfile>> = {
   [Program.PostHogIntegration]:
     posthogIntegrationE2e.profile as WizardE2eProfile,
+  [Program.AiObservability]: aiObservabilityE2e.profile as WizardE2eProfile,
 };
 
 const VARIATIONS: Partial<Record<ProgramId, WizardE2eVariation[]>> = {
   [Program.PostHogIntegration]:
     posthogIntegrationE2e.variations as WizardE2eVariation[],
+  [Program.AiObservability]:
+    aiObservabilityE2e.variations as WizardE2eVariation[],
 };
 
 /** The e2e profile for a program, or the happy-path default if none is set. */

scripts/tui-host.no-jest.ts

@@ -17,10 +17,13 @@ import fs from 'fs';
 import net from 'net';
 import { startTUI } from '@ui/tui/start-tui';
 import { VERSION } from '@lib/version';
-import { Program } from '@lib/programs/program-registry';
+import {
+  Program,
+  getProgramConfig,
+  type ProgramId,
+} from '@lib/programs/program-registry';
 import type { Harness, Sequence } from '@lib/constants';
 import { buildSession } from '@lib/wizard-session';
-import { posthogIntegrationConfig } from '@lib/programs/posthog-integration';
 import { runAgent } from '@lib/agent/agent-runner';
 import { getOrAskForProjectData } from '@utils/setup-utils';
 import { logToFile } from '@utils/debug';
@@ -43,13 +46,22 @@ async function main() {
   ).trim();
   const projectId = process.env.PROJECT_ID!;
 
-  const { store } = startTUI(VERSION, Program.PostHogIntegration);
+  // Which program to drive — PROGRAM env from the workbench e2e runner;
+  // defaults to the integration flow. getProgramConfig throws on unknown ids.
+  const programId = (process.env.PROGRAM ||
+    Program.PostHogIntegration) as ProgramId;
+  const programConfig = getProgramConfig(programId);
+
+  const { store } = startTUI(VERSION, programId);
   store.session = buildSession({
     installDir: process.env.APP_DIR!,
     ci: true,
     apiKey,
     projectId,
     region: 'us',
+    // Local skills + MCP (context-mill dev server on :8765, MCP on :8787) —
+    // same env-backed flag the bin's --local-mcp reads.
+    localMcp: process.env.POSTHOG_WIZARD_LOCAL_MCP === 'true',
     // Switchboard variation overrides (see e2e.json `variations`), threaded by
     // the snapshot driver as one run per variation. Empty ⇒ resolved default.
     harness: (process.env.SNAP_HARNESS || undefined) as Harness | undefined,
@@ -66,7 +78,7 @@ async function main() {
       ci: true,
       apiKey,
       projectId: Number(projectId),
-      programId: Program.PostHogIntegration,
+      programId,
     });
     store.setCredentials({
       accessToken: d.accessToken,
@@ -76,12 +88,12 @@ async function main() {
     });
   };
 
-  // Pass the intro and health-check gates and run the real integration agent.
+  // Pass the intro and health-check gates and run the program's real agent.
   // The auth and run screens never advance on their own; this is what moves them.
   const runIntegration = async () => {
     await store.getGate('intro');
     await store.getGate('health-check');
-    await runAgent(posthogIntegrationConfig, store.session);
+    await runAgent(programConfig, store.session);
   };
 
   if (process.env.MODE === 'serve') return serve();
@@ -166,7 +178,7 @@ async function main() {
   // ---- CI route: self-drive the fixed profile, snapshot each screen ----
   async function fixed() {
     const CTRL = process.env.SNAP_CTRL!;
-    const profile: WizardE2eProfile = profileFor(Program.PostHogIntegration);
+    const profile: WizardE2eProfile = profileFor(programId);
     const screenPath: string[] = [];
     // Snapshot on key moments — a screen change, a task-list update, or a
     // runPhase change — so the run screen's progression (the agent working) is

src/commands/ai-observability.ts

@@ -0,0 +1,19 @@
+import { aiObservabilityConfig } from '@lib/programs/ai-observability/index';
+
+import type { Command } from './command';
+import { nativeCommandFactory } from './factories/native-command-factory';
+
+/**
+ * `wizard ai-observability` — flat skill command, wire AI Observability into a
+ * project today.
+ *
+ * Installs the OpenTelemetry SDK, the PostHog span processor, and the
+ * provider-specific instrumentation so LLM calls emit `$ai_generation` events.
+ * The `ai-observability` context-mill skill has one variant per (LLM provider ×
+ * language); the agent picks the right one at run time by scanning the
+ * project's manifest and (when ambiguous) asking the user via `wizard_ask`.
+ * Stays flat while a single "add AIO to a project" flow is the only action.
+ */
+export const aiObservabilityCommand: Command = nativeCommandFactory(
+  aiObservabilityConfig,
+);

src/lib/agent/runner/__tests__/switchboard.test.ts

@@ -20,6 +20,7 @@ import {
   GPT5_6_LUNA_MODEL,
   GPT5_6_SOL_MODEL,
   GPT5_6_TERRA_MODEL,
+  SONNET_5_MODEL,
   Harness,
   Sequence,
   WIZARD_ORCHESTRATOR_FLAG_KEY,
@@ -60,11 +61,23 @@ describe('switchboard PROGRAM_BINDINGS', () => {
   // Pins today's behavior: the seam changes nothing until a binding is moved.
   it('resolves every program, unflagged, to the same default binding', () => {
     for (const program of PROGRAM_IDS) {
+      if (program === 'ai-observability') continue; // pinned below
       expect(resolveBinding({ program, flags: {} })).toEqual(DEFAULT_RESOLVED);
     }
   });
 
   runBindingCases([
+    {
+      name: 'binds ai-observability to anthropic + sonnet 5',
+      ctx: { program: 'ai-observability', flags: {} },
+      binding: {
+        sequence: Sequence.linear,
+        harness: Harness.anthropic,
+        model: SONNET_5_MODEL,
+        thinkingLevel: undefined,
+      },
+      trace: { harness: 'binding', model: 'binding', sequence: 'binding' },
+    },
     {
       name: 'falls back to DEFAULT_BINDING for an unmapped program',
       ctx: { program: 'not-a-program', flags: {} },
@@ -187,8 +200,13 @@ describe('switchboard composed clamp', () => {
         trace: {},
       };
       // Only the orchestrator flag is on → harness/model stay at the binding
-      // default, and the composed clamp holds the sequence at linear.
-      expect(resolveBinding(ctx)).toEqual(DEFAULT_RESOLVED);
+      // (sonnet 5 for ai-observability, the default elsewhere), and the
+      // composed clamp holds the sequence at linear.
+      expect(resolveBinding(ctx)).toEqual(
+        program === 'ai-observability'
+          ? { ...DEFAULT_RESOLVED, model: SONNET_5_MODEL }
+          : DEFAULT_RESOLVED,
+      );
       expect(ctx.trace?.sequence).toBe('composed');
     }
   });

src/lib/agent/runner/switchboard/index.ts

@@ -7,7 +7,12 @@
  * Model ids are gateway strings — add new ones as constants in `@lib/constants`.
  */
 
-import { DEFAULT_AGENT_MODEL, Harness, Sequence } from '@lib/constants';
+import {
+  DEFAULT_AGENT_MODEL,
+  SONNET_5_MODEL,
+  Harness,
+  Sequence,
+} from '@lib/constants';
 import type { ProgramId } from '@lib/programs/program-registry';
 import { resolveHarness } from './harness';
 import type { EffortLevel } from './models';
@@ -130,6 +135,11 @@ export const PROGRAM_BINDINGS: Partial<Record<ProgramId, ProgramBinding>> = {
   'mcp-remove': DEFAULT_BINDING,
   'mcp-tutorial': DEFAULT_BINDING,
   'mcp-analytics': DEFAULT_BINDING,
+  'ai-observability': {
+    sequence: Sequence.linear,
+    harness: Harness.anthropic,
+    model: SONNET_5_MODEL,
+  },
   slack: DEFAULT_BINDING,
 };