feat: add audit-3000 wizard command (#452)

Co-authored-by: Jon <148268521+consultingsultan@users.noreply.github.com>
Co-authored-by: Edwin Lim <edwin@posthog.com>

Author: 3 people

src/lib/agent/agent-interface.ts

@@ -288,6 +288,8 @@ export type AgentConfig = {
   /** Feature flag key -> variant (evaluated at start of run). */
   wizardFlags?: Record<string, string>;
   wizardMetadata?: Record<string, string>;
+  /** Workflow identifier — selects the model for that workflow. */
+  integrationLabel?: string;
 };
 
 /**
@@ -663,10 +665,17 @@ export async function initializeAgent(
     });
     mcpServers['wizard-tools'] = wizardToolsServer;
 
+    // audit-3000 needs Opus 4.7's depth for the multi-phase audit chain;
+    // every other workflow runs on Sonnet 4.6.
+    const model =
+      config.integrationLabel === 'audit-3000'
+        ? 'anthropic/claude-opus-4-7'
+        : 'anthropic/claude-sonnet-4-6';
+
     const agentRunConfig: AgentRunConfig = {
       workingDirectory: config.workingDirectory,
       mcpServers,
-      model: 'anthropic/claude-sonnet-4-6',
+      model,
       wizardFlags: config.wizardFlags,
       wizardMetadata: config.wizardMetadata,
     };

src/lib/agent/agent-runner.ts

@@ -312,6 +312,7 @@ export async function runWorkflow(
       skillsBaseUrl,
       wizardFlags,
       wizardMetadata,
+      integrationLabel: config.integrationLabel,
     },
     sessionToOptions(session),
   );

src/lib/workflows/agent-skill/index.ts

@@ -57,6 +57,7 @@ export function createSkillWorkflow(
     description: opts.description,
     flowKey: opts.flowKey,
     steps: AGENT_SKILL_STEPS,
+    reportFile: opts.reportFile,
     run: {
       skillId: opts.skillId,
       integrationLabel: opts.integrationLabel,

src/lib/workflows/audit-3000/index.ts

@@ -0,0 +1,251 @@
+import fs from 'fs';
+import path from 'path';
+import {
+  AGENT_SKILL_STEPS,
+  createSkillWorkflow,
+} from '../agent-skill/index.js';
+import type { Workflow, WorkflowConfig } from '../workflow-step.js';
+import type { WorkflowRun } from '../../agent/agent-runner.js';
+import type { WizardSession } from '../../wizard-session.js';
+import { AUDIT_ABORT_CASES } from '../audit/detect.js';
+import {
+  AUDIT_CHECKS_FILE,
+  AUDIT_CHECKS_KEY,
+  type AuditCheck,
+} from '../audit/types.js';
+import { AUDIT_SEED_CHECKS } from '../audit/seed.js';
+import { logToFile } from '../../../utils/debug';
+
+const AUDIT3000_REPORT_FILE = 'posthog-audit-3000-report.md';
+
+// Extra checks the v3000 audit adds on top of the base 10. IDs must match
+// those referenced in the audit-3000 skill's step files (Event Quality,
+// stale feature-flag review, session replay [fix + optimize], per-product
+// use-case expansion, and phase markers for the post-flags chain).
+const AUDIT3000_EXTRA_CHECKS: AuditCheck[] = [
+  // ── Event Quality (Step 5) ──
+  {
+    id: 'event-naming-standardization',
+    area: 'Event Quality',
+    label: 'Event naming convention is consistent',
+    status: 'pending',
+  },
+  {
+    id: 'event-duplicates-and-bloat',
+    area: 'Event Quality',
+    label: 'No duplicate or bloated event capture',
+    status: 'pending',
+  },
+  {
+    id: 'event-quality-context-review',
+    area: 'Event Quality',
+    label: 'Event property context reviewed',
+    status: 'pending',
+  },
+  {
+    id: 'event-usage-coverage',
+    area: 'Event Quality',
+    label: 'Captured events match insights / dashboards usage',
+    status: 'pending',
+  },
+  // ── Feature Flags (Step 6) ──
+  {
+    id: 'stale-feature-flags-reviewed',
+    area: 'Feature Flags',
+    label: 'Stale feature flags reviewed',
+    status: 'pending',
+  },
+  // ── Session Replay — fix (Step 6b) ──
+  {
+    id: 'replay-minimum-duration-set',
+    area: 'Session Replay',
+    label: 'Minimum duration set on init',
+    status: 'pending',
+  },
+  {
+    id: 'replay-mask-config',
+    area: 'Session Replay',
+    label: 'Mask config covers sensitive surfaces',
+    status: 'pending',
+  },
+  {
+    id: 'replay-disabled-in-test-envs',
+    area: 'Session Replay',
+    label: 'Disabled in test / CI environments',
+    status: 'pending',
+  },
+  {
+    id: 'replay-strict-minimum-duration',
+    area: 'Session Replay',
+    label: 'Strict minimum duration enforced',
+    status: 'pending',
+  },
+  // ── Session Replay — optimize (Step 6b cost wave) ──
+  {
+    id: 'replay-sampling-rate',
+    area: 'Session Replay — Optimize',
+    label: 'Sampling rate tuned for cost',
+    status: 'pending',
+  },
+  {
+    id: 'replay-triggers-configured',
+    area: 'Session Replay — Optimize',
+    label: 'Triggers configured (event / URL / flag)',
+    status: 'pending',
+  },
+  {
+    id: 'replay-network-recording-filtered',
+    area: 'Session Replay — Optimize',
+    label: 'Network recording filtered',
+    status: 'pending',
+  },
+  {
+    id: 'replay-mobile-sampling',
+    area: 'Session Replay — Optimize',
+    label: 'Mobile sampling configured',
+    status: 'pending',
+  },
+  // ── Use Case: Expansion (Step 9) ──
+  {
+    id: 'expansion-product-analytics',
+    area: 'Use Case: Expansion',
+    label: 'Product analytics coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-error-tracking',
+    area: 'Use Case: Expansion',
+    label: 'Error tracking coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-llm-observability',
+    area: 'Use Case: Expansion',
+    label: 'LLM observability coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-session-replay',
+    area: 'Use Case: Expansion',
+    label: 'Session replay coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-feature-flags',
+    area: 'Use Case: Expansion',
+    label: 'Feature flags coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-surveys',
+    area: 'Use Case: Expansion',
+    label: 'Surveys coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-logs',
+    area: 'Use Case: Expansion',
+    label: 'Logs coverage',
+    status: 'pending',
+  },
+  {
+    id: 'expansion-web-analytics',
+    area: 'Use Case: Expansion',
+    label: 'Web analytics coverage',
+    status: 'pending',
+  },
+  // ── Additional Sections (Steps 7, 8, 10 phase markers) ──
+  // Tracked in the ledger so the UI can surface "did it run / was it
+  // skipped" alongside the regular checks. use-case-expansion is omitted
+  // because the eight `expansion-*` checks above cover that phase.
+  {
+    id: 'customer-enrichment',
+    area: 'Additional Sections',
+    label: 'Customer enrichment (Harmonic + PDL)',
+    status: 'pending',
+  },
+  {
+    id: 'use-case-match',
+    area: 'Additional Sections',
+    label: 'Use-case match',
+    status: 'pending',
+  },
+  {
+    id: 'final-report',
+    area: 'Additional Sections',
+    label: 'Final audit report written',
+    status: 'pending',
+  },
+];
+
+const AUDIT3000_SEED_CHECKS: AuditCheck[] = [
+  ...AUDIT_SEED_CHECKS,
+  ...AUDIT3000_EXTRA_CHECKS,
+];
+
+// Audit-3000 has its own arcade-flavoured intro / run / outro screens. The
+// shared audit screens stay reserved for the original `audit` workflow.
+const AUDIT3000_SCREEN_BY_STEP: Record<string, string> = {
+  intro: 'audit-3000-intro',
+  run: 'audit-3000-run',
+  outro: 'audit-3000-outro',
+};
+
+const seedAudit3000Ledger = (installDir: string): void => {
+  const target = path.join(installDir, AUDIT_CHECKS_FILE);
+  const tmp = `${target}.tmp`;
+  fs.writeFileSync(tmp, JSON.stringify(AUDIT3000_SEED_CHECKS, null, 2), 'utf8');
+  fs.renameSync(tmp, target);
+  logToFile(
+    `seedAudit3000Ledger: wrote ${AUDIT3000_SEED_CHECKS.length} entries to ${target}`,
+  );
+};
+
+const seedBeforeAudit3000Run = (session: WizardSession): void => {
+  seedAudit3000Ledger(session.installDir);
+  session.frameworkContext[AUDIT_CHECKS_KEY] = AUDIT3000_SEED_CHECKS;
+};
+
+const withAudit3000Screens = (steps: Workflow): Workflow =>
+  steps.map((step) => {
+    const override = AUDIT3000_SCREEN_BY_STEP[step.id];
+    return override ? { ...step, screen: override } : step;
+  });
+
+const audit3000Steps: Workflow = withAudit3000Screens(AGENT_SKILL_STEPS);
+
+const baseConfig = createSkillWorkflow({
+  skillId: 'audit-3000',
+  command: 'audit-3000',
+  flowKey: 'audit-3000',
+  description:
+    'Audit an existing PostHog integration (v3000 — adds event quality, stale-flag hygiene, customer enrichment, use-case match)',
+  integrationLabel: 'audit-3000',
+  customPrompt:
+    'Run the audit-3000 skill end-to-end. Follow the step chain starting at references/1-version.md. Do not modify any project files — only create the final audit report and (when enrichment is enabled) the enrichment report.',
+  successMessage: `Audit complete! View the report at ./${AUDIT3000_REPORT_FILE}`,
+  reportFile: AUDIT3000_REPORT_FILE,
+  docsUrl: 'https://posthog.com/docs/product-analytics/best-practices',
+  spinnerMessage: 'Running PostHog Audit 3000...',
+  estimatedDurationMinutes: 6,
+  requires: ['posthog-integration'],
+  abortCases: AUDIT_ABORT_CASES,
+});
+
+const audit3000Run = async (session: WizardSession): Promise<WorkflowRun> => {
+  seedBeforeAudit3000Run(session);
+
+  if (!baseConfig.run) {
+    throw new Error('audit-3000 workflow has no run configuration.');
+  }
+
+  return typeof baseConfig.run === 'function'
+    ? baseConfig.run(session)
+    : baseConfig.run;
+};
+
+export const audit3000Config: WorkflowConfig = {
+  ...baseConfig,
+  steps: audit3000Steps,
+  run: audit3000Run,
+};

src/lib/workflows/workflow-registry.ts

@@ -15,12 +15,14 @@ import type { WorkflowConfig } from './workflow-step.js';
 import { posthogIntegrationConfig } from './posthog-integration/index.js';
 import { revenueAnalyticsConfig } from './revenue-analytics/index.js';
 import { auditConfig } from './audit/index.js';
+import { audit3000Config } from './audit-3000/index.js';
 import { posthogDoctorConfig } from './posthog-doctor/index.js';
 
 export const WORKFLOW_REGISTRY: WorkflowConfig[] = [
   posthogIntegrationConfig,
   revenueAnalyticsConfig,
   auditConfig,
+  audit3000Config,
   posthogDoctorConfig,
 ];
 

src/lib/workflows/workflow-step.ts

@@ -123,6 +123,12 @@ export interface WorkflowConfig {
   run?: WorkflowRun | ((session: WizardSession) => Promise<WorkflowRun>);
   /** Prerequisites: other workflow flowKeys that must have run first */
   requires?: string[];
+  /**
+   * Path (relative to installDir) of the report file the workflow writes.
+   * Mirrors `run.reportFile` but lifted to the top level so UI screens can
+   * read it synchronously without resolving a deferred `run` function.
+   */
+  reportFile?: string;
 }
 
 /**

src/ui/tui/flows.ts

@@ -27,6 +27,9 @@ export enum Screen {
   AuditIntro = 'audit-intro',
   AuditRun = 'audit-run',
   AuditOutro = 'audit-outro',
+  Audit3000Intro = 'audit-3000-intro',
+  Audit3000Run = 'audit-3000-run',
+  Audit3000Outro = 'audit-3000-outro',
   HealthCheck = 'health-check',
   DoctorIntro = 'doctor-intro',
   DoctorReport = 'doctor-report',
@@ -46,6 +49,7 @@ export enum Flow {
   PostHogIntegration = 'posthog-integration',
   RevenueAnalyticsSetup = 'revenue-analytics-setup',
   Audit = 'audit',
+  Audit3000 = 'audit-3000',
   PosthogDoctor = 'posthog-doctor',
   AgentSkill = 'agent-skill',
   McpAdd = 'mcp-add',

src/ui/tui/screen-registry.tsx

@@ -24,6 +24,9 @@ import { AgentSkillIntroScreen } from './screens/AgentSkillIntroScreen.js';
 import { AuditIntroScreen } from './screens/audit/AuditIntroScreen.js';
 import { AuditRunScreen } from './screens/audit/AuditRunScreen.js';
 import { AuditOutroScreen } from './screens/audit/AuditOutroScreen.js';
+import { Audit3000IntroScreen } from './screens/audit-3000/Audit3000IntroScreen.js';
+import { Audit3000RunScreen } from './screens/audit-3000/Audit3000RunScreen.js';
+import { Audit3000OutroScreen } from './screens/audit-3000/Audit3000OutroScreen.js';
 import { SetupScreen } from './screens/SetupScreen.js';
 import { AuthScreen } from './screens/AuthScreen.js';
 import { RunScreen } from './screens/RunScreen.js';
@@ -63,6 +66,9 @@ export function createScreens(
     [Screen.AuditIntro]: <AuditIntroScreen store={store} />,
     [Screen.AuditRun]: <AuditRunScreen store={store} />,
     [Screen.AuditOutro]: <AuditOutroScreen store={store} />,
+    [Screen.Audit3000Intro]: <Audit3000IntroScreen store={store} />,
+    [Screen.Audit3000Run]: <Audit3000RunScreen store={store} />,
+    [Screen.Audit3000Outro]: <Audit3000OutroScreen store={store} />,
     [Screen.HealthCheck]: <HealthCheckScreen store={store} />,
     [Screen.DoctorIntro]: <DoctorIntroScreen store={store} />,
     [Screen.DoctorReport]: <DoctorReportScreen store={store} />,