diff --git a/bin.ts b/bin.ts index cd1c45f6..85d82037 100644 --- a/bin.ts +++ b/bin.ts @@ -428,12 +428,27 @@ const cli = yargs(hideBin(process.argv)) .help(); }); +// Audit-only options. `--areas` constrains the run to a subset of audit +// areas (Installation, Identification, Web Analytics, …). When omitted +// the agent runs everything its discovery determines applies. +const auditSubcommandOptions = { + areas: { + describe: + 'Comma-separated audit areas to constrain the run to (e.g. "Web Analytics, Feature Flags"). See --help for the full list.', + type: 'string' as const, + }, +}; + // ── Skill-based workflow subcommands (derived from registry) ───────── for (const wfConfig of getSubcommandWorkflows()) { + const isAudit = wfConfig.command === 'audit'; cli.command( wfConfig.command!, wfConfig.description, - (y) => y.options(skillSubcommandOptions), + (y) => + isAudit + ? y.options({ ...skillSubcommandOptions, ...auditSubcommandOptions }) + : y.options(skillSubcommandOptions), (argv) => { const options = { ...argv }; if (options.ci) { @@ -478,6 +493,8 @@ function runWizard( const tui = startTUI(WIZARD_VERSION, config.flowKey as any); + const auditAreas = await maybeParseAuditAreas(config, options); + const session = buildSession({ debug: options.debug as boolean | undefined, forceInstall: options.forceInstall as boolean | undefined, @@ -492,6 +509,7 @@ function runWizard( integration: options.integration as any, benchmark: options.benchmark as boolean | undefined, yaraReport: options.yaraReport as boolean | undefined, + auditAreas, }); session.workflowLabel = config.flowKey; if (options.skillId) { @@ -618,6 +636,8 @@ function runWizardCI( ? (options.installDir as string) : path.join(process.cwd(), options.installDir as string); + const auditAreas = await maybeParseAuditAreas(config, options); + const session = buildSession({ debug: options.debug as boolean | undefined, forceInstall: options.forceInstall as boolean | undefined, @@ -632,6 +652,7 @@ function runWizardCI( projectId: options.projectId as string | undefined, benchmark: options.benchmark as boolean | undefined, yaraReport: options.yaraReport as boolean | undefined, + auditAreas, ...env, }); session.workflowLabel = config.flowKey; @@ -704,3 +725,30 @@ function runWizardCI( process.exit(1); }); } + +/** + * Parse `--areas` into a typed AuditArea[]. Returns `undefined` when the + * workflow isn't audit or no flag was passed. Logs a hint listing every + * allowed area when unknown values are passed (and ignores them). + */ +async function maybeParseAuditAreas( + config: WorkflowConfig, + options: Record, +): Promise { + if (config.command !== 'audit') return undefined; + const areasArg = options.areas as string | undefined; + if (!areasArg) return undefined; + + const { parseAuditAreas, formatAreasHint } = await import( + './src/lib/workflows/audit/areas.js' + ); + const { areas, unknown } = parseAuditAreas(areasArg); + if (unknown.length > 0) { + getUI().log.warn( + `Ignoring unknown audit area(s): ${unknown.join( + ', ', + )}. Allowed areas: ${formatAreasHint()}.`, + ); + } + return areas.length > 0 ? areas : undefined; +} diff --git a/src/lib/__tests__/wizard-tools.test.ts b/src/lib/__tests__/wizard-tools.test.ts index a725033e..e920dea6 100644 --- a/src/lib/__tests__/wizard-tools.test.ts +++ b/src/lib/__tests__/wizard-tools.test.ts @@ -298,4 +298,8 @@ describe('WIZARD_TOOL_NAMES', () => { it('exposes audit_add_checks so future workflows can append checks through the MCP server', () => { expect(WIZARD_TOOL_NAMES).toContain('wizard-tools:audit_add_checks'); }); + + it('exposes audit_get_areas so the agent can fetch the wizard-supplied area constraint at runtime', () => { + expect(WIZARD_TOOL_NAMES).toContain('wizard-tools:audit_get_areas'); + }); }); diff --git a/src/lib/agent/agent-interface.ts b/src/lib/agent/agent-interface.ts index 26b76c79..24aeec66 100644 --- a/src/lib/agent/agent-interface.ts +++ b/src/lib/agent/agent-interface.ts @@ -288,6 +288,11 @@ export type AgentConfig = { /** Feature flag key -> variant (evaluated at start of run). */ wizardFlags?: Record; wizardMetadata?: Record; + /** + * Audit-only: areas the wizard is constraining the run to. Surfaced + * to the agent via the `audit_get_areas` MCP tool. + */ + auditAreas?: ReadonlyArray; }; /** @@ -657,6 +662,7 @@ export async function initializeAgent( workingDirectory: config.workingDirectory, detectPackageManager: config.detectPackageManager, skillsBaseUrl: config.skillsBaseUrl, + auditAreas: config.auditAreas, }); mcpServers['wizard-tools'] = wizardToolsServer; diff --git a/src/lib/agent/agent-runner.ts b/src/lib/agent/agent-runner.ts index af843abf..099b4786 100644 --- a/src/lib/agent/agent-runner.ts +++ b/src/lib/agent/agent-runner.ts @@ -312,6 +312,7 @@ export async function runWorkflow( skillsBaseUrl, wizardFlags, wizardMetadata, + auditAreas: session.auditAreas, }, sessionToOptions(session), ); diff --git a/src/lib/wizard-session.ts b/src/lib/wizard-session.ts index 9544e9b0..efcf6c41 100644 --- a/src/lib/wizard-session.ts +++ b/src/lib/wizard-session.ts @@ -14,6 +14,7 @@ import type { Integration } from './constants'; import type { FrameworkConfig } from './framework-config'; import type { WizardReadinessResult } from './health-checks/readiness'; import type { SettingsConflict } from './agent/agent-interface'; +import type { AuditArea } from './workflows/audit/areas'; export interface Credentials { accessToken: string; @@ -168,6 +169,13 @@ export interface WizardSession { // Resolved framework config (set after integration is known) frameworkConfig: FrameworkConfig | null; + + /** + * Audit-only: area-level constraint from `--areas`. Empty / undefined + * means the run is unconstrained. Surfaced to the agent via the + * `audit_get_areas` MCP tool. + */ + auditAreas?: AuditArea[]; } /** @@ -189,6 +197,7 @@ export function buildSession(args: { benchmark?: boolean; yaraReport?: boolean; projectId?: string; + auditAreas?: AuditArea[]; }): WizardSession { return { debug: args.debug ?? false, @@ -234,5 +243,6 @@ export function buildSession(args: { workflowLabel: null, skillId: null, frameworkConfig: null, + auditAreas: args.auditAreas, }; } diff --git a/src/lib/wizard-tools.ts b/src/lib/wizard-tools.ts index 6ff1fc56..0f37f832 100644 --- a/src/lib/wizard-tools.ts +++ b/src/lib/wizard-tools.ts @@ -22,6 +22,7 @@ import { type AuditCheck, type AuditStatus, } from './workflows/audit/types'; +import { ALL_AUDIT_AREAS, type AuditArea } from './workflows/audit/areas'; // --------------------------------------------------------------------------- // SDK dynamic import (ESM module loaded once, cached) @@ -175,6 +176,13 @@ export interface WizardToolsOptions { /** Base URL for the skills server (e.g. http://localhost:8765 or GitHub releases URL) */ skillsBaseUrl: string; + + /** + * Audit-only: subset of audit areas the run is constrained to. + * Surfaced via the `audit_get_areas` MCP tool. Omit / pass `[]` for + * an unconstrained run. + */ + auditAreas?: ReadonlyArray; } // --------------------------------------------------------------------------- @@ -432,6 +440,7 @@ const SERVER_NAME = 'wizard-tools'; */ export async function createWizardToolsServer(options: WizardToolsOptions) { const { workingDirectory, detectPackageManager, skillsBaseUrl } = options; + const auditAreas: ReadonlyArray = options.auditAreas ?? []; const sdk = await getSDKModule(); const { tool, createSdkMcpServer } = sdk; @@ -811,6 +820,35 @@ export async function createWizardToolsServer(options: WizardToolsOptions) { }, ); + // -- audit_get_areas ------------------------------------------------------ + + const auditGetAreas = tool( + 'audit_get_areas', + `Return the audit areas the wizard is constraining this run to. An empty array means the run is unconstrained — the agent should run every area its discovery determines applies. When non-empty, the agent must skip any area not in the list. Allowed values (canonical capitalization): ${ALL_AUDIT_AREAS.join( + ', ', + )}.`, + {}, + () => { + logToFile( + `audit_get_areas: returning ${auditAreas.length} area(s)${ + auditAreas.length > 0 ? ` [${auditAreas.join(', ')}]` : '' + }`, + ); + return Promise.resolve({ + content: [ + { + type: 'text' as const, + text: JSON.stringify({ + areas: [...auditAreas], + allowed: [...ALL_AUDIT_AREAS], + constrained: auditAreas.length > 0, + }), + }, + ], + }); + }, + ); + // -- Assemble server ------------------------------------------------------ return createSdkMcpServer({ @@ -825,6 +863,7 @@ export async function createWizardToolsServer(options: WizardToolsOptions) { auditSeedChecks, auditAddChecks, auditResolveChecks, + auditGetAreas, ], }); } @@ -839,6 +878,7 @@ export const WIZARD_TOOL_NAMES = [ `${SERVER_NAME}:audit_seed_checks`, `${SERVER_NAME}:audit_add_checks`, `${SERVER_NAME}:audit_resolve_checks`, + `${SERVER_NAME}:audit_get_areas`, ]; // --------------------------------------------------------------------------- diff --git a/src/lib/workflows/audit/__tests__/selection.test.ts b/src/lib/workflows/audit/__tests__/selection.test.ts new file mode 100644 index 00000000..0dc87a93 --- /dev/null +++ b/src/lib/workflows/audit/__tests__/selection.test.ts @@ -0,0 +1,67 @@ +import { AUDIT_SEED_CHECKS, AUDIT_CORE_CHECKS } from '../seed'; +import { AUDIT_SPECIALISTS } from '../specialists'; +import { parseAuditAreas, normalizeAuditArea, ALL_AUDIT_AREAS } from '../areas'; + +describe('audit basic specialist registry', () => { + it('exposes identification + event-capture as the basic specialists', () => { + expect(AUDIT_SPECIALISTS.map((s) => s.area)).toEqual([ + 'Identification', + 'Event Capture', + ]); + }); +}); + +describe('audit seed', () => { + it('seeds 3 core + 4 identification + 3 event-capture (10 entries) unconditionally', () => { + expect(AUDIT_SEED_CHECKS).toHaveLength(10); + expect(AUDIT_SEED_CHECKS.slice(0, 3)).toEqual(AUDIT_CORE_CHECKS); + expect( + AUDIT_SEED_CHECKS.filter((c) => c.area === 'Identification'), + ).toHaveLength(4); + expect( + AUDIT_SEED_CHECKS.filter((c) => c.area === 'Event Capture'), + ).toHaveLength(3); + }); + + it('every seeded check starts as pending', () => { + for (const check of AUDIT_SEED_CHECKS) { + expect(check.status).toBe('pending'); + } + }); +}); + +describe('audit areas', () => { + it('exposes the canonical 8-area enum', () => { + expect(ALL_AUDIT_AREAS).toEqual([ + 'Installation', + 'Identification', + 'Event Capture', + 'Web Analytics', + 'Feature Flags', + 'Experiments', + 'LLM Analytics', + 'Error Tracking', + ]); + }); + + it('normalizes case-insensitively to canonical capitalization', () => { + expect(normalizeAuditArea('web analytics')).toBe('Web Analytics'); + expect(normalizeAuditArea('LLM ANALYTICS')).toBe('LLM Analytics'); + expect(normalizeAuditArea(' feature flags ')).toBe('Feature Flags'); + expect(normalizeAuditArea('not-a-real-area')).toBeNull(); + }); + + it('parses --areas, dedupes, and surfaces unknown values for hinting', () => { + const { areas, unknown } = parseAuditAreas( + 'Web Analytics, llm analytics, web analytics, bogus, feature flags', + ); + expect(areas).toEqual(['Web Analytics', 'LLM Analytics', 'Feature Flags']); + expect(unknown).toEqual(['bogus']); + }); + + it('returns empty when no input is provided', () => { + const { areas, unknown } = parseAuditAreas(undefined); + expect(areas).toEqual([]); + expect(unknown).toEqual([]); + }); +}); diff --git a/src/lib/workflows/audit/areas.ts b/src/lib/workflows/audit/areas.ts new file mode 100644 index 00000000..4d7b4426 --- /dev/null +++ b/src/lib/workflows/audit/areas.ts @@ -0,0 +1,73 @@ +/** + * Audit areas — the canonical, wizard-side enum of areas the audit runner + * may produce. + * + * The wizard exposes the user-supplied subset to the agent at runtime via + * the `audit_get_areas` MCP tool, so the agent (and its discovery subagent) + * can constrain dispatch to those areas only. An empty list means no + * constraint — the agent runs everything. + * + * Adding a new area: append it here AND add a row to context-mill's + * audit description.md "Discoverable specialists" table whose specialist + * produces findings under that area. + */ + +export const AUDIT_AREAS = [ + 'Installation', + 'Identification', + 'Event Capture', + 'Web Analytics', + 'Feature Flags', + 'Experiments', + 'LLM Analytics', + 'Error Tracking', +] as const; + +export type AuditArea = (typeof AUDIT_AREAS)[number]; + +export const ALL_AUDIT_AREAS: ReadonlyArray = AUDIT_AREAS; + +/** Case-insensitive lookup. Returns the canonical capitalization on hit. */ +export function normalizeAuditArea(value: string): AuditArea | null { + const folded = value.trim().toLowerCase(); + return AUDIT_AREAS.find((a) => a.toLowerCase() === folded) ?? null; +} + +export function isAuditArea(value: string): value is AuditArea { + return AUDIT_AREAS.includes(value as AuditArea); +} + +export interface ParsedAreas { + areas: AuditArea[]; + unknown: string[]; +} + +/** + * Parse a comma-separated `--areas` value. Tokens are matched + * case-insensitively against the canonical enum. Duplicates are dropped. + * Unknown tokens are returned separately so callers can surface a hint + * to the user. + */ +export function parseAuditAreas(input: string | undefined): ParsedAreas { + if (!input) return { areas: [], unknown: [] }; + const tokens = input + .split(',') + .map((s) => s.trim()) + .filter(Boolean); + const areas: AuditArea[] = []; + const unknown: string[] = []; + for (const token of tokens) { + const canonical = normalizeAuditArea(token); + if (canonical) { + if (!areas.includes(canonical)) areas.push(canonical); + } else { + unknown.push(token); + } + } + return { areas, unknown }; +} + +/** Human-readable hint listing every allowed area, comma-separated. */ +export function formatAreasHint(): string { + return AUDIT_AREAS.join(', '); +} diff --git a/src/lib/workflows/audit/index.ts b/src/lib/workflows/audit/index.ts index 6fbabc04..9717f8eb 100644 --- a/src/lib/workflows/audit/index.ts +++ b/src/lib/workflows/audit/index.ts @@ -7,7 +7,7 @@ import type { WorkflowRun } from '../../agent/agent-runner.js'; import type { WizardSession } from '../../wizard-session.js'; import { AUDIT_ABORT_CASES } from './detect.js'; import { AUDIT_CHECKS_KEY, AUDIT_REPORT_FILE } from './types.js'; -import { AUDIT_SEED_CHECKS, seedAuditLedger } from './seed.js'; +import { seedAuditLedger } from './seed.js'; /** Audit-specific screens for the shared agent-skill pipeline. */ const AUDIT_SCREEN_BY_STEP: Record = { @@ -16,9 +16,14 @@ const AUDIT_SCREEN_BY_STEP: Record = { outro: 'audit-outro', }; +const BASE_AUDIT_PROMPT = + 'Run a comprehensive audit of the existing PostHog integration. ' + + 'Follow the skill workflow steps in order. ' + + 'Do not modify any project files — only create the final audit report.'; + const seedBeforeAuditRun = (session: WizardSession): void => { - seedAuditLedger(session.installDir); - session.frameworkContext[AUDIT_CHECKS_KEY] = AUDIT_SEED_CHECKS; + const checks = seedAuditLedger(session.installDir); + session.frameworkContext[AUDIT_CHECKS_KEY] = checks; }; const withAuditScreens = (steps: Workflow): Workflow => @@ -36,8 +41,7 @@ const baseConfig = createSkillWorkflow({ description: 'Audit an existing PostHog integration for correctness and best practices', integrationLabel: 'audit', - customPrompt: - 'Run a comprehensive audit of the existing PostHog integration. Follow the skill workflow steps in order. Do not modify any project files — only create the final audit report.', + customPrompt: BASE_AUDIT_PROMPT, successMessage: 'Audit complete! You can view the audit report at ./posthog-audit-report.md', reportFile: AUDIT_REPORT_FILE, diff --git a/src/lib/workflows/audit/seed.ts b/src/lib/workflows/audit/seed.ts index 581995b5..9046e56a 100644 --- a/src/lib/workflows/audit/seed.ts +++ b/src/lib/workflows/audit/seed.ts @@ -2,9 +2,10 @@ import fs from 'fs'; import path from 'path'; import { logToFile } from '../../../utils/debug'; import { AUDIT_CHECKS_FILE, type AuditCheck } from './types.js'; +import { AUDIT_SPECIALISTS } from './specialists.js'; -/** The 10 data-integrity checks the audit runs. */ -export const AUDIT_SEED_CHECKS: AuditCheck[] = [ +/** Core install/init checks the runner executes inline (no specialist). */ +export const AUDIT_CORE_CHECKS: AuditCheck[] = [ { id: 'sdk-installed', area: 'Installation', @@ -23,52 +24,26 @@ export const AUDIT_SEED_CHECKS: AuditCheck[] = [ label: 'Initialization is correct', status: 'pending', }, - { - id: 'identify-stable-distinct-id', - area: 'Identification', - label: 'Stable distinct_id (not session UUID)', - status: 'pending', - }, - { - id: 'identify-not-late', - area: 'Identification', - label: 'identify() called before captures / flag evals', - status: 'pending', - }, - { - id: 'cross-runtime-distinct-id', - area: 'Identification', - label: 'Same distinct_id across client and server', - status: 'pending', - }, - { - id: 'identify-reset-on-logout', - area: 'Identification', - label: 'reset() called on logout / account switch', - status: 'pending', - }, - { - id: 'capture-event-names-static', - area: 'Event Capture', - label: 'Event names are static and consistent', - status: 'pending', - }, - { - id: 'capture-uses-proxy', - area: 'Event Capture', - label: 'Captures route through a reverse proxy', - status: 'pending', - }, - { - id: 'capture-growth-events', - area: 'Event Capture', - label: 'Key activation events captured', - status: 'pending', - }, +]; + +/** + * The 10 always-pre-seeded checks: 3 core install/init + every basic + * specialist's checks (identification, event-capture). Discoverable + * specialists' checks are enrolled mid-run by the runner via + * `audit_add_checks` and never appear here. + */ +export const AUDIT_SEED_CHECKS: AuditCheck[] = [ + ...AUDIT_CORE_CHECKS, + ...AUDIT_SPECIALISTS.flatMap((specialist) => + specialist.checks.map((check) => ({ + ...check, + status: 'pending' as const, + })), + ), ]; /** Atomically write the seeded ledger to the project's audit checks file. */ -export function seedAuditLedger(installDir: string): void { +export function seedAuditLedger(installDir: string): AuditCheck[] { const target = path.join(installDir, AUDIT_CHECKS_FILE); const tmp = `${target}.tmp`; fs.writeFileSync(tmp, JSON.stringify(AUDIT_SEED_CHECKS, null, 2), 'utf8'); @@ -76,4 +51,5 @@ export function seedAuditLedger(installDir: string): void { logToFile( `seedAuditLedger: wrote ${AUDIT_SEED_CHECKS.length} entries to ${target}`, ); + return AUDIT_SEED_CHECKS; } diff --git a/src/lib/workflows/audit/specialists.ts b/src/lib/workflows/audit/specialists.ts new file mode 100644 index 00000000..23935410 --- /dev/null +++ b/src/lib/workflows/audit/specialists.ts @@ -0,0 +1,72 @@ +/** + * Audit basic specialists — the always-on subagents whose checks the + * wizard pre-seeds in the ledger. The runner dispatches these + * unconditionally after install/init. + * + * Discoverable specialists (web-analytics, feature-flags, experiments, + * llm-analytics, error-tracking) are owned by context-mill's audit + * SKILL.md, not the wizard. The runner's discovery / dispatch agent + * decides which to run and enrolls their checks via the + * `audit_add_checks` MCP tool. The wizard never pre-seeds them. + */ + +import type { AuditCheck } from './types.js'; + +export interface AuditSpecialist { + /** Context-mill skill ID the runner installs before dispatch. */ + skillId: string; + /** Area label used in the report and ledger. */ + area: string; + /** Pre-seeded checks owned by this specialist. */ + checks: ReadonlyArray>; +} + +export const AUDIT_SPECIALISTS: ReadonlyArray = [ + { + skillId: 'audit-subagents-identification', + area: 'Identification', + checks: [ + { + id: 'identify-stable-distinct-id', + area: 'Identification', + label: 'Stable distinct_id (not session UUID)', + }, + { + id: 'identify-not-late', + area: 'Identification', + label: 'identify() called before captures / flag evals', + }, + { + id: 'cross-runtime-distinct-id', + area: 'Identification', + label: 'Same distinct_id across client and server', + }, + { + id: 'identify-reset-on-logout', + area: 'Identification', + label: 'reset() called on logout / account switch', + }, + ], + }, + { + skillId: 'audit-subagents-event-capture', + area: 'Event Capture', + checks: [ + { + id: 'capture-event-names-static', + area: 'Event Capture', + label: 'Event names are static and consistent', + }, + { + id: 'capture-uses-proxy', + area: 'Event Capture', + label: 'Captures route through a reverse proxy', + }, + { + id: 'capture-growth-events', + area: 'Event Capture', + label: 'Key activation events captured', + }, + ], + }, +]; diff --git a/src/ui/tui/screens/audit/AuditAreaPane.tsx b/src/ui/tui/screens/audit/AuditAreaPane.tsx index 9111d857..de4db486 100644 --- a/src/ui/tui/screens/audit/AuditAreaPane.tsx +++ b/src/ui/tui/screens/audit/AuditAreaPane.tsx @@ -1,16 +1,25 @@ /** - * AuditAreaPane — left-pane slide that follows whatever area the agent is - * currently checking, plus a wrap-up state once every check is resolved - * and the agent has moved on to writing the report. + * AuditAreaPane — left-pane content that follows the agent's progress. * - * Three states, gated top-down on the ledger: - * 1. firstPending defined → render the slide for that area - * 2. checks empty → blank (the seed hook fires before - * this screen mounts in practice; - * this is just defensive) - * 3. all checks non-pending → "writing report" wrap-up + * Five states, gated top-down on the ledger + latest status string: + * 1. a basic area has a pending check → render that area's slide + * (Installation / Identification / Event Capture) + * 2. ledger empty → blank (defensive — the seed + * hook fires synchronously) + * 3. status indicates report writing → "wrapped up" wrap-up + * 4. discoverable areas exist in + * the ledger → "running expert subagents in + * parallel: " (variant B) + * 5. otherwise (basic resolved, no + * discoverable areas yet) → "essentials checked, dispatching + * experts" (variant A) * - * Pressing `O` opens the active slide's docs URL. + * The discovery / second-wave dispatch can leave the ledger fully resolved + * for several seconds while the dispatch agent decides what comes next. + * Variants A and B fill that window with truthful messaging until either + * new pending checks arrive or the report-writing status fires. + * + * Pressing `O` opens the active basic-area slide's docs URL. */ import { Fragment } from 'react'; @@ -18,8 +27,20 @@ import { Box, Text, useInput } from 'ink'; import { spawn } from 'node:child_process'; import { Colors } from '../../styles.js'; import { type AuditCheck } from '../../../../lib/workflows/audit/types.js'; +import { AUDIT_CORE_CHECKS } from '../../../../lib/workflows/audit/seed.js'; +import { AUDIT_SPECIALISTS } from '../../../../lib/workflows/audit/specialists.js'; import { AUDIT_AREA_SLIDES, type AreaSlide } from './slides/index.js'; +/** + * Areas owned by the basic / pre-seeded specialists (Installation, + * Identification, Event Capture). Anything else is discoverable — + * second-wave content the runner enrolls mid-run. + */ +const BASIC_AREAS: ReadonlySet = new Set([ + ...AUDIT_CORE_CHECKS.map((c) => c.area), + ...AUDIT_SPECIALISTS.map((s) => s.area), +]); + // ── Helpers ────────────────────────────────────────────────────────── const FINDING_STATUSES: AuditCheck['status'][] = [ @@ -52,11 +73,33 @@ const openLink = (url: string) => { interface AuditAreaPaneProps { checks: AuditCheck[]; reportPath: string; + /** Latest `[STATUS]` line emitted by the agent, if any. */ + latestStatus?: string; } -export const AuditAreaPane = ({ checks, reportPath }: AuditAreaPaneProps) => { - const pendingChecks = checks.filter((c) => c.status === 'pending'); - const activeArea = pendingChecks[0]?.area; +/** + * Heuristic: does the latest status line indicate the agent has reached + * the report-writing phase? Matches the canonical `[STATUS] Writing audit + * report` line emitted from `references/aggregation.md` (and a couple of + * close paraphrases the model occasionally produces). + */ +function isWritingReportStatus(status: string | undefined): boolean { + if (!status) return false; + return /\b(writing|composing|preparing).*(audit )?report\b/i.test(status); +} + +export const AuditAreaPane = ({ + checks, + reportPath, + latestStatus, +}: AuditAreaPaneProps) => { + // Pending check that belongs to a basic area. While any of these are + // pending, the active-area slide takes precedence — discoverable + // specialists run in parallel after the basic ones finish. + const basicPending = checks.find( + (c) => c.status === 'pending' && BASIC_AREAS.has(c.area), + ); + const activeArea = basicPending?.area; const slide = activeArea ? AUDIT_AREA_SLIDES.find((s) => s.area === activeArea) ?? fallbackSlide(activeArea) @@ -68,21 +111,34 @@ export const AuditAreaPane = ({ checks, reportPath }: AuditAreaPaneProps) => { } }); - // Active area — agent is still resolving checks for this slide's area. if (slide) { const hasFindings = checks.some(isFinding); return ; } - // Ledger empty — the seed hook fires synchronously at intro `onReady`, - // so this only happens if the seed file write failed. Render nothing - // rather than misleading the user with a "wrapped up" message. + // Ledger empty — defensive only; the seed hook fires synchronously. if (checks.length === 0) { return null; } - // Every check is resolved and the agent is composing the report. - return ; + if (isWritingReportStatus(latestStatus)) { + return ; + } + + // Distinct discoverable areas the runner has enrolled, in first-seen + // order. Empty until the dispatch agent picks specialists and the + // runner calls `audit_add_checks`. + const discoverAreas: string[] = []; + for (const check of checks) { + if (!BASIC_AREAS.has(check.area) && !discoverAreas.includes(check.area)) { + discoverAreas.push(check.area); + } + } + + if (discoverAreas.length > 0) { + return ; + } + return ; }; // ── States ─────────────────────────────────────────────────────────── @@ -126,6 +182,39 @@ const ActiveSlide = ({ ); +const DispatchingSubagents = () => ( + + + Essentials checked + + + + We've just checked your integration essentials. We're now going to run + expert subagents to check your product integration in more detail. + + +); + +const RunningSubagents = ({ areas }: { areas: string[] }) => ( + + + Running expert subagents + + + + We're running subagents to check against best practices for these products + in parallel: + + + {areas.map((area) => ( + + {' - '} + {area} + + ))} + +); + const WritingReport = ({ reportPath }: { reportPath: string }) => ( diff --git a/src/ui/tui/screens/audit/AuditRunScreen.tsx b/src/ui/tui/screens/audit/AuditRunScreen.tsx index 0e878f7d..0165e3b2 100644 --- a/src/ui/tui/screens/audit/AuditRunScreen.tsx +++ b/src/ui/tui/screens/audit/AuditRunScreen.tsx @@ -39,12 +39,19 @@ export const AuditRunScreen = ({ store }: AuditRunScreenProps) => { const statuses = store.statusMessages.length > 0 ? store.statusMessages : undefined; + const latestStatus = statuses?.[statuses.length - 1]; const [columns] = useStdoutDimensions(); const checks = getAuditChecks(store.session); const reportPath = `./${AUDIT_REPORT_FILE}`; const pendingChecksList = ; - const areaPane = ; + const areaPane = ( + + ); // Narrow terminals: drop the area pane. const statusComponent = diff --git a/src/ui/tui/screens/audit/slides/errorTracking.tsx b/src/ui/tui/screens/audit/slides/errorTracking.tsx new file mode 100644 index 00000000..3d5bb855 --- /dev/null +++ b/src/ui/tui/screens/audit/slides/errorTracking.tsx @@ -0,0 +1,10 @@ +import type { AreaSlide } from './shared.js'; + +export const ErrorTrackingSlide: AreaSlide = { + area: 'Error Tracking', + intro: [ + "We're checking that exceptions are captured through PostHog's `captureException` and that source maps are uploaded so stack traces are readable.", + 'Without both, errors land in your project as unsymbolicated noise and the issue list becomes hard to triage.', + ], + docsUrl: 'https://posthog.com/docs/error-tracking', +}; diff --git a/src/ui/tui/screens/audit/slides/experiments.tsx b/src/ui/tui/screens/audit/slides/experiments.tsx new file mode 100644 index 00000000..53814a50 --- /dev/null +++ b/src/ui/tui/screens/audit/slides/experiments.tsx @@ -0,0 +1,10 @@ +import type { AreaSlide } from './shared.js'; + +export const ExperimentsSlide: AreaSlide = { + area: 'Experiments', + intro: [ + "We're checking that exposure events fire when users see an experiment variant, and that variant assignment stays stable across a user's sessions.", + "Without reliable exposures or stable assignments, experiment results drift and you can't trust the lift numbers.", + ], + docsUrl: 'https://posthog.com/docs/experiments', +}; diff --git a/src/ui/tui/screens/audit/slides/featureFlags.tsx b/src/ui/tui/screens/audit/slides/featureFlags.tsx new file mode 100644 index 00000000..de7a3e5d --- /dev/null +++ b/src/ui/tui/screens/audit/slides/featureFlags.tsx @@ -0,0 +1,10 @@ +import type { AreaSlide } from './shared.js'; + +export const FeatureFlagsSlide: AreaSlide = { + area: 'Feature Flags', + intro: [ + "We're checking how your app evaluates feature flags — that flags are evaluated after PostHog initializes, and that bootstrap is configured for SSR or SPA setups.", + 'Flags evaluated too early or without bootstrap can cause flicker, flash-of-wrong-content, and inconsistent rollout coverage.', + ], + docsUrl: 'https://posthog.com/docs/feature-flags', +}; diff --git a/src/ui/tui/screens/audit/slides/index.ts b/src/ui/tui/screens/audit/slides/index.ts index 91b4b0d8..cd7855dc 100644 --- a/src/ui/tui/screens/audit/slides/index.ts +++ b/src/ui/tui/screens/audit/slides/index.ts @@ -8,6 +8,11 @@ import type { AreaSlide } from './shared.js'; import { InstallationSlide } from './installation.js'; import { IdentificationSlide } from './identification.js'; import { EventCaptureSlide } from './eventCapture.js'; +import { WebAnalyticsSlide } from './webAnalytics.js'; +import { FeatureFlagsSlide } from './featureFlags.js'; +import { ExperimentsSlide } from './experiments.js'; +import { LLMAnalyticsSlide } from './llmAnalytics.js'; +import { ErrorTrackingSlide } from './errorTracking.js'; export type { AreaSlide }; @@ -15,4 +20,9 @@ export const AUDIT_AREA_SLIDES: AreaSlide[] = [ InstallationSlide, IdentificationSlide, EventCaptureSlide, + WebAnalyticsSlide, + FeatureFlagsSlide, + ExperimentsSlide, + LLMAnalyticsSlide, + ErrorTrackingSlide, ]; diff --git a/src/ui/tui/screens/audit/slides/llmAnalytics.tsx b/src/ui/tui/screens/audit/slides/llmAnalytics.tsx new file mode 100644 index 00000000..e1bbbbb8 --- /dev/null +++ b/src/ui/tui/screens/audit/slides/llmAnalytics.tsx @@ -0,0 +1,10 @@ +import type { AreaSlide } from './shared.js'; + +export const LLMAnalyticsSlide: AreaSlide = { + area: 'LLM Analytics', + intro: [ + "We're checking that your AI generations are captured with `$ai_generation` events and that token counts plus cost properties are attached.", + 'Without these, model spend, latency, and prompt-level debugging stay invisible — and there is no path to evaluate quality across runs.', + ], + docsUrl: 'https://posthog.com/docs/ai-engineering', +}; diff --git a/src/ui/tui/screens/audit/slides/webAnalytics.tsx b/src/ui/tui/screens/audit/slides/webAnalytics.tsx new file mode 100644 index 00000000..fa14b565 --- /dev/null +++ b/src/ui/tui/screens/audit/slides/webAnalytics.tsx @@ -0,0 +1,10 @@ +import type { AreaSlide } from './shared.js'; + +export const WebAnalyticsSlide: AreaSlide = { + area: 'Web Analytics', + intro: [ + "We're checking how your browser SDK is configured for web analytics — reverse proxy coverage, authorized URLs, pageleave tracking, web vitals, and canonical URL handling.", + 'Misconfigurations here usually show up later as inflated session counts, missing exits, or duplicate pageviews across hosts.', + ], + docsUrl: 'https://posthog.com/docs/web-analytics', +};