feat(analytics): identify the user so feature flags can target by email

identifyUser sets the distinct id and person properties (email, name) before flag evaluation, and getFeatureFlag/getAllFlagsForWizard now send them. Previously the wizard sent only $app_name, so email-targeted flags never matched.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: gewenyu99

src/lib/agent/agent-interface.ts

@@ -803,6 +803,11 @@ export async function runAgent(
     abortCases?: readonly AbortCaseMatcher[];
     /** Request the end-of-run reflection remark. Defaults to true. */
     requestRemark?: boolean;
+    /**
+     * Extra properties attached to this run's `agent completed` / `agent
+     * aborted` events (e.g. the orchestrator's task type and id).
+     */
+    analyticsProperties?: Record<string, unknown>;
   },
   middleware?: {
     onMessage(message: any): void;
@@ -880,9 +885,27 @@ export async function runAgent(
       analytics.capture(WIZARD_REMARK_EVENT_NAME, { remark });
     }
 
+    // Token usage comes from the SDK result message and is per agent run —
+    // for the orchestrator that means per task, the secondary cost to watch.
+    const usage = lastResultMessage?.usage as
+      | {
+          input_tokens?: number;
+          output_tokens?: number;
+          cache_creation_input_tokens?: number;
+          cache_read_input_tokens?: number;
+        }
+      | undefined;
     analytics.wizardCapture('agent completed', {
       duration_ms: durationMs,
       duration_seconds: durationSeconds,
+      model: agentConfig.model,
+      num_turns: lastResultMessage?.num_turns,
+      total_cost_usd: lastResultMessage?.total_cost_usd,
+      input_tokens: usage?.input_tokens,
+      output_tokens: usage?.output_tokens,
+      cache_creation_input_tokens: usage?.cache_creation_input_tokens,
+      cache_read_input_tokens: usage?.cache_read_input_tokens,
+      ...config?.analyticsProperties,
     });
     try {
       middleware?.finalize(lastResultMessage, durationMs);
@@ -1280,6 +1303,8 @@ export async function runAgent(
       analytics.wizardCapture('agent aborted', {
         duration_ms: durationMs,
         duration_seconds: Math.round(durationMs / 1000),
+        model: agentConfig.model,
+        ...config?.analyticsProperties,
       });
     }
   }

src/lib/agent/agent-runner.ts

@@ -326,12 +326,18 @@ async function bootstrapProgram(
   getUI().setRoleAtOrganization(roleAtOrganization);
   getUI().setApiUser(user);
 
+  // Identify the user (email, name) before evaluating flags, so flags can target
+  // the individual user and not just $app_name.
+  if (user) analytics.identifyUser(user);
   analytics.setGroups(groupsFromUser(user, host));
 
   // Feature flags, variant metadata, and MCP url. Both arms need these, and the
   // fork decision reads the flags.
   const wizardFlags = await analytics.getAllFlagsForWizard();
   const wizardMetadata = buildWizardMetadata(wizardFlags);
+  // Tag every wizard event with the variant so runs segment in PostHog; the
+  // orchestrator arm overwrites this with its own variant when it forks.
+  analytics.setTag('variant', wizardMetadata.VARIANT);
 
   const mcpUrl = session.localMcp
     ? 'http://localhost:8787/mcp'

src/lib/programs/orchestrator/__tests__/agent-prompt-loader.test.ts

@@ -3,11 +3,14 @@ import * as os from 'os';
 import * as path from 'path';
 import {
   agentRunTools,
+  assembleTaskPrompt,
   buildRegistry,
   parseAgentPrompt,
   resolveTask,
+  taskModel,
   type AgentPrompt,
   type AgentRegistry,
+  type OrchestratorPromptContext,
 } from '../agent-prompt-loader';
 import { QueueStore } from '../queue';
 
@@ -203,3 +206,42 @@ describe('resolveTask', () => {
     );
   });
 });
+
+describe('taskModel', () => {
+  const prompt = parseAgentPrompt(
+    '---\nmodel: prompt-model\n---\nx',
+    'capture',
+  );
+
+  it('prefers the enqueue override, then the prompt, then the default', () => {
+    const registry = registryOf([prompt]);
+    const task = { type: 'capture' };
+    expect(taskModel(registry, { ...task, model: 'override' } as never)).toBe(
+      'override',
+    );
+    expect(taskModel(registry, task as never)).toBe('prompt-model');
+    expect(taskModel(registryOf([]), task as never)).toBe('claude-sonnet-4-6');
+  });
+});
+
+describe('assembleTaskPrompt', () => {
+  const ctx: OrchestratorPromptContext = {
+    projectId: 1,
+    projectApiKey: 'phc_x',
+    host: 'https://us.posthog.com',
+  };
+
+  it('points the agent at its installed task instructions', () => {
+    const assembled = assembleTaskPrompt(ctx, 'do the task', [
+      '.posthog-wizard/skills/capture/SKILL.md',
+    ]);
+    expect(assembled).toContain('.posthog-wizard/skills/capture/SKILL.md');
+    expect(assembled).toContain('do the task');
+  });
+
+  it('omits the instructions section when no skills are installed', () => {
+    expect(assembleTaskPrompt(ctx, 'do the task')).not.toContain(
+      'task instructions',
+    );
+  });
+});

src/lib/programs/orchestrator/__tests__/queue.test.ts

@@ -7,6 +7,10 @@ import {
   type TaskHandoff,
 } from '@lib/programs/orchestrator/queue';
 
+jest.mock('@utils/analytics', () => ({
+  analytics: { captureException: jest.fn(), wizardCapture: jest.fn() },
+}));
+
 function tmpDir(): string {
   return fs.mkdtempSync(path.join(os.tmpdir(), 'queue-test-'));
 }
@@ -132,4 +136,40 @@ describe('QueueStore', () => {
     expect(file.tasks[0].status).toBe('done');
     expect(file.tasks[0].handoff?.did).toBe('d');
   });
+
+  it('notifies the transition listener with post-transition task state', () => {
+    const seen: Array<{ event: string; status: string; attempts: number }> = [];
+    const listened = new QueueStore(dir, 'run-2', {
+      onTransition: (event, task) =>
+        seen.push({ event, status: task.status, attempts: task.attempts }),
+    });
+
+    const t = listened.enqueue({ type: 'install' });
+    listened.start(t.id);
+    listened.fail(t.id, { type: 'API_ERROR', message: 'boom' });
+    listened.requeue(t.id);
+    listened.start(t.id);
+    listened.complete(t.id);
+
+    expect(seen).toEqual([
+      { event: 'enqueue', status: 'pending', attempts: 0 },
+      { event: 'start', status: 'in_progress', attempts: 1 },
+      { event: 'fail', status: 'failed', attempts: 1 },
+      { event: 'requeue', status: 'pending', attempts: 1 },
+      { event: 'start', status: 'in_progress', attempts: 2 },
+      { event: 'complete', status: 'done', attempts: 2 },
+    ]);
+  });
+
+  it('a throwing listener does not break transitions', () => {
+    const listened = new QueueStore(dir, 'run-3', {
+      onTransition: () => {
+        throw new Error('listener boom');
+      },
+    });
+    const t = listened.enqueue({ type: 'install' });
+    listened.start(t.id);
+    listened.complete(t.id);
+    expect(listened.get(t.id)?.status).toBe('done');
+  });
 });

src/lib/programs/orchestrator/agent-prompt-loader.ts

@@ -51,12 +51,30 @@ const TASK_BASICS = `You are one isolated task in a larger PostHog workflow, run
 
 const SEED_BASICS = `You are the orchestrator. Plan the work and seed the queue with enqueue_task — each call returns an id you can pass as a dependency to a later task. Give each task a short label for the UI — the action in a few words, not file names, class names, or other specifics. You are not a task yourself: do not call complete_task and do not edit the project.`;
 
+/**
+ * Points the agent at its installed task instructions (the HOW). They live under
+ * the wizard's run dir, not `.claude/skills/`, so the SDK does not auto-load
+ * them — the prompt has to name them.
+ */
+function skillReference(paths: readonly string[]): string | null {
+  if (paths.length === 0) return null;
+  const list = paths.map((p) => `\`${p}\``).join(', ');
+  return `Your task instructions are at ${list}. Read them before you start and follow them. They are wizard scaffolding, not part of the project.`;
+}
+
 /** A task agent's full prompt: injected basics, then the authored intent. */
 export function assembleTaskPrompt(
   ctx: OrchestratorPromptContext,
   body: string,
+  skillPaths: readonly string[] = [],
 ): string {
-  return [projectContext(ctx), exampleReference(ctx), TASK_BASICS, body]
+  return [
+    projectContext(ctx),
+    exampleReference(ctx),
+    skillReference(skillPaths),
+    TASK_BASICS,
+    body,
+  ]
     .filter(Boolean)
     .join('\n\n');
 }
@@ -288,9 +306,14 @@ export function resolveTask(
     .join('\n\n');
 
   return {
-    model: task.model ?? prompt.model ?? DEFAULT_TASK_MODEL,
+    model: taskModel(registry, task),
     ...agentRunTools(prompt),
     prompt: body,
     skills: prompt.skills,
   };
 }
+
+/** The model a task runs on: enqueue override, then prompt frontmatter, then default. */
+export function taskModel(registry: AgentRegistry, task: QueuedTask): string {
+  return task.model ?? registry.get(task.type)?.model ?? DEFAULT_TASK_MODEL;
+}