Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 43 additions & 0 deletions src/lib/agent/agent-interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -684,6 +684,11 @@ export async function runAgent(
errorMessage?: string;
additionalFeatureQueue?: readonly AdditionalFeature[];
abortCases?: readonly AbortCaseMatcher[];
/**
* Emit a `wizard: step` event on each agent task transition. Threaded from
* `ProgramRun.trackStepProgress`; defaults off for every other caller.
*/
emitStepEvents?: boolean;
/** Request the end-of-run reflection remark. Defaults to true. */
requestRemark?: boolean;
/**
Expand All @@ -702,6 +707,7 @@ export async function runAgent(
successMessage = 'PostHog integration complete',
errorMessage = 'Integration failed',
abortCases = [],
emitStepEvents = false,
} = config ?? {};

logToFile('Starting agent run');
Expand Down Expand Up @@ -1023,6 +1029,7 @@ export async function runAgent(
receivedSuccessResult,
tasks,
isOrchestratorEnabled(agentConfig.wizardFlags ?? {}),
emitStepEvents,
);

// [ABORT] detection: the skill emits "[ABORT] <reason>" when it
Expand Down Expand Up @@ -1255,6 +1262,8 @@ type TaskEntry = { content: string; status: string; activeForm?: string };
interface TaskStore {
tasks: Map<string, TaskEntry>;
sync: () => void;
/** When true, emit a `wizard: step` event on each status transition. */
emitStepEvents?: boolean;
}

interface ToolUseBlock {
Expand Down Expand Up @@ -1294,6 +1303,36 @@ function handleTaskUpdate(block: ToolUseBlock, store: TaskStore): void {
if (input.status === 'deleted') {
store.tasks.delete(input.taskId);
} else {
// Per-step drop-off signal for programs that opt in via `trackStepProgress`
// (threaded here as `emitStepEvents`). Emit `wizard: step` on each real
// status transition so analytics can see how far a run got — even a silent
// step (no wizard_ask) that dies mid-run surfaces as its last `in_progress`
// with no matching `completed`. Generic: the step name is whatever the
// agent set; the `command` tag already identifies the program.
if (
store.emitStepEvents &&
input.status &&
input.status !== existing.status &&
(input.status === 'in_progress' || input.status === 'completed')
) {
const keys = [...store.tasks.keys()];
analytics.wizardCapture('step', {
// The task's display label lives on `activeForm` (what the TUI renders,
// e.g. "Checking access"); `content`/`subject` are typically empty on a
// status-only TaskUpdate. Prefer the stored entry, then the update, so
// the name is never null. Named `step_name` (not `step`): a bare
// `properties.step` doesn't resolve in HogQL — `step_name` queries
// cleanly, like `step_index` / `step_count`.
step_name:
existing.activeForm ??
input.activeForm ??
existing.content ??
input.subject,
Comment thread
sortafreel marked this conversation as resolved.
status: input.status,
step_index: keys.indexOf(input.taskId),
step_count: keys.length,
});
}
store.tasks.set(input.taskId, {
content: input.subject ?? existing.content,
status: input.status ?? existing.status,
Expand Down Expand Up @@ -1377,6 +1416,9 @@ function handleSDKMessage(
// The orchestrator owns the TUI task panel (it renders its queue). Suppress the
// agent's own TaskCreate/TaskUpdate rendering so it does not clobber the queue.
suppressTaskRender = false,
// Opt-in per-step analytics, threaded from runAgent's `emitStepEvents`
// (ProgramRun.trackStepProgress). Off for every program that doesn't opt in.
emitStepEvents = false,
): void {
// Map preserves insertion order (the order the agent created the tasks).
// Within that, group by status: completed first, then in_progress, then
Expand Down Expand Up @@ -1462,6 +1504,7 @@ function handleSDKMessage(
dispatchTaskToolUse(block as ToolUseBlock, {
tasks,
sync: syncTasks,
emitStepEvents,
});
}

Expand Down
1 change: 1 addition & 0 deletions src/lib/agent/runner/linear.ts
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,7 @@ export async function runLinearProgram(
errorMessage: config.errorMessage ?? `${config.integrationLabel} failed`,
additionalFeatureQueue: config.additionalFeatureQueue ?? [],
abortCases: config.abortCases,
emitStepEvents: config.trackStepProgress ?? false,
},
middleware,
);
Expand Down
8 changes: 8 additions & 0 deletions src/lib/agent/runner/shared/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,14 @@ export interface ProgramRun {
* key in the browser) before they can answer.
*/
askTimeoutMs?: number;
/**
* Emit a `wizard: step` analytics event on each agent task-list transition
* (in_progress / completed) so this program can build a step-level drop-off
* funnel — including silent steps that ask the user nothing. The step name is
* whatever the agent set on the task. Defaults to off, so no other program's
* analytics change; opt in per program.
*/
trackStepProgress?: boolean;
}

/**
Expand Down
6 changes: 6 additions & 0 deletions src/lib/programs/self-driving/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,12 @@ const run: ProgramRun = {
// from a decline (both resolve to __cancelled__). Match upload-source-maps.
askTimeoutMs: 30 * 60 * 1000,

// Emit a `wizard: step` analytics event on each agent task transition so we
// can build a step-level drop-off funnel (where a run stops — GitHub connect,
// scout enable, etc.), including silent steps with no wizard_ask. Opt-in, so
// only self-driving runs emit these; every other program is unchanged.
trackStepProgress: true,

postRun: async (session) => {
await removeInstalledSkill(session.installDir);
},
Expand Down
Loading