feat(telemetry): report task completion telemetry on every attempt_completion call

Author: edelauna

packages/telemetry/src/TelemetryService.ts

@@ -171,14 +171,27 @@ export class TelemetryService {
 	 * Captures task completion, optionally summarizing the per-task tool and
 	 * message counts that were previously reported as separate per-turn events
 	 * (`Tool Used`, `Conversation Message`) to reduce Product Analytics volume.
+	 *
+	 * A single task may emit this more than once over its lifetime (e.g. an
+	 * "idle" or "shutdown" installment followed later by a final
+	 * "attempt_completion" one) -- toolsUsed/messageCount are always the delta
+	 * since the previous emission for that task, not a running total, so
+	 * summing installments for a taskId reconstructs the full-task counts
+	 * without double-counting.
+	 *
+	 * Note "attempt_completion" means the model called that tool, not that the
+	 * user accepted the result -- it fires the same way whether the user goes
+	 * on to accept, decline, or give feedback instead.
 	 */
 	public captureTaskCompleted(
 		taskId: string,
 		toolsUsed?: ToolUsage,
 		messageCount?: { user: number; assistant: number },
+		completionReason: "attempt_completion" | "idle" | "shutdown" = "attempt_completion",
 	): void {
 		this.captureEvent(TelemetryEventName.TASK_COMPLETED, {
 			taskId,
+			completionReason,
 			...(toolsUsed !== undefined && { toolsUsed }),
 			...(messageCount !== undefined && { messageCount }),
 		})

packages/telemetry/src/__tests__/TelemetryService.task-completed.test.ts

@@ -18,14 +18,14 @@ describe("TelemetryService.captureTaskCompleted", () => {
 		}
 	})
 
-	it("captures Task Completed with the taskId when no summary is provided", () => {
+	it("captures Task Completed with the taskId and a default 'attempt_completion' completionReason when no summary is provided", () => {
 		const service = new TelemetryService([mockClient])
 
 		service.captureTaskCompleted("task_1")
 
 		expect(mockClient.capture).toHaveBeenCalledWith({
 			event: TelemetryEventName.TASK_COMPLETED,
-			properties: { taskId: "task_1" },
+			properties: { taskId: "task_1", completionReason: "attempt_completion" },
 		})
 	})
 
@@ -42,9 +42,25 @@ describe("TelemetryService.captureTaskCompleted", () => {
 			event: TelemetryEventName.TASK_COMPLETED,
 			properties: {
 				taskId: "task_1",
+				completionReason: "attempt_completion",
 				toolsUsed: { read_file: { attempts: 3, failures: 0 }, apply_diff: { attempts: 1, failures: 1 } },
 				messageCount: { user: 4, assistant: 5 },
 			},
 		})
 	})
+
+	it("includes the given completionReason for idle/shutdown installments", () => {
+		const service = new TelemetryService([mockClient])
+
+		service.captureTaskCompleted("task_1", { read_file: { attempts: 1, failures: 0 } }, undefined, "idle")
+
+		expect(mockClient.capture).toHaveBeenCalledWith({
+			event: TelemetryEventName.TASK_COMPLETED,
+			properties: {
+				taskId: "task_1",
+				completionReason: "idle",
+				toolsUsed: { read_file: { attempts: 1, failures: 0 } },
+			},
+		})
+	})
 })

packages/types/src/telemetry.ts

@@ -140,15 +140,25 @@ export const taskPropertiesSchema = z.object({
 			pending: z.number(),
 		})
 		.optional(),
-	// Per-task tool/message summaries, captured once on Task Completed instead
-	// of as separate per-turn events (reduces Product Analytics volume).
+	// Per-task tool/message summaries, captured once per Task Completed
+	// installment instead of as separate per-turn events (reduces Product
+	// Analytics volume). A single task may emit more than one installment
+	// (see completionReason); each installment's counts are the delta since
+	// the previous installment for that taskId, not a running total.
 	toolsUsed: z.record(z.string(), z.object({ attempts: z.number(), failures: z.number() })).optional(),
 	messageCount: z
 		.object({
 			user: z.number(),
 			assistant: z.number(),
 		})
 		.optional(),
+	// Why this Task Completed installment was emitted: the model called
+	// attempt_completion ("attempt_completion" -- regardless of whether the
+	// user went on to accept, decline, or give feedback; this is NOT a signal
+	// that the user accepted the result), the task went idle with unreported
+	// activity ("idle"), or the extension/task was shut down with unreported
+	// activity still pending ("shutdown").
+	completionReason: z.enum(["attempt_completion", "idle", "shutdown"]).optional(),
 })
 
 export type TaskProperties = z.infer<typeof taskPropertiesSchema>

src/__tests__/nested-delegation-resume.spec.ts

@@ -199,10 +199,12 @@ describe("Nested delegation resume (A → B → C)", () => {
 			emit: vi.fn(),
 			getTokenUsage: vi.fn(() => ({})),
 			toolUsage: {},
+			messageCounts: { user: 0, assistant: 0 },
 			clineMessages: [],
 			userMessageContent: [],
 			consecutiveMistakeCount: 0,
 			emitFinalTokenUsageUpdate: vi.fn(),
+			flushTelemetryInstallment: vi.fn(),
 		} as unknown as Task
 
 		const blockC = {
@@ -246,10 +248,12 @@ describe("Nested delegation resume (A → B → C)", () => {
 			emit: vi.fn(),
 			getTokenUsage: vi.fn(() => ({})),
 			toolUsage: {},
+			messageCounts: { user: 0, assistant: 0 },
 			clineMessages: [],
 			userMessageContent: [],
 			consecutiveMistakeCount: 0,
 			emitFinalTokenUsageUpdate: vi.fn(),
+			flushTelemetryInstallment: vi.fn(),
 		} as unknown as Task
 
 		const blockB = {

src/core/task/Task.ts

@@ -322,10 +322,25 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 	consecutiveNoAssistantMessagesCount: number = 0
 	toolUsage: ToolUsage = {}
 
-	// Conversation message counts, summarized once on Task Completed instead
-	// of emitting a separate telemetry event per turn.
+	// Conversation message counts, summarized once per Task Completed
+	// installment instead of emitting a separate telemetry event per turn.
 	messageCounts: { user: number; assistant: number } = { user: 0, assistant: 0 }
 
+	// Idle/shutdown telemetry flush: reports toolUsage/messageCounts for tasks that
+	// go quiet or get torn down without the model ever calling attempt_completion
+	// (or without the user accepting it), so long-running/abandoned tasks aren't
+	// invisible to telemetry. Each flush reports only what changed since the previous
+	// one, tracked via telemetryToolUsageBaseline/telemetryMessageCountsBaseline --
+	// task.toolUsage/messageCounts themselves are never mutated by this, since they're
+	// also read as running totals by the public TaskCompleted API event and the UI.
+	// Checked on an interval rather than hooked into every say()/ask() call site.
+	private static readonly IDLE_TELEMETRY_CHECK_INTERVAL_MS = 5 * 60 * 1000
+	private static readonly IDLE_TELEMETRY_THRESHOLD_MS = 30 * 60 * 1000
+	private idleTelemetryCheckInterval?: NodeJS.Timeout
+	private lastTelemetryFlushAt: number = Date.now()
+	private telemetryToolUsageBaseline: ToolUsage = {}
+	private telemetryMessageCountsBaseline: { user: number; assistant: number } = { user: 0, assistant: 0 }
+
 	// Checkpoints
 	enableCheckpoints: boolean
 	checkpointTimeout: number
@@ -598,6 +613,8 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 			{ leading: true, trailing: true, maxWait: this.TOKEN_USAGE_EMIT_INTERVAL_MS },
 		)
 
+		this.startIdleTelemetryCheck()
+
 		onCreated?.(this)
 
 		if (startTask) {
@@ -2238,6 +2255,17 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 	public dispose(): void {
 		console.log(`[Task#dispose] disposing task ${this.taskId}.${this.instanceId}`)
 
+		// Stop the idle telemetry check and report any unflushed activity as a
+		// shutdown installment, so a task torn down mid-work (panel closed, task
+		// switched, extension deactivated) isn't invisible to telemetry.
+		try {
+			clearInterval(this.idleTelemetryCheckInterval)
+			this.idleTelemetryCheckInterval = undefined
+			this.flushTelemetryInstallment("shutdown")
+		} catch (error) {
+			console.error("Error flushing shutdown telemetry:", error)
+		}
+
 		// Cancel any in-progress HTTP request
 		try {
 			this.cancelCurrentRequest()
@@ -4645,6 +4673,70 @@ export class Task extends EventEmitter<TaskEvents> implements TaskLike {
 		}
 	}
 
+	/**
+	 * Emits a Task Completed installment for whatever toolUsage/messageCounts have
+	 * changed since the previous installment (from any reason), then advances the
+	 * telemetry baseline so a later installment reports only its own delta. Does
+	 * NOT touch task.toolUsage/messageCounts themselves -- those stay running totals
+	 * for the public TaskCompleted API event and the UI. No-ops if nothing changed
+	 * since the last installment, so idle/shutdown checks don't emit empty events
+	 * for tasks that were already fully reported (e.g. right after attempt_completion).
+	 */
+	public flushTelemetryInstallment(reason: "attempt_completion" | "idle" | "shutdown"): void {
+		const toolUsageDelta: ToolUsage = {}
+
+		for (const [toolName, usage] of Object.entries(this.toolUsage) as [ToolName, ToolUsage[ToolName]][]) {
+			if (!usage) {
+				continue
+			}
+
+			const baseline = this.telemetryToolUsageBaseline[toolName]
+			const attempts = usage.attempts - (baseline?.attempts ?? 0)
+			const failures = usage.failures - (baseline?.failures ?? 0)
+
+			if (attempts > 0 || failures > 0) {
+				toolUsageDelta[toolName] = { attempts, failures }
+			}
+		}
+
+		const messageCountDelta = {
+			user: this.messageCounts.user - this.telemetryMessageCountsBaseline.user,
+			assistant: this.messageCounts.assistant - this.telemetryMessageCountsBaseline.assistant,
+		}
+
+		const hasToolUsageDelta = Object.keys(toolUsageDelta).length > 0
+		const hasMessageDelta = messageCountDelta.user > 0 || messageCountDelta.assistant > 0
+
+		if (!hasToolUsageDelta && !hasMessageDelta) {
+			return
+		}
+
+		this.emitFinalTokenUsageUpdate()
+		TelemetryService.instance.captureTaskCompleted(this.taskId, toolUsageDelta, messageCountDelta, reason)
+
+		this.telemetryToolUsageBaseline = JSON.parse(JSON.stringify(this.toolUsage))
+		this.telemetryMessageCountsBaseline = { ...this.messageCounts }
+		this.lastTelemetryFlushAt = Date.now()
+	}
+
+	private startIdleTelemetryCheck(): void {
+		this.idleTelemetryCheckInterval = setInterval(() => {
+			// lastMessageTs only moves forward on activity, so comparing it against the
+			// last flush tells us whether anything happened since that flush -- if the
+			// task has been quiet since well before the last flush, there's nothing new
+			// to report and flushTelemetryInstallment's own empty-check would no-op anyway,
+			// but skipping here avoids waking up to do that check needlessly.
+			const idleForMs = Date.now() - (this.lastMessageTs ?? this.lastTelemetryFlushAt)
+
+			if (idleForMs >= Task.IDLE_TELEMETRY_THRESHOLD_MS) {
+				this.flushTelemetryInstallment("idle")
+			}
+		}, Task.IDLE_TELEMETRY_CHECK_INTERVAL_MS)
+
+		// Don't hold the process open just for this timer.
+		this.idleTelemetryCheckInterval?.unref?.()
+	}
+
 	// Getters
 
 	public get taskStatus(): TaskStatus {

src/core/task/__tests__/Task.persistence.spec.ts

@@ -303,6 +303,10 @@ describe("Task persistence", () => {
 				task: "test task",
 				startTask: false,
 			})
+			// Dispose the idle-telemetry-check interval before running all timers, so
+			// runAllTimersAsync doesn't treat it as an infinite loop (it's unrelated
+			// to what this test exercises).
+			task.dispose()
 
 			const promise = task.retrySaveApiConversationHistory()
 			await vi.runAllTimersAsync()
@@ -325,6 +329,10 @@ describe("Task persistence", () => {
 				task: "test task",
 				startTask: false,
 			})
+			// Dispose the idle-telemetry-check interval before running all timers, so
+			// runAllTimersAsync doesn't treat it as an infinite loop (it's unrelated
+			// to what this test exercises).
+			task.dispose()
 
 			const promise = task.retrySaveApiConversationHistory()
 			await vi.runAllTimersAsync()