feat: add execution context to telemetry events (#1074)

patrikbraborec · web-flow · commit 9288ff886864 · 2026-04-13T11:56:55.000+03:00
Track exit code, duration, CI/AI agent environment, interactivity, and
command retry detection in telemetry data for better CLI usage insights.
diff --git a/package.json b/package.json
@@ -85,6 +85,7 @@
         "archiver": "~7.0.1",
         "axios": "^1.11.0",
         "chalk": "~5.6.0",
+        "ci-info": "~4.4.0",
         "cli-table3": "^0.6.5",
         "computer-name": "~0.1.0",
         "configparser": "~0.3.10",
diff --git a/src/lib/command-framework/apify-command.ts b/src/lib/command-framework/apify-command.ts
@@ -9,8 +9,10 @@ import widestLine from 'widest-line';
 import wrapAnsi from 'wrap-ansi';
 
 import { cachedStdinInput } from '../../entrypoints/_shared.js';
+import { detectAiAgent, detectCi, detectIsInteractive } from '../hooks/telemetry/detectEnvironment.js';
 import type { TrackEventMap } from '../hooks/telemetry/trackEvent.js';
 import { trackEvent } from '../hooks/telemetry/trackEvent.js';
+import { checkAndUpdateLastCommand } from '../hooks/telemetry/useTelemetryState.js';
 import { useCLIMetadata } from '../hooks/useCLIMetadata.js';
 import { ProjectLanguage, useCwdProject } from '../hooks/useCwdProject.js';
 import { error } from '../outputs.js';
@@ -220,6 +222,12 @@ export abstract class ApifyCommand<T extends typeof BuiltApifyCommand = typeof B
 
 		this.telemetryData.commandString = commandString;
 		this.telemetryData.entrypoint = entrypoint;
+
+		const ci = detectCi();
+		this.telemetryData.aiAgent = detectAiAgent();
+		this.telemetryData.isCi = ci.isCi;
+		this.telemetryData.ciProvider = ci.ciProvider;
+		this.telemetryData.isInteractive = detectIsInteractive();
 	}
 
 	abstract run(): Awaitable<void>;
@@ -243,6 +251,7 @@ export abstract class ApifyCommand<T extends typeof BuiltApifyCommand = typeof B
 	}
 
 	private async _run(parseResult: ParseResult) {
+		const startTime = Date.now();
 		const { values: rawFlags, positionals: rawArgs, tokens: rawTokens } = parseResult;
 
 		if (rawFlags.help) {
@@ -329,9 +338,13 @@ export abstract class ApifyCommand<T extends typeof BuiltApifyCommand = typeof B
 				});
 			}
 
-			this.telemetryData.flagsUsed = Object.keys(this.flags);
-
 			if (!this.skipTelemetry) {
+				this.telemetryData.flagsUsed = Object.keys(this.flags);
+				this.telemetryData.exitCode = typeof process.exitCode === 'number' ? process.exitCode : 0;
+				this.telemetryData.durationMs = Date.now() - startTime;
+
+				this.telemetryData.wasRetried = await checkAndUpdateLastCommand(this.commandString);
+
 				await trackEvent(
 					`cli_command_${this.commandString.replaceAll(' ', '_').toLowerCase()}` as const,
 					this.telemetryData,
diff --git a/src/lib/hooks/telemetry/detectEnvironment.ts b/src/lib/hooks/telemetry/detectEnvironment.ts
@@ -0,0 +1,35 @@
+import ciInfo from 'ci-info';
+
+const AI_AGENT_ENV_VARS: [string, string][] = [
+	['CLAUDECODE', 'claude_code'],
+	['CLAUDE_CODE_ENTRYPOINT', 'claude_code'],
+	['CURSOR_AGENT', 'cursor'],
+	['CLINE_ACTIVE', 'cline'],
+	['CODEX_SANDBOX', 'codex_cli'],
+	['CODEX_THREAD_ID', 'codex_cli'],
+	['GEMINI_CLI', 'gemini_cli'],
+	['OPENCODE', 'open_code'],
+	['OPENCLAW_SHELL', 'openclaw'],
+];
+
+export function detectAiAgent(): string | undefined {
+	for (const [envVar, agent] of AI_AGENT_ENV_VARS) {
+		if (process.env[envVar]) {
+			return agent;
+		}
+	}
+
+	return undefined;
+}
+
+export function detectCi(): { isCi: boolean; ciProvider: string | undefined } {
+	if (!ciInfo.isCI) {
+		return { isCi: false, ciProvider: undefined };
+	}
+
+	return { isCi: true, ciProvider: ciInfo.id?.toLowerCase() ?? 'unknown' };
+}
+
+export function detectIsInteractive(): boolean {
+	return !!process.stdin.isTTY && !!process.stdout.isTTY;
+}
diff --git a/src/lib/hooks/telemetry/trackEvent.ts b/src/lib/hooks/telemetry/trackEvent.ts
@@ -37,6 +37,15 @@ export interface TrackEventMap {
 
 		// init command
 		actorWrapper?: string;
+
+		// execution context
+		exitCode?: number;
+		durationMs?: number;
+		aiAgent?: string;
+		isCi?: boolean;
+		ciProvider?: string;
+		isInteractive?: boolean;
+		wasRetried?: boolean;
 	};
 }
 
diff --git a/src/lib/hooks/telemetry/useTelemetryState.ts b/src/lib/hooks/telemetry/useTelemetryState.ts
@@ -22,6 +22,8 @@ interface TelemetryStateV1 {
 	enabled: boolean;
 	userId?: string | null;
 	anonymousId: string;
+	lastCommand?: string;
+	lastCommandTimestamp?: number;
 }
 
 const telemetryWarningText = [
@@ -71,7 +73,13 @@ export async function useTelemetryState(): Promise<LatestTelemetryState> {
 		});
 
 		// First time we are tracking telemetry, so we want to notify user about it.
-		info({ message: telemetryWarningText });
+		// Skip the notice if telemetry is disabled via env var — the user already opted out.
+		if (
+			!process.env.APIFY_CLI_DISABLE_TELEMETRY ||
+			['false', '0'].includes(process.env.APIFY_CLI_DISABLE_TELEMETRY)
+		) {
+			info({ message: telemetryWarningText });
+		}
 
 		return useTelemetryState();
 	}
@@ -110,6 +118,33 @@ export async function updateUserId(userId: string | null) {
 	});
 }
 
+/** Max time (ms) between identical commands to consider the second one a retry (e.g. user re-running after a failure). */
+const RETRY_WINDOW_MS = 10_000;
+
+/**
+ * Checks whether the same command was executed within {@link RETRY_WINDOW_MS} and updates the
+ * last-command state for future calls. Detection is best-effort — concurrent invocations may
+ * both read stale state, which is acceptable for an analytics heuristic.
+ */
+export async function checkAndUpdateLastCommand(commandString: string): Promise<boolean> {
+	try {
+		const state = await useTelemetryState();
+		const now = Date.now();
+
+		const wasRetried =
+			state.lastCommand === commandString && now - (state.lastCommandTimestamp ?? 0) < RETRY_WINDOW_MS;
+
+		updateTelemetryState(state, (stateToUpdate) => {
+			stateToUpdate.lastCommand = commandString;
+			stateToUpdate.lastCommandTimestamp = now;
+		});
+
+		return wasRetried;
+	} catch {
+		return false;
+	}
+}
+
 export async function updateTelemetryEnabled(enabled: boolean) {
 	const state = await useTelemetryState();
 
diff --git a/test/lib/hooks/telemetry/checkAndUpdateLastCommand.test.ts b/test/lib/hooks/telemetry/checkAndUpdateLastCommand.test.ts
@@ -0,0 +1,222 @@
+import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join } from 'node:path';
+
+let telemetryFilePath: string;
+
+vi.mock('../../../../src/lib/consts.js', async (importOriginal) => {
+	const original = await importOriginal<typeof import('../../../../src/lib/consts.js')>();
+
+	return {
+		...original,
+		TELEMETRY_FILE_PATH: () => telemetryFilePath,
+	};
+});
+
+vi.mock('../../../../src/lib/utils.js', () => ({
+	getLocalUserInfo: async () => ({}),
+}));
+
+vi.mock('../../../../src/lib/outputs.js', () => ({
+	info: () => {
+		/* noop */
+	},
+}));
+
+function writeTelemetryState(state: Record<string, unknown>) {
+	const dir = dirname(telemetryFilePath);
+	if (!existsSync(dir)) {
+		mkdirSync(dir, { recursive: true });
+	}
+	writeFileSync(telemetryFilePath, JSON.stringify(state, null, '\t'));
+}
+
+function readTelemetryState() {
+	return JSON.parse(readFileSync(telemetryFilePath, 'utf-8'));
+}
+
+describe('checkAndUpdateLastCommand', () => {
+	let testDir: string;
+	let counter = 0;
+
+	beforeEach(() => {
+		counter++;
+		testDir = join(tmpdir(), `apify-cli-test-telemetry-${process.pid}-${counter}-${Date.now()}`);
+		telemetryFilePath = join(testDir, 'telemetry.json');
+		vi.useFakeTimers();
+	});
+
+	afterEach(() => {
+		vi.useRealTimers();
+		// Clean up temp files
+		if (existsSync(testDir)) {
+			rmSync(testDir, { recursive: true, force: true });
+		}
+	});
+
+	test('returns false on first invocation (no prior command)', async () => {
+		vi.setSystemTime(1000);
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(false);
+	});
+
+	test('stores the command and timestamp in telemetry state', async () => {
+		vi.setSystemTime(50_000);
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		await checkAndUpdateLastCommand('apify push');
+
+		const state = readTelemetryState();
+		expect(state.lastCommand).toBe('apify push');
+		expect(state.lastCommandTimestamp).toBe(50_000);
+	});
+
+	test('returns true when the same command is repeated within the retry window', async () => {
+		vi.setSystemTime(100_000);
+
+		// Seed state with a recent identical command
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 95_000, // 5 seconds ago — within the 10s window
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(true);
+	});
+
+	test('returns false when the same command is repeated outside the retry window', async () => {
+		vi.setSystemTime(100_000);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 80_000, // 20 seconds ago — outside the 10s window
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(false);
+	});
+
+	test('returns false when a different command is run within the retry window', async () => {
+		vi.setSystemTime(100_000);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 95_000,
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify push');
+
+		expect(result).toBe(false);
+	});
+
+	test('updates state after checking so the next call sees the new command', async () => {
+		vi.setSystemTime(100_000);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 90_000,
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		await checkAndUpdateLastCommand('apify push');
+
+		const state = readTelemetryState();
+		expect(state.lastCommand).toBe('apify push');
+		expect(state.lastCommandTimestamp).toBe(100_000);
+	});
+
+	test('returns false when lastCommandTimestamp is missing', async () => {
+		vi.setSystemTime(100_000);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			// no lastCommandTimestamp
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(false);
+	});
+
+	test('returns false when telemetry state file is corrupted', async () => {
+		// Write invalid JSON
+		const dir = dirname(telemetryFilePath);
+		mkdirSync(dir, { recursive: true });
+		writeFileSync(telemetryFilePath, '{{{invalid json');
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(false);
+	});
+
+	test('returns true at exactly the retry window boundary', async () => {
+		// Command was run exactly 9999ms ago (just inside the 10_000ms window)
+		vi.setSystemTime(109_999);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 100_000,
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(true);
+	});
+
+	test('returns false at exactly the retry window boundary (equal to window)', async () => {
+		// Command was run exactly 10_000ms ago (at the boundary, not strictly less than)
+		vi.setSystemTime(110_000);
+
+		writeTelemetryState({
+			version: 1,
+			enabled: true,
+			anonymousId: 'CLI:test',
+			lastCommand: 'apify run',
+			lastCommandTimestamp: 100_000,
+		});
+
+		const { checkAndUpdateLastCommand } = await import('../../../../src/lib/hooks/telemetry/useTelemetryState.js');
+
+		const result = await checkAndUpdateLastCommand('apify run');
+
+		expect(result).toBe(false);
+	});
+});
diff --git a/test/lib/hooks/telemetry/detectEnvironment.test.ts b/test/lib/hooks/telemetry/detectEnvironment.test.ts
diff --git a/yarn.lock b/yarn.lock
