feat: add telemetry for CLI and remote setup (#993)

Adds structured, bounded traces for the CLI and remote-setup flows,
following the instrumentation conventions (snake_case keys, subject-first
phases, typed unions, error.type / abort_stage via recordError/recordAborted).

Traces:
- cli.resolve: parent trace over cache_lookup, version_check, lock_wait,
  and fallback_to_existing_binary phases; surfaces outcome, cache_source,
  download_reason/action, error.type, and fallback_reason on the parent.
- cli.download: reason + downloaded_bytes, with a verify child phase.
- cli.configure: silent, credential_source, error.type, abort_stage.
- remote.setup: child phases for each setup step, with a typed outcome
  for non-throwing early exits.
- websocket reconnect: adds attempts, max_backoff_ms, total_duration_ms.

Implementation:
- Telemetry threaded explicitly via a trace/span arg; public methods are
  thin wrappers over private business logic.
- cliManager unit and telemetry tests share a setupCliManager() harness.

Closes #985

Author: EhabY

src/core/cliManager.ts

@@ -0,0 +1,210 @@
+import { CredentialFileError } from "./credentials";
+import { recordAborted, recordError } from "./outcomes";
+
+import type { CallerPropertyValue } from "../telemetry/event";
+import type { TelemetryService } from "../telemetry/service";
+import type { Span } from "../telemetry/span";
+
+export type CliCacheSource = "file-path" | "directory" | "not-found";
+export type CliDownloadReason = "missing" | "version_mismatch" | "unreadable";
+export type CliDownloadAction = "download" | "fallback" | "blocked";
+export type CliCredentialSource = "session_token" | "empty_token";
+export type CliResolveOutcome =
+	| "cache_hit"
+	| "downloaded"
+	| "lock_wait_cache_hit"
+	| "download_disabled_fallback"
+	| "fallback_to_existing_binary";
+export type CliVersionCheckOutcome =
+	| "missing"
+	| "match"
+	| "mismatch"
+	| "unreadable";
+export type CliConfigureErrorType =
+	| "filesystem"
+	| "credential_store"
+	| "unknown";
+export type CliResolveErrorType =
+	| "downloads_disabled"
+	| "download"
+	| "fallback_declined";
+
+interface CliConfigureOptions {
+	readonly silent: boolean;
+	readonly credentialSource: CliCredentialSource;
+}
+
+export class CliDownloadsDisabledError extends Error {
+	public constructor() {
+		super("Unable to download CLI because downloads are disabled");
+		this.name = "CliDownloadsDisabledError";
+	}
+}
+
+export class CliFallbackDeclinedError extends Error {
+	public constructor(cause: unknown) {
+		super(
+			cause instanceof Error ? cause.message : "CLI binary fallback declined",
+			{
+				cause,
+			},
+		);
+		this.name = "CliFallbackDeclinedError";
+	}
+}
+
+export class CliTelemetry {
+	public constructor(private readonly telemetry: TelemetryService) {}
+
+	public traceResolve<T>(
+		fn: (trace: CliResolveTrace) => Promise<T>,
+	): Promise<T> {
+		return this.telemetry.trace("cli.resolve", (span) =>
+			fn(new CliResolveTrace(span)),
+		);
+	}
+
+	public traceDownload<T>(
+		reason: CliDownloadReason,
+		fn: (span: Span) => Promise<T>,
+	): Promise<T> {
+		return this.telemetry.trace("cli.download", fn, { reason });
+	}
+
+	public traceConfigure<T>(
+		options: CliConfigureOptions,
+		fn: (trace: CliConfigureTrace) => Promise<T>,
+	): Promise<T> {
+		return this.telemetry.trace("cli.configure", (span) => {
+			span.setProperty("silent", options.silent);
+			span.setProperty("credential_source", options.credentialSource);
+			return fn(new CliConfigureTrace(span));
+		});
+	}
+}
+
+export class CliResolveTrace {
+	public constructor(private readonly span: Span) {}
+
+	public setOutcome(outcome: CliResolveOutcome): void {
+		this.span.setProperty("outcome", outcome);
+	}
+
+	public error(category: CliResolveErrorType | "unknown"): void {
+		recordError(this.span, category);
+	}
+
+	public cacheLookup<T extends { readonly source: CliCacheSource }>(
+		fn: () => Promise<T>,
+	): Promise<T> {
+		return this.tracedPhase("cache_lookup", fn, (r) => r.source, {
+			child: "source",
+			parent: "cache_source",
+		});
+	}
+
+	public versionCheck<T extends { readonly outcome: CliVersionCheckOutcome }>(
+		fn: () => Promise<T>,
+	): Promise<T> {
+		return this.tracedPhase("version_check", fn, (r) => r.outcome, {
+			child: "outcome",
+			parent: "version_check",
+		});
+	}
+
+	public lockWait<T extends { readonly waited: boolean }>(
+		fn: () => Promise<T>,
+	): Promise<T> {
+		return this.tracedPhase("lock_wait", fn, (r) => r.waited, {
+			child: "waited",
+		});
+	}
+
+	public lockRecheck<T extends { readonly outcome: CliVersionCheckOutcome }>(
+		fn: () => Promise<T>,
+	): Promise<T> {
+		return this.tracedPhase("lock_wait_recheck", fn, (r) => r.outcome, {
+			child: "outcome",
+		});
+	}
+
+	public setDownloadDecision(
+		reason: CliDownloadReason,
+		action: CliDownloadAction,
+	): void {
+		this.span.setProperty("download_reason", reason);
+		this.span.setProperty("download_action", action);
+	}
+
+	public async fallback<T>(error: unknown, fn: () => Promise<T>): Promise<T> {
+		// Why we fell back, recorded even when the fallback succeeds.
+		this.span.setProperty("fallback_reason", categorizeResolveError(error));
+		const result = await this.span.phase(
+			"fallback_to_existing_binary",
+			async (child) => {
+				try {
+					return await fn();
+				} catch (fallbackError) {
+					// Fallback failed too: tag the phase and parent with its error.type.
+					const category = categorizeResolveError(fallbackError);
+					recordError(child, category);
+					this.error(category);
+					throw fallbackError;
+				}
+			},
+		);
+		this.setOutcome("fallback_to_existing_binary");
+		return result;
+	}
+
+	/**
+	 * Run `fn` as a child phase tagged with `select(result)`, mirroring it onto
+	 * the parent when `keys.parent` is given.
+	 */
+	private async tracedPhase<T>(
+		name: string,
+		fn: () => Promise<T>,
+		select: (result: T) => CallerPropertyValue,
+		keys: { readonly child: string; readonly parent?: string },
+	): Promise<T> {
+		const result = await this.span.phase(name, async (child) => {
+			const value = await fn();
+			child.setProperty(keys.child, select(value));
+			return value;
+		});
+		if (keys.parent) {
+			this.span.setProperty(keys.parent, select(result));
+		}
+		return result;
+	}
+}
+
+export class CliConfigureTrace {
+	public constructor(private readonly span: Span) {}
+
+	public abort(): void {
+		recordAborted(this.span, "credential_store");
+	}
+
+	public error(error: unknown): void {
+		recordError(this.span, categorizeConfigureError(error));
+	}
+}
+
+function categorizeConfigureError(error: unknown): CliConfigureErrorType {
+	// A CredentialFileError is a file-write failure; anything else is keyring/CLI.
+	if (error instanceof CredentialFileError) {
+		return "filesystem";
+	}
+	return error instanceof Error ? "credential_store" : "unknown";
+}
+
+function categorizeResolveError(error: unknown): CliResolveErrorType {
+	if (error instanceof CliDownloadsDisabledError) {
+		return "downloads_disabled";
+	}
+	if (error instanceof CliFallbackDeclinedError) {
+		return "fallback_declined";
+	}
+	return "download";
+}

src/instrumentation/cli.ts

@@ -1,16 +1,23 @@
 import type { TelemetryService } from "../telemetry/service";
 
 export type RemoteSetupPhase =
+	| "cli_resolve"
+	| "cli_configure"
+	| "compatibility_check"
 	| "workspace_lookup"
+	| "workspace_monitor_setup"
 	| "workspace_ready"
-	| "resolve_agent"
-	| "ssh_config_write";
+	| "agent_resolve"
+	| "ssh_config_write"
+	| "ssh_monitor_setup"
+	| "connection_handoff";
 
 /** Reason for a non-throwing early exit from `remote.setup`. */
 export type RemoteSetupOutcome = "workspace_not_found" | "incompatible_server";
 
 /** Helpers scoped to the remote.setup trace's lifetime. */
 export interface RemoteSetupTracer {
+	/** Emit a typed child phase of `remote.setup`. */
 	phase<T>(name: RemoteSetupPhase, fn: () => T | PromiseLike<T>): Promise<T>;
 	/** Mark this setup as aborted with a typed reason; emits as `outcome` on the parent event. */
 	markAborted(reason: RemoteSetupOutcome): void;

src/instrumentation/remoteSetup.ts

@@ -39,6 +39,7 @@ interface ReconnectCycle {
 	readonly startMs: number;
 	readonly reason: ConnectionStateReason;
 	attempts: number;
+	maxBackoffMs: number;
 }
 
 interface DropOptions {
@@ -85,7 +86,7 @@ export class WebSocketTelemetry {
 		this.#telemetry.log(
 			"connection.opened",
 			{ route: normalizeRoute(route) },
-			{ connectDurationMs: now - start },
+			{ connect_duration_ms: now - start },
 		);
 		this.#finishReconnect({ result: "success" });
 	}
@@ -104,10 +105,10 @@ export class WebSocketTelemetry {
 
 		const properties: CallerProperties = { cause };
 		if (closeCode !== undefined) {
-			properties.closeCode = closeCode;
+			properties.close_code = closeCode;
 		}
 		const measurements = {
-			connectionDurationMs: performance.now() - openedAtMs,
+			connection_duration_ms: performance.now() - openedAtMs,
 		};
 		if (error === undefined) {
 			this.#telemetry.log("connection.dropped", properties, measurements);
@@ -136,6 +137,7 @@ export class WebSocketTelemetry {
 			startMs: performance.now(),
 			reason,
 			attempts: 0,
+			maxBackoffMs: 0,
 		};
 	}
 
@@ -146,9 +148,17 @@ export class WebSocketTelemetry {
 	}
 
 	/** Drop and (re)open a reconnect cycle. */
-	public retrying(reason: ConnectionStateReason, options: DropOptions): void {
+	public retrying(
+		reason: ConnectionStateReason,
+		options: DropOptions,
+		backoffMs: number,
+	): void {
 		this.dropped(options.cause, options.code, options.error);
 		this.reconnectStarted(reason);
+		const cycle = this.#reconnectCycle;
+		if (cycle) {
+			cycle.maxBackoffMs = Math.max(cycle.maxBackoffMs, backoffMs);
+		}
 	}
 
 	#finishReconnect(outcome: ReconnectOutcome): void {
@@ -163,11 +173,12 @@ export class WebSocketTelemetry {
 			reason: cycle.reason,
 		};
 		if (outcome.result === "error") {
-			properties.terminationReason = outcome.terminationReason;
+			properties.termination_reason = outcome.terminationReason;
 		}
 		this.#telemetry.log("connection.reconnect_resolved", properties, {
 			attempts: cycle.attempts,
-			totalDurationMs: performance.now() - cycle.startMs,
+			max_backoff_ms: cycle.maxBackoffMs,
+			total_duration_ms: performance.now() - cycle.startMs,
 		});
 	}
 }