feat(telemetry): add wire-format module and export event reader (#959)

Foundation for the telemetry export flow (#903): a single source of truth for the on-disk JSONL shape, a streaming reader for exported events, and the date-range primitives the export UI will build on.

- Add `wireFormat.ts` with zod-derived `TelemetryEvent` (DeepReadonly) and shared `serializeTelemetryEventLine` / `parseTelemetryEventLine` so writer and reader cannot drift. `TelemetryFileParseError` lets readers tell parse failures apart from IO failures.
- Add `export/files.ts`: `listTelemetryFilesForRange` (date-filtered, chronologically ordered) and `streamTelemetryEvents` (line-by-line `AsyncIterable` with deterministic fd cleanup).
- Add `export/range.ts`: preset and custom UTC ranges, UTC date validation, and a calendar-day pre-filter so out-of-range files are skipped without being opened.
- Switch `LocalJsonlSink` to the shared serializer; drop the bespoke writer.

Refs #903

Author: EhabY

src/telemetry/event.ts

@@ -3,6 +3,14 @@ import * as vscode from "vscode";
 
 import { toError } from "../error/errorUtils";
 
+export type {
+	SessionContext,
+	TelemetryContext,
+	TelemetryEvent,
+} from "./wireFormat";
+
+import type { SessionContext, TelemetryEvent } from "./wireFormat";
+
 /** Telemetry level, mirrors `coder.telemetry.level`. Ordered: off < local. */
 export type TelemetryLevel = "off" | "local";
 
@@ -19,47 +27,6 @@ export type CallerMeasurements = Record<string, number> & {
 	durationMs?: never;
 };
 
-/** Session-stable resource attributes. Field names are inspired by OTel
- * resource attributes; they are camelCase TypeScript and not a 1:1 mapping. */
-export interface SessionContext {
-	readonly extensionVersion: string;
-	readonly machineId: string;
-	readonly sessionId: string;
-	readonly osType: string;
-	readonly osVersion: string;
-	readonly hostArch: string;
-	readonly platformName: string;
-	readonly platformVersion: string;
-}
-
-/** Per-event context: session attributes plus the current deployment URL. */
-export interface TelemetryContext extends SessionContext {
-	readonly deploymentUrl: string;
-}
-
-export interface TelemetryEvent {
-	readonly eventId: string;
-	readonly eventName: string;
-	readonly timestamp: string;
-	readonly eventSequence: number;
-
-	readonly context: TelemetryContext;
-
-	readonly properties: Readonly<Record<string, string>>;
-	readonly measurements: Readonly<Record<string, number>>;
-
-	/** Shared by all events in a trace. Maps to OTel `trace_id`. */
-	readonly traceId?: string;
-	/** Set on phase children only. Equals the parent event's `eventId`. Maps to OTel `parent_span_id`. */
-	readonly parentEventId?: string;
-
-	readonly error?: Readonly<{
-		message: string;
-		type?: string;
-		code?: string;
-	}>;
-}
-
 /**
  * Sink for telemetry events. `write` is sync and must buffer in memory; I/O
  * happens in `flush`/`dispose`. The service filters by `minLevel`; sinks can

src/telemetry/export/files.ts

@@ -0,0 +1,128 @@
+import { createReadStream } from "node:fs";
+import * as fs from "node:fs/promises";
+import * as path from "node:path";
+import * as readline from "node:readline";
+
+import { toError } from "../../error/errorUtils";
+import {
+	parseTelemetryEventLine,
+	TelemetryFileParseError,
+} from "../wireFormat";
+
+import {
+	fileDateCanContainRangeEvent,
+	isTimestampInRange,
+	type TelemetryDateRange,
+} from "./range";
+
+import type { TelemetryEvent } from "../event";
+
+interface TelemetryLogFile {
+	readonly path: string;
+	readonly date: string;
+	readonly session: string;
+	readonly part: number;
+}
+
+/**
+ * Filename shape written by the sink:
+ * `telemetry-YYYY-MM-DD-{session}[.{part}].jsonl`. We need the date to filter
+ * and (session, part) to order files within a day.
+ */
+const TELEMETRY_FILE_PATTERN =
+	/^telemetry-(\d{4}-\d{2}-\d{2})-([^.]+)(?:\.(\d+))?\.jsonl$/;
+
+/** Log files that could contain events in `range`, in chronological order. */
+export async function listTelemetryFilesForRange(
+	telemetryDir: string,
+	range: TelemetryDateRange,
+): Promise<string[]> {
+	let names: string[];
+	try {
+		names = await fs.readdir(telemetryDir);
+	} catch (err) {
+		if ((err as NodeJS.ErrnoException).code === "ENOENT") {
+			return [];
+		}
+		throw err;
+	}
+
+	return names
+		.map((name) => parseLogFilename(telemetryDir, name))
+		.filter(
+			(file): file is TelemetryLogFile =>
+				file !== undefined && fileDateCanContainRangeEvent(file.date, range),
+		)
+		.sort(compareLogFiles)
+		.map(({ path: filePath }) => filePath);
+}
+
+/**
+ * Yields events from `filePaths` in order, keeping only those whose timestamp
+ * falls inside `range`. Reads line-by-line so memory stays flat on big files.
+ */
+export async function* streamTelemetryEvents(
+	filePaths: readonly string[],
+	range: TelemetryDateRange,
+): AsyncIterable<TelemetryEvent> {
+	for (const filePath of filePaths) {
+		const name = path.basename(filePath);
+		const stream = createReadStream(filePath, { encoding: "utf8" });
+		const lines = readline.createInterface({
+			input: stream,
+			crlfDelay: Infinity,
+		});
+		let lineNumber = 0;
+		try {
+			for await (const line of lines) {
+				lineNumber += 1;
+				if (line.trim() === "") {
+					continue;
+				}
+				const event = parseTelemetryEventLine(line, name, lineNumber);
+				if (isTimestampInRange(event.timestamp, range)) {
+					yield event;
+				}
+			}
+		} catch (err) {
+			if (err instanceof TelemetryFileParseError) {
+				throw err;
+			}
+			const at = lineNumber > 0 ? `:${lineNumber}` : "";
+			throw new Error(
+				`Failed to read telemetry file ${name}${at}: ${toError(err).message}`,
+				{ cause: err },
+			);
+		} finally {
+			try {
+				lines.close();
+			} finally {
+				stream.destroy();
+			}
+		}
+	}
+}
+
+function parseLogFilename(
+	dir: string,
+	name: string,
+): TelemetryLogFile | undefined {
+	const match = TELEMETRY_FILE_PATTERN.exec(name);
+	if (!match) {
+		return undefined;
+	}
+	return {
+		path: path.join(dir, name),
+		date: match[1],
+		session: match[2],
+		part: match[3] === undefined ? 0 : Number(match[3]),
+	};
+}
+
+function compareLogFiles(a: TelemetryLogFile, b: TelemetryLogFile): number {
+	return (
+		a.date.localeCompare(b.date) ||
+		a.session.localeCompare(b.session) ||
+		a.part - b.part
+	);
+}

src/telemetry/export/range.ts

@@ -0,0 +1,149 @@
+import { z } from "zod";
+
+const DAY_MS = 24 * 60 * 60 * 1000;
+const UtcDateSchema = z.iso.date();
+
+/**
+ * Half-open UTC window `[startMs, endMs)` used to filter telemetry. Either
+ * bound may be `undefined` (e.g. "all time"). `label` is for the UI and
+ * `filenamePart` is for export filenames.
+ */
+export interface TelemetryDateRange {
+	readonly label: string;
+	readonly filenamePart: string;
+	readonly startMs?: number;
+	readonly endMs?: number;
+}
+
+export interface TelemetryRangePreset {
+	readonly id: TelemetryRangePresetId;
+	readonly label: string;
+	readonly detail: string;
+}
+
+export type TelemetryRangePresetId = keyof typeof PRESETS;
+
+const PRESETS = {
+	last24Hours: {
+		label: "Last 24 hours",
+		detail: "Export telemetry from the last day.",
+		filenamePart: "last-24-hours",
+		durationMs: DAY_MS,
+	},
+	last7Days: {
+		label: "Last 7 days",
+		detail: "Export telemetry from the last week.",
+		filenamePart: "last-7-days",
+		durationMs: 7 * DAY_MS,
+	},
+	last30Days: {
+		label: "Last 30 days",
+		detail: "Export telemetry from the last month.",
+		filenamePart: "last-30-days",
+		durationMs: 30 * DAY_MS,
+	},
+	allTime: {
+		label: "All time",
+		detail: "Export all stored telemetry.",
+		filenamePart: "all-time",
+		durationMs: undefined,
+	},
+} as const;
+
+/** Presets the export UI shows, in display order. */
+export const TELEMETRY_RANGE_PRESETS: readonly TelemetryRangePreset[] =
+	Object.entries(PRESETS).map(([id, p]) => ({
+		id: id as TelemetryRangePresetId,
+		label: p.label,
+		detail: p.detail,
+	}));
+
+/** Range from a preset id, anchored at `now`. */
+export function createPresetDateRange(
+	id: TelemetryRangePresetId,
+	now: Date = new Date(),
+): TelemetryDateRange {
+	const { label, filenamePart, durationMs } = PRESETS[id];
+	if (durationMs === undefined) {
+		return { label, filenamePart };
+	}
+	const endMs = now.getTime();
+	return { label, filenamePart, startMs: endMs - durationMs, endMs };
+}
+
+/**
+ * UTC range that includes the full 24h of `endDate`; `endMs` lands at
+ * exclusive midnight of the day after.
+ */
+export function createCustomDateRange(
+	startDate: string,
+	endDate: string,
+): TelemetryDateRange {
+	const startDateMs = parseUtcDate(startDate);
+	const endDateMs = parseUtcDate(endDate);
+	if (endDateMs < startDateMs) {
+		throw new Error("End date must be on or after start date.");
+	}
+	return {
+		label: `${startDate} to ${endDate}`,
+		filenamePart: `${startDate}_to_${endDate}`,
+		startMs: startDateMs,
+		endMs: endDateMs + DAY_MS,
+	};
+}
+
+/** User-facing error string if `value` isn't a UTC date, else `undefined`. */
+export function validateUtcDateInput(value: string): string | undefined {
+	if (!/^\d{4}-\d{2}-\d{2}$/.test(value)) {
+		return "Use YYYY-MM-DD.";
+	}
+	return UtcDateSchema.safeParse(value).success
+		? undefined
+		: "Enter a valid calendar date.";
+}
+
+/** True if the ISO `timestamp` falls inside the range. */
+export function isTimestampInRange(
+	timestamp: string,
+	range: TelemetryDateRange,
+): boolean {
+	const ms = Date.parse(timestamp);
+	if (!Number.isFinite(ms)) {
+		throw new Error(`Invalid telemetry timestamp '${timestamp}'.`);
+	}
+	return (
+		(range.startMs === undefined || ms >= range.startMs) &&
+		(range.endMs === undefined || ms < range.endMs)
+	);
+}
+
+/**
+ * Coarse calendar-day filter: could a file dated `date` (YYYY-MM-DD) hold any
+ * event in `range`? Lets us skip files without reading them.
+ */
+export function fileDateCanContainRangeEvent(
+	date: string,
+	range: TelemetryDateRange,
+): boolean {
+	const startDate =
+		range.startMs === undefined ? undefined : utcDateString(range.startMs);
+	const endDate =
+		range.endMs === undefined ? undefined : utcDateString(range.endMs - 1);
+	return (
+		(startDate === undefined || date >= startDate) &&
+		(endDate === undefined || date <= endDate)
+	);
+}
+
+function parseUtcDate(value: string): number {
+	const error = validateUtcDateInput(value);
+	if (error !== undefined) {
+		throw new Error(`Invalid date '${value}': ${error}`);
+	}
+	const [y, m, d] = value.split("-").map(Number);
+	return Date.UTC(y, m - 1, d);
+}
+
+function utcDateString(ms: number): string {
+	return new Date(ms).toISOString().slice(0, 10);
+}

src/telemetry/sinks/localJsonlSink.ts

@@ -12,6 +12,7 @@ import {
 	cleanupFiles,
 	type FileCleanupCandidate,
 } from "../../util/fileCleanup";
+import { serializeTelemetryEventLine } from "../wireFormat";
 
 import type { Logger } from "../../logging/logger";
 import type { TelemetryEvent, TelemetryLevel, TelemetrySink } from "../event";
@@ -88,7 +89,7 @@ export class LocalJsonlSink implements TelemetrySink, vscode.Disposable {
 		}
 		let line: string;
 		try {
-			line = serializeEvent(event);
+			line = serializeTelemetryEventLine(event);
 		} catch (err) {
 			this.#logger.warn(`Telemetry sink '${this.name}' serialize failed`, err);
 			return;
@@ -313,32 +314,3 @@ function toSessionSlug(sessionId: string): string {
 	const cleaned = sessionId.replace(/[^a-zA-Z0-9]/g, "");
 	return cleaned.slice(0, 8) || "anon0000";
 }
-
-function serializeEvent(event: TelemetryEvent): string {
-	return (
-		JSON.stringify({
-			event_id: event.eventId,
-			event_name: event.eventName,
-			timestamp: event.timestamp,
-			event_sequence: event.eventSequence,
-			context: {
-				extension_version: event.context.extensionVersion,
-				machine_id: event.context.machineId,
-				session_id: event.context.sessionId,
-				os_type: event.context.osType,
-				os_version: event.context.osVersion,
-				host_arch: event.context.hostArch,
-				platform_name: event.context.platformName,
-				platform_version: event.context.platformVersion,
-				deployment_url: event.context.deploymentUrl,
-			},
-			properties: event.properties,
-			measurements: event.measurements,
-			...(event.traceId !== undefined && { trace_id: event.traceId }),
-			...(event.parentEventId !== undefined && {
-				parent_event_id: event.parentEventId,
-			}),
-			...(event.error !== undefined && { error: event.error }),
-		}) + "\n"
-	);
-}