From c79135debd85f03cd5fde341e3c0b563d5fdcac9 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Wed, 8 Jul 2026 21:41:49 +0530 Subject: [PATCH 01/10] =?UTF-8?q?feat(editor):=20keystroke=20overlay=20fou?= =?UTF-8?q?ndation=20=E2=80=94=20capture=20+=20display=20model?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the capture and display-logic foundation for an on-screen keystroke overlay (typed keys / shortcut chords during playback + export — a common tutorial/demo need; fits "additional editor tools and workflow polish"). This is part 1 of 2 and is intentionally inert: the feature flag defaults OFF and the renderer/settings land in a follow-up, so there is no user-facing change and no risk to existing recording/edit/export flows. Capture (main process), privacy-gated: - Listen for keydown on the existing uiohook interaction hook; record {timeMs, key, modifiers} into a bounded keystroke telemetry buffer. - Resolve keycodes to stable semantic tokens via uiohook's UiohookKey table so the renderer never handles raw platform keycodes. - Collapse auto-repeat; gate ALL storage on isKeystrokeCaptureActive (default OFF — a global key hook can observe secrets typed while recording). Display-logic core (pure, renderer), fully unit-tested: - keyLabels: token -> glyph, platform-aware modifiers (Cmd/Opt/Shift/Ctrl vs Ctrl/Alt/Win/Super). - keystrokeCoalescing: chord grouping, rapid-key merging, fade/TTL, and progressive reveal for scrub-accurate editor replay. Frame-rate independent. Tests: 24 new unit tests (incl. a fast-check property test); existing cursor interaction/telemetry tests still pass. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/ipc/constants.ts | 3 + electron/ipc/cursor/interaction.ts | 88 +++++++++-- electron/ipc/cursor/telemetry.ts | 45 ++++-- electron/ipc/state.ts | 17 ++ electron/ipc/types.ts | 38 ++++- .../keystrokeOverlay/keyLabels.test.ts | 81 ++++++++++ .../keystrokeOverlay/keyLabels.ts | 148 ++++++++++++++++++ .../keystrokeCoalescing.test.ts | 112 +++++++++++++ .../keystrokeOverlay/keystrokeCoalescing.ts | 102 ++++++++++++ .../keystrokeOverlay/keystrokeTypes.ts | 59 +++++++ 10 files changed, 670 insertions(+), 23 deletions(-) create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.test.ts create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts diff --git a/electron/ipc/constants.ts b/electron/ipc/constants.ts index 2c5cf8f17..c92a8cb09 100644 --- a/electron/ipc/constants.ts +++ b/electron/ipc/constants.ts @@ -29,3 +29,6 @@ export const COMPANION_AUDIO_LAYOUTS = [ export const CURSOR_TELEMETRY_VERSION = 2; export const CURSOR_SAMPLE_INTERVAL_MS = 33; export const MAX_CURSOR_SAMPLES = 60 * 60 * 30; // 1 hour @ 30Hz + +export const KEYSTROKE_TELEMETRY_VERSION = 1; +export const MAX_KEYSTROKE_SAMPLES = 60 * 60 * 20; // generous cap; keystrokes are sparse vs cursor samples diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index 9b6f3ac92..ffddc9a83 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -1,23 +1,31 @@ import fs from "node:fs"; import { createRequire } from "node:module"; import path from "node:path"; -import type { HookMouseEvent, UiohookLike, UiohookModuleNamespace, CursorInteractionType } from "../types"; import { - isCursorCaptureActive, - interactionCaptureCleanup, - setInteractionCaptureCleanup, hasLoggedInteractionHookFailure, - setHasLoggedInteractionHookFailure, + interactionCaptureCleanup, + isCursorCaptureActive, + isKeystrokeCaptureActive, lastLeftClick, + setHasLoggedInteractionHookFailure, + setInteractionCaptureCleanup, setLastLeftClick, setLinuxCursorScreenPoint, } from "../state"; +import type { + CursorInteractionType, + HookKeyboardEvent, + HookMouseEvent, + UiohookLike, + UiohookModuleNamespace, +} from "../types"; import { - getNormalizedCursorPoint, getCursorCaptureElapsedMs, getHookCursorScreenPoint, + getNormalizedCursorPoint, isCursorCapturePaused, pushCursorSample, + pushKeystrokeSample, } from "./telemetry"; const nodeRequire = createRequire(import.meta.url); @@ -172,6 +180,31 @@ function loadUiohookModule() { } } +/** + * Build a keycode → semantic-token map from uiohook-napi's UiohookKey table so + * the renderer never has to know raw platform keycodes. Tolerant of the table + * being absent — callers fall back to the numeric keycode string. + */ +function buildKeycodeTokenMap(): Map { + const map = new Map(); + try { + const moduleExports = nodeRequire("uiohook-napi") as { + UiohookKey?: Record; + }; + const table = moduleExports?.UiohookKey; + if (table) { + for (const [name, code] of Object.entries(table)) { + if (typeof code === "number" && !map.has(code)) { + map.set(code, name); + } + } + } + } catch { + // Table unavailable — renderer falls back to the numeric keycode string. + } + return map; +} + export async function startInteractionCapture() { if (!isCursorCaptureActive) { return; @@ -255,11 +288,7 @@ export async function startInteractionCapture() { }; const onMouseMove = (event: HookMouseEvent) => { - if ( - process.platform !== "linux" || - !isCursorCaptureActive || - isCursorCapturePaused() - ) { + if (process.platform !== "linux" || !isCursorCaptureActive || isCursorCapturePaused()) { return; } @@ -271,8 +300,43 @@ export async function startInteractionCapture() { setLinuxCursorScreenPoint({ x: point.x, y: point.y, updatedAt: Date.now() }); }; + const keycodeTokens = buildKeycodeTokenMap(); + let lastKeystrokeCode = -1; + let lastKeystrokeTimeMs = Number.NEGATIVE_INFINITY; + + const onKeyDown = (event: HookKeyboardEvent) => { + // Privacy gate: only record keystrokes when the overlay is enabled. + if (!isCursorCaptureActive || !isKeystrokeCaptureActive || isCursorCapturePaused()) { + return; + } + + const keycode = typeof event.keycode === "number" ? event.keycode : event.data?.keycode; + if (typeof keycode !== "number") { + return; + } + + const timeMs = getCursorCaptureElapsedMs(); + + // Collapse auto-repeat: ignore the same physical key re-firing rapidly. + if (keycode === lastKeystrokeCode && timeMs - lastKeystrokeTimeMs < 45) { + return; + } + lastKeystrokeCode = keycode; + lastKeystrokeTimeMs = timeMs; + + pushKeystrokeSample({ + timeMs, + key: keycodeTokens.get(keycode) ?? String(keycode), + ctrl: Boolean(event.ctrlKey ?? event.data?.ctrlKey), + alt: Boolean(event.altKey ?? event.data?.altKey), + shift: Boolean(event.shiftKey ?? event.data?.shiftKey), + meta: Boolean(event.metaKey ?? event.data?.metaKey), + }); + }; + hook.on("mousedown", onMouseDown); hook.on("mouseup", onMouseUp); + hook.on("keydown", onKeyDown); if (process.platform === "linux") { hook.on("mousemove", onMouseMove); } @@ -282,12 +346,14 @@ export async function startInteractionCapture() { if (typeof hook.off === "function") { hook.off("mousedown", onMouseDown); hook.off("mouseup", onMouseUp); + hook.off("keydown", onKeyDown); if (process.platform === "linux") { hook.off("mousemove", onMouseMove); } } else if (typeof hook.removeListener === "function") { hook.removeListener("mousedown", onMouseDown); hook.removeListener("mouseup", onMouseUp); + hook.removeListener("keydown", onKeyDown); if (process.platform === "linux") { hook.removeListener("mousemove", onMouseMove); } diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index ebedfe72a..b8695be04 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -3,9 +3,11 @@ import { CURSOR_SAMPLE_INTERVAL_MS, CURSOR_TELEMETRY_VERSION, MAX_CURSOR_SAMPLES, + MAX_KEYSTROKE_SAMPLES, } from "../constants"; import { activeCursorSamples, + activeKeystrokeSamples, currentCursorVisualType, cursorCaptureAccumulatedPausedMs, cursorCaptureInterval, @@ -22,7 +24,12 @@ import { setCursorCapturePauseStartedAtMs, setPendingCursorSamples, } from "../state"; -import type { CursorInteractionType, CursorTelemetryPoint, CursorVisualType } from "../types"; +import type { + CursorInteractionType, + CursorTelemetryPoint, + CursorVisualType, + KeystrokeTelemetryPoint, +} from "../types"; import { getScreen, getTelemetryPathForVideo } from "../utils"; export function clamp(value: number, min: number, max: number) { @@ -91,11 +98,7 @@ export async function writeCursorTelemetry(videoPath: string, samples: unknown) await fs.writeFile( telemetryPath, - JSON.stringify( - { version: CURSOR_TELEMETRY_VERSION, samples: normalizedSamples }, - null, - 2, - ), + JSON.stringify({ version: CURSOR_TELEMETRY_VERSION, samples: normalizedSamples }, null, 2), "utf-8", ); @@ -144,9 +147,7 @@ export function resumeCursorCapture(resumedAtMs: number) { } const pauseDurationMs = Math.max(0, resumedAtMs - cursorCapturePauseStartedAtMs); - setCursorCaptureAccumulatedPausedMs( - cursorCaptureAccumulatedPausedMs + pauseDurationMs, - ); + setCursorCaptureAccumulatedPausedMs(cursorCaptureAccumulatedPausedMs + pauseDurationMs); setCursorCapturePauseStartedAtMs(null); } @@ -217,7 +218,16 @@ export function getNormalizedCursorPoint() { } export function getHookCursorScreenPoint( - event: { x?: number; y?: number; data?: { x?: number; y?: number; screenX?: number; screenY?: number }; screenX?: number; screenY?: number } | null | undefined, + event: + | { + x?: number; + y?: number; + data?: { x?: number; y?: number; screenX?: number; screenY?: number }; + screenX?: number; + screenY?: number; + } + | null + | undefined, ): { x: number; y: number } | null { const rawX = event?.x ?? event?.data?.x ?? event?.screenX ?? event?.data?.screenX; const rawY = event?.y ?? event?.data?.y ?? event?.screenY ?? event?.data?.screenY; @@ -254,6 +264,21 @@ export function pushCursorSample( } } +export function pushKeystrokeSample(sample: KeystrokeTelemetryPoint) { + activeKeystrokeSamples.push({ + timeMs: Math.max(0, sample.timeMs), + key: sample.key, + ctrl: sample.ctrl, + alt: sample.alt, + shift: sample.shift, + meta: sample.meta, + }); + + if (activeKeystrokeSamples.length > MAX_KEYSTROKE_SAMPLES) { + activeKeystrokeSamples.shift(); + } +} + export function sampleCursorPoint() { const point = getNormalizedCursorPoint(); pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "move"); diff --git a/electron/ipc/state.ts b/electron/ipc/state.ts index a0a41744e..cf8cb9527 100644 --- a/electron/ipc/state.ts +++ b/electron/ipc/state.ts @@ -3,6 +3,7 @@ import type { CursorInteractionType, CursorTelemetryPoint, CursorVisualType, + KeystrokeTelemetryPoint, NativeCaptureDiagnostics, RecordingSessionData, SelectedSource, @@ -82,6 +83,13 @@ export let activeCursorSamples: CursorTelemetryPoint[] = []; export let pendingCursorSamples: CursorTelemetryPoint[] = []; export let isCursorCaptureActive = false; export let interactionCaptureCleanup: (() => void) | null = null; + +// ── Keystroke telemetry (opt-in; privacy-gated) ────────────────────────────── +// Disabled by default: a global key hook can observe secrets typed while +// recording, so nothing is stored unless the keystroke overlay is enabled. +export let isKeystrokeCaptureActive = false; +export let activeKeystrokeSamples: KeystrokeTelemetryPoint[] = []; +export let pendingKeystrokeSamples: KeystrokeTelemetryPoint[] = []; export let hasLoggedInteractionHookFailure = false; export let lastLeftClick: { timeMs: number; cx: number; cy: number } | null = null; export let linuxCursorScreenPoint: { x: number; y: number; updatedAt: number } | null = null; @@ -254,6 +262,15 @@ export function setPendingCursorSamples(v: CursorTelemetryPoint[]) { export function setIsCursorCaptureActive(v: boolean) { isCursorCaptureActive = v; } +export function setIsKeystrokeCaptureActive(v: boolean) { + isKeystrokeCaptureActive = v; +} +export function setActiveKeystrokeSamples(v: KeystrokeTelemetryPoint[]) { + activeKeystrokeSamples = v; +} +export function setPendingKeystrokeSamples(v: KeystrokeTelemetryPoint[]) { + pendingKeystrokeSamples = v; +} export function setInteractionCaptureCleanup(v: (() => void) | null) { interactionCaptureCleanup = v; } diff --git a/electron/ipc/types.ts b/electron/ipc/types.ts index 58f5425bd..9ce2d751c 100644 --- a/electron/ipc/types.ts +++ b/electron/ipc/types.ts @@ -106,6 +106,21 @@ export interface CursorTelemetryPoint { cursorType?: CursorVisualType; } +/** + * A single captured keystroke. `key` is a stable semantic token resolved from + * uiohook-napi's UiohookKey table (e.g. "A", "Enter", "Space", "Comma"), or the + * raw numeric keycode as a string when the token is unknown. Modifier booleans + * describe which modifiers were held at press time. + */ +export interface KeystrokeTelemetryPoint { + timeMs: number; + key: string; + ctrl?: boolean; + alt?: boolean; + shift?: boolean; + meta?: boolean; +} + export type NativeMacWindowSource = { id: string; name: string; @@ -120,7 +135,7 @@ export type NativeMacWindowSource = { height?: number; }; -export type HookEventName = "mousedown" | "mouseup" | "mousemove"; +export type HookEventName = "mousedown" | "mouseup" | "mousemove" | "keydown" | "keyup"; export type HookMouseEvent = { button?: number; @@ -139,7 +154,26 @@ export type HookMouseEvent = { }; }; -export type HookEventListener = (event: HookMouseEvent) => void; +export type HookKeyboardEvent = { + keycode?: number; + rawcode?: number; + altKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; + data?: { + keycode?: number; + altKey?: boolean; + ctrlKey?: boolean; + metaKey?: boolean; + shiftKey?: boolean; + }; +}; + +/** Union of the mouse and keyboard event shapes a single uiohook listener may receive. */ +export type HookInputEvent = HookMouseEvent & HookKeyboardEvent; + +export type HookEventListener = (event: HookInputEvent) => void; export type UiohookLike = { on: (eventName: HookEventName, listener: HookEventListener) => void; diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.test.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.test.ts new file mode 100644 index 000000000..e2524084a --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.test.ts @@ -0,0 +1,81 @@ +import { describe, expect, it } from "vitest"; +import { formatModifierLabels, isModifierToken, keyTokenToLabel } from "./keyLabels"; + +describe("keyTokenToLabel", () => { + it("uppercases single letters", () => { + expect(keyTokenToLabel("a")).toBe("A"); + expect(keyTokenToLabel("A")).toBe("A"); + }); + + it("maps digit tokens across spellings", () => { + expect(keyTokenToLabel("1")).toBe("1"); + expect(keyTokenToLabel("Num1")).toBe("1"); + expect(keyTokenToLabel("Digit1")).toBe("1"); + expect(keyTokenToLabel("Numpad9")).toBe("9"); + }); + + it("maps named keys to glyphs", () => { + expect(keyTokenToLabel("Enter")).toBe("↵"); + expect(keyTokenToLabel("Backspace")).toBe("⌫"); + expect(keyTokenToLabel("Tab")).toBe("⇥"); + expect(keyTokenToLabel("Escape")).toBe("Esc"); + }); + + it("maps arrows to arrow glyphs regardless of spelling", () => { + expect(keyTokenToLabel("Up")).toBe("↑"); + expect(keyTokenToLabel("ArrowUp")).toBe("↑"); + expect(keyTokenToLabel("Right")).toBe("→"); + }); + + it("maps punctuation tokens", () => { + expect(keyTokenToLabel("Comma")).toBe(","); + expect(keyTokenToLabel("Slash")).toBe("/"); + expect(keyTokenToLabel("BracketLeft")).toBe("["); + }); + + it("passes function keys through", () => { + expect(keyTokenToLabel("F5")).toBe("F5"); + expect(keyTokenToLabel("F12")).toBe("F12"); + }); + + it("falls back to the token itself for unknown keys", () => { + expect(keyTokenToLabel("187")).toBe("187"); + expect(keyTokenToLabel("")).toBe(""); + }); +}); + +describe("formatModifierLabels", () => { + it("uses mac glyphs in Ctrl/Alt/Shift/Meta order", () => { + expect( + formatModifierLabels({ ctrl: true, alt: true, shift: true, meta: true }, "mac"), + ).toEqual(["⌃", "⌥", "⇧", "⌘"]); + }); + + it("uses text labels on windows, with Win for meta", () => { + expect(formatModifierLabels({ ctrl: true, meta: true }, "windows")).toEqual([ + "Ctrl", + "Win", + ]); + }); + + it("uses Super for meta on linux", () => { + expect(formatModifierLabels({ meta: true }, "linux")).toEqual(["Super"]); + }); + + it("returns nothing when no modifiers are held", () => { + expect(formatModifierLabels({}, "mac")).toEqual([]); + }); +}); + +describe("isModifierToken", () => { + it("flags modifier and lock keys", () => { + expect(isModifierToken("Shift")).toBe(true); + expect(isModifierToken("MetaRight")).toBe(true); + expect(isModifierToken("CapsLock")).toBe(true); + }); + + it("does not flag ordinary keys", () => { + expect(isModifierToken("A")).toBe(false); + expect(isModifierToken("Enter")).toBe(false); + }); +}); diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts new file mode 100644 index 000000000..df3eab0f7 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts @@ -0,0 +1,148 @@ +// Keystroke overlay — token → display label mapping. +// +// Pure functions only. Given a semantic key token (from the capture layer) and +// the current platform glyph style, produce the human-facing keycap label. + +import type { KeystrokeEvent, ModifierGlyphStyle } from "./keystrokeTypes"; + +/** + * Tokens that represent a modifier or lock key on their own. A lone press of + * one of these should NOT produce a keycap — modifiers only appear as part of a + * chord (see coalescing). + */ +const MODIFIER_TOKENS = new Set([ + "Ctrl", + "Control", + "ControlLeft", + "ControlRight", + "CtrlRight", + "Alt", + "AltLeft", + "AltRight", + "AltGr", + "Shift", + "ShiftLeft", + "ShiftRight", + "Meta", + "MetaLeft", + "MetaRight", + "Cmd", + "Command", + "Super", + "Win", + "CapsLock", + "NumLock", + "ScrollLock", + "Fn", +]); + +export function isModifierToken(token: string): boolean { + return MODIFIER_TOKENS.has(token); +} + +/** + * Ordered modifier labels present on an event. Order is Ctrl, Alt, Shift, Meta — + * matching the conventional left-to-right reading order for shortcut chords. + */ +export function formatModifierLabels( + event: Pick, + style: ModifierGlyphStyle, +): string[] { + const isMac = style === "mac"; + const labels: string[] = []; + + if (event.ctrl) { + labels.push(isMac ? "⌃" : "Ctrl"); + } + if (event.alt) { + labels.push(isMac ? "⌥" : "Alt"); + } + if (event.shift) { + labels.push(isMac ? "⇧" : "Shift"); + } + if (event.meta) { + labels.push(isMac ? "⌘" : style === "linux" ? "Super" : "Win"); + } + + return labels; +} + +const KEY_LABEL_MAP: Record = { + Enter: "↵", + Return: "↵", + Escape: "Esc", + Esc: "Esc", + Backspace: "⌫", + Delete: "⌦", + Del: "⌦", + Tab: "⇥", + Space: "Space", + Spacebar: "Space", + Up: "↑", + ArrowUp: "↑", + Down: "↓", + ArrowDown: "↓", + Left: "←", + ArrowLeft: "←", + Right: "→", + ArrowRight: "→", + Home: "Home", + End: "End", + PageUp: "PgUp", + PageDown: "PgDn", + Insert: "Ins", + Comma: ",", + Period: ".", + Slash: "/", + Backslash: "\\", + Semicolon: ";", + Quote: "'", + Apostrophe: "'", + BracketLeft: "[", + BracketRight: "]", + Minus: "-", + Equal: "=", + Plus: "+", + Backquote: "`", + Grave: "`", +}; + +/** + * Map a semantic key token to its display label. Handles named keys, letters, + * digits (top-row / numpad / "DigitN" spellings), and function keys, with a + * graceful humanized fallback for anything unrecognised. + */ +export function keyTokenToLabel(token: string): string { + if (!token) { + return ""; + } + + const mapped = KEY_LABEL_MAP[token]; + if (mapped) { + return mapped; + } + + // Single letter → uppercase keycap. + if (/^[A-Za-z]$/.test(token)) { + return token.toUpperCase(); + } + + // Digit tokens across spellings: "1", "Num1", "Digit1", "Numpad1". + const digitMatch = token.match(/^(?:Num|Digit|Numpad)?([0-9])$/); + if (digitMatch) { + return digitMatch[1]; + } + + // Function keys F1..F24. + if (/^F(?:[1-9]|1[0-9]|2[0-4])$/.test(token)) { + return token; + } + + // Already a single printable character. + if (token.length === 1) { + return token; + } + + // Unknown token — show it as-is rather than dropping the keystroke. + return token; +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts new file mode 100644 index 000000000..7fe43c428 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts @@ -0,0 +1,112 @@ +import fc from "fast-check"; +import { describe, expect, it } from "vitest"; +import { buildKeycapGroups, computeOpacity } from "./keystrokeCoalescing"; +import { DEFAULT_KEYSTROKE_OVERLAY_POLICY, type KeystrokeEvent } from "./keystrokeTypes"; + +const POLICY = DEFAULT_KEYSTROKE_OVERLAY_POLICY; + +function ev(timeMs: number, key: string, mods: Partial = {}): KeystrokeEvent { + return { timeMs, key, ...mods }; +} + +describe("computeOpacity", () => { + it("is fully opaque during the hold, then fades to zero", () => { + expect(computeOpacity(0, POLICY)).toBe(1); + expect(computeOpacity(POLICY.holdMs, POLICY)).toBe(1); + expect(computeOpacity(POLICY.holdMs + POLICY.fadeMs / 2, POLICY)).toBeCloseTo(0.5, 5); + expect(computeOpacity(POLICY.holdMs + POLICY.fadeMs, POLICY)).toBe(0); + expect(computeOpacity(POLICY.holdMs + POLICY.fadeMs + 1000, POLICY)).toBe(0); + }); +}); + +describe("buildKeycapGroups", () => { + it("shows a single plain key right after it is pressed", () => { + const groups = buildKeycapGroups([ev(1000, "A")], 1000, "mac"); + expect(groups).toHaveLength(1); + expect(groups[0].labels).toEqual(["A"]); + expect(groups[0].isChord).toBe(false); + expect(groups[0].opacity).toBe(1); + }); + + it("does not reveal a key before its timestamp", () => { + const groups = buildKeycapGroups([ev(2000, "A")], 1999, "mac"); + expect(groups).toHaveLength(0); + }); + + it("groups a modifier chord into one keycap group", () => { + const groups = buildKeycapGroups([ev(1000, "P", { meta: true, shift: true })], 1000, "mac"); + expect(groups).toHaveLength(1); + expect(groups[0].isChord).toBe(true); + expect(groups[0].labels).toEqual(["⇧", "⌘", "P"]); + }); + + it("merges rapid plain keystrokes into one running group", () => { + const events = [ev(1000, "H"), ev(1100, "E"), ev(1200, "L"), ev(1300, "L"), ev(1400, "O")]; + const groups = buildKeycapGroups(events, 1400, "mac"); + expect(groups).toHaveLength(1); + expect(groups[0].labels).toEqual(["H", "E", "L", "L", "O"]); + }); + + it("splits keystrokes separated by more than the group window", () => { + const events = [ev(1000, "A"), ev(1000 + POLICY.groupWindowMs + 50, "B")]; + const groups = buildKeycapGroups(events, events[1].timeMs, "mac"); + expect(groups).toHaveLength(2); + expect(groups.map((g) => g.labels)).toEqual([["A"], ["B"]]); + }); + + it("keeps a chord separate from adjacent typing", () => { + const events = [ev(1000, "A"), ev(1050, "C", { meta: true }), ev(1100, "B")]; + const groups = buildKeycapGroups(events, 1100, "mac"); + expect(groups.map((g) => g.isChord)).toEqual([false, true, false]); + }); + + it("ignores lone modifier presses", () => { + const groups = buildKeycapGroups([ev(1000, "Shift"), ev(1010, "Meta")], 1010, "mac"); + expect(groups).toHaveLength(0); + }); + + it("caps the number of simultaneously visible groups", () => { + const events = Array.from({ length: 10 }, (_, i) => + ev(1000 + i * (POLICY.groupWindowMs + 100), "A", { meta: true }), + ); + const last = events[events.length - 1].timeMs; + const groups = buildKeycapGroups(events, last, "mac"); + expect(groups.length).toBeLessThanOrEqual(POLICY.maxGroups); + }); + + it("drops a group once it has fully faded", () => { + const groups = buildKeycapGroups( + [ev(1000, "A")], + 1000 + POLICY.holdMs + POLICY.fadeMs + 1, + "mac", + ); + expect(groups).toHaveLength(0); + }); + + it("property: never reveals future keys and opacity stays in (0,1]", () => { + fc.assert( + fc.property( + fc.array( + fc.record({ + timeMs: fc.integer({ min: 0, max: 200_000 }), + key: fc.constantFrom("A", "B", "C", "Enter", "Space", "Shift", "1"), + meta: fc.boolean(), + shift: fc.boolean(), + }), + { maxLength: 40 }, + ), + fc.integer({ min: 0, max: 200_000 }), + (events, t) => { + const groups = buildKeycapGroups(events as KeystrokeEvent[], t, "mac"); + for (const group of groups) { + expect(group.opacity).toBeGreaterThan(0); + expect(group.opacity).toBeLessThanOrEqual(1); + expect(group.startMs).toBeLessThanOrEqual(t); + expect(group.lastMs).toBeLessThanOrEqual(t); + expect(group.labels.length).toBeGreaterThan(0); + } + }, + ), + ); + }); +}); diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts new file mode 100644 index 000000000..c6aa578bf --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts @@ -0,0 +1,102 @@ +// Keystroke overlay — coalescing / display policy. +// +// Pure function of (events, currentTimeMs, style, policy) → visible keycap +// groups. Because it depends only on event timestamps and the queried time, it +// is inherently frame-rate independent: the same playback time always yields +// the same overlay, whether the editor renders at 30fps or 120fps. +// +// Two grouping rules: +// 1. A key pressed with a non-shift modifier (Ctrl/Alt/Meta) is a *chord* — +// modifiers + key render together as one group ("⌘⇧P") and never merge. +// 2. Plain keys typed in quick succession merge into one running group so +// "hello" reads as a word instead of five flickering caps. + +import { formatModifierLabels, isModifierToken, keyTokenToLabel } from "./keyLabels"; +import { + DEFAULT_KEYSTROKE_OVERLAY_POLICY, + type KeycapGroup, + type KeystrokeEvent, + type KeystrokeOverlayPolicy, + type ModifierGlyphStyle, +} from "./keystrokeTypes"; + +export function computeOpacity(ageMs: number, policy: KeystrokeOverlayPolicy): number { + if (ageMs <= policy.holdMs) { + return 1; + } + if (ageMs >= policy.holdMs + policy.fadeMs) { + return 0; + } + return 1 - (ageMs - policy.holdMs) / policy.fadeMs; +} + +export function buildKeycapGroups( + events: KeystrokeEvent[], + currentTimeMs: number, + style: ModifierGlyphStyle, + policy: KeystrokeOverlayPolicy = DEFAULT_KEYSTROKE_OVERLAY_POLICY, +): KeycapGroup[] { + const visibleSpanMs = policy.holdMs + policy.fadeMs; + // Only events within this lookback window can possibly still be visible. + const lookbackMs = visibleSpanMs + policy.groupWindowMs * (policy.maxKeysPerGroup + 1) + 2000; + const windowStart = currentTimeMs - lookbackMs; + + const relevant = events + .filter( + (event) => + event.timeMs <= currentTimeMs && + event.timeMs >= windowStart && + !isModifierToken(event.key), + ) + .sort((a, b) => a.timeMs - b.timeMs); + + const groups: KeycapGroup[] = []; + + for (const event of relevant) { + // Shift alone does not start a chord (Shift+letter is just an uppercase + // keystroke); Ctrl/Alt/Meta do. + const isChord = Boolean(event.ctrl || event.alt || event.meta); + const keyLabel = keyTokenToLabel(event.key); + + if (isChord) { + groups.push({ + id: `k${event.timeMs}-${groups.length}`, + labels: [...formatModifierLabels(event, style), keyLabel], + isChord: true, + startMs: event.timeMs, + lastMs: event.timeMs, + opacity: 1, + }); + continue; + } + + const open = groups[groups.length - 1]; + const canExtend = + open && + !open.isChord && + event.timeMs - open.lastMs <= policy.groupWindowMs && + open.labels.length < policy.maxKeysPerGroup; + + if (open && canExtend) { + open.labels.push(keyLabel); + open.lastMs = event.timeMs; + } else { + groups.push({ + id: `k${event.timeMs}-${groups.length}`, + labels: [keyLabel], + isChord: false, + startMs: event.timeMs, + lastMs: event.timeMs, + opacity: 1, + }); + } + } + + return groups + .map((group) => ({ + ...group, + opacity: computeOpacity(currentTimeMs - group.lastMs, policy), + })) + .filter((group) => group.opacity > 0) + .slice(-policy.maxGroups); +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts new file mode 100644 index 000000000..40b65fcfb --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts @@ -0,0 +1,59 @@ +// Keystroke overlay — shared types and default display policy. +// +// The main process captures a stable semantic key token (resolved from +// uiohook-napi's UiohookKey table) plus modifier booleans. All interpretation +// and layout lives in the renderer so it stays pure and unit-testable. + +export interface KeystrokeEvent { + /** Elapsed capture time in ms — same clock as cursor telemetry. */ + timeMs: number; + /** + * Semantic key token (e.g. "A", "Enter", "Space", "Up", "Comma"). Falls back + * to the raw numeric keycode as a string when the token is unknown. + */ + key: string; + ctrl?: boolean; + alt?: boolean; + shift?: boolean; + meta?: boolean; +} + +export type ModifierGlyphStyle = "mac" | "windows" | "linux"; + +export interface KeystrokeOverlayPolicy { + /** Consecutive plain keystrokes within this gap merge into one keycap group. */ + groupWindowMs: number; + /** Fully-opaque hold time after a group's last keystroke. */ + holdMs: number; + /** Fade-out duration after the hold elapses. */ + fadeMs: number; + /** Max keys shown in a single running (typed) group before it wraps. */ + maxKeysPerGroup: number; + /** Max simultaneously-visible groups; older ones drop off. */ + maxGroups: number; +} + +// ── Display policy ──────────────────────────────────────────────────────────── +// These five constants ARE the feel of the overlay. They are intentionally +// isolated here so the behaviour can be tuned (or surfaced as user settings) +// without touching the coalescing algorithm or the renderer. +export const DEFAULT_KEYSTROKE_OVERLAY_POLICY: KeystrokeOverlayPolicy = { + groupWindowMs: 650, + holdMs: 1200, + fadeMs: 350, + maxKeysPerGroup: 12, + maxGroups: 3, +}; + +export interface KeycapGroup { + /** Deterministic id derived from the group's first keystroke. */ + id: string; + /** Display labels in order, e.g. ["⌘","⇧","P"] or ["H","e","l","l","o"]. */ + labels: string[]; + /** True when the group is a modifier chord rather than running typed text. */ + isChord: boolean; + startMs: number; + lastMs: number; + /** Visibility 0..1 at the queried time. */ + opacity: number; +} From d741c80e96c50e7da6c02f73cd4e4ee1713d7d68 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Wed, 8 Jul 2026 21:59:53 +0530 Subject: [PATCH 02/10] fix(keystroke-overlay): stable keycap ids + full auto-repeat suppression Address CodeRabbit review on #733: - keystrokeCoalescing: derive KeycapGroup.id from the first keystroke's identity (time + key + modifiers) instead of its position in the sliding lookback window, so a group keeps a stable id across frames and does not remount/flicker during scrub. Adds a regression test. - interaction: bump lastKeystrokeTimeMs on collapsed auto-repeats so a held key is fully suppressed instead of leaking one event every ~45ms. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/ipc/cursor/interaction.ts | 3 +++ .../keystrokeCoalescing.test.ts | 15 +++++++++++++++ .../keystrokeOverlay/keystrokeCoalescing.ts | 17 +++++++++++++++-- 3 files changed, 33 insertions(+), 2 deletions(-) diff --git a/electron/ipc/cursor/interaction.ts b/electron/ipc/cursor/interaction.ts index ffddc9a83..30d8169b2 100644 --- a/electron/ipc/cursor/interaction.ts +++ b/electron/ipc/cursor/interaction.ts @@ -318,7 +318,10 @@ export async function startInteractionCapture() { const timeMs = getCursorCaptureElapsedMs(); // Collapse auto-repeat: ignore the same physical key re-firing rapidly. + // Bump the timestamp on collapsed repeats too, so a held key stays + // suppressed for its whole duration instead of leaking every other repeat. if (keycode === lastKeystrokeCode && timeMs - lastKeystrokeTimeMs < 45) { + lastKeystrokeTimeMs = timeMs; return; } lastKeystrokeCode = keycode; diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts index 7fe43c428..e637f1351 100644 --- a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.test.ts @@ -83,6 +83,21 @@ describe("buildKeycapGroups", () => { expect(groups).toHaveLength(0); }); + it("keeps a group's id stable across frames even as the window drops older groups", () => { + // X is >10s before A; at t=11500 X is still in the lookback window (A is + // the 2nd group), at t=12200 X has dropped (A is the 1st group). A must + // keep the same id despite its changing position — otherwise it remounts. + const events = [ev(0, "X", { meta: true }), ev(11000, "A")]; + const idEarly = buildKeycapGroups(events, 11500, "mac").find( + (g) => g.labels[0] === "A", + )?.id; + const idLater = buildKeycapGroups(events, 12200, "mac").find( + (g) => g.labels[0] === "A", + )?.id; + expect(idEarly).toBeDefined(); + expect(idLater).toBe(idEarly); + }); + it("property: never reveals future keys and opacity stays in (0,1]", () => { fc.assert( fc.property( diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts index c6aa578bf..03c1c6af9 100644 --- a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts @@ -20,6 +20,19 @@ import { type ModifierGlyphStyle, } from "./keystrokeTypes"; +/** + * Stable id for a keycap group. Derived only from the group's first keystroke's + * intrinsic identity (time + key + modifiers), never from its position in the + * sliding lookback window — otherwise the same logical group would get a new id + * as the window shifts, remounting it every frame and flickering during scrub. + */ +function groupId(event: KeystrokeEvent): string { + const mods = + `${event.ctrl ? "c" : ""}${event.alt ? "a" : ""}` + + `${event.shift ? "s" : ""}${event.meta ? "m" : ""}`; + return `k${event.timeMs}:${event.key}:${mods}`; +} + export function computeOpacity(ageMs: number, policy: KeystrokeOverlayPolicy): number { if (ageMs <= policy.holdMs) { return 1; @@ -60,7 +73,7 @@ export function buildKeycapGroups( if (isChord) { groups.push({ - id: `k${event.timeMs}-${groups.length}`, + id: groupId(event), labels: [...formatModifierLabels(event, style), keyLabel], isChord: true, startMs: event.timeMs, @@ -82,7 +95,7 @@ export function buildKeycapGroups( open.lastMs = event.timeMs; } else { groups.push({ - id: `k${event.timeMs}-${groups.length}`, + id: groupId(event), labels: [keyLabel], isChord: false, startMs: event.timeMs, From 5491398dc9d56d61d954ca134f7425fd4a83696a Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 13:15:13 +0530 Subject: [PATCH 03/10] =?UTF-8?q?feat(editor):=20keystroke=20overlay=20?= =?UTF-8?q?=E2=80=94=20settings=20persistence=20+=20Pixi=20render=20layer?= =?UTF-8?q?=20(2/3)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Builds on the capture + display foundation (#733): - projectPersistence + editorPreferences: persist showKeystrokes / keystrokePosition / keystrokeSize (mirrors the cursor prefs), normalized and defaulted (off / bottom-center / 1x). - PixiKeystrokeOverlay: a parent-agnostic PIXI layer mirroring PixiCursorOverlay's construct/update/reset/destroy contract, driven entirely by the tested coalescing core; pooled Graphics + Text keycaps, screen-anchored (bottom/top), size-scaled, with per-group fade. - detectGlyphStyle(): platform-aware modifier glyphs (guarded for Node tests). tsc clean (electron + renderer); 25 pure-core tests pass. Remaining wiring (settings-panel controls + parent threading, VideoPlayback instantiation, keystroke sidecar persistence + IPC, export parity) needs a local dev loop to verify and is tracked as the follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../video-editor/editorPreferences.ts | 12 + .../video-editor/projectPersistence.ts | 11 + .../keystrokeOverlay/PixiKeystrokeOverlay.ts | 222 ++++++++++++++++++ .../keystrokeOverlay/keyLabels.ts | 19 ++ .../keystrokeOverlay/keystrokeTypes.ts | 6 + 5 files changed, 270 insertions(+) create mode 100644 src/components/video-editor/videoPlayback/keystrokeOverlay/PixiKeystrokeOverlay.ts diff --git a/src/components/video-editor/editorPreferences.ts b/src/components/video-editor/editorPreferences.ts index fbf354ab7..7a384df30 100644 --- a/src/components/video-editor/editorPreferences.ts +++ b/src/components/video-editor/editorPreferences.ts @@ -30,6 +30,9 @@ type PersistedEditorControls = Pick< | "showCursor" | "loopCursor" | "cursorStyle" + | "showKeystrokes" + | "keystrokePosition" + | "keystrokeSize" | "cursorSize" | "cursorSmoothing" | "cursorSpringStiffnessMultiplier" @@ -118,6 +121,9 @@ export const DEFAULT_EDITOR_PREFERENCES: EditorPreferences = { showCursor: DEFAULT_EDITOR_CONTROLS.showCursor, loopCursor: DEFAULT_EDITOR_CONTROLS.loopCursor, cursorStyle: DEFAULT_EDITOR_CONTROLS.cursorStyle, + showKeystrokes: DEFAULT_EDITOR_CONTROLS.showKeystrokes, + keystrokePosition: DEFAULT_EDITOR_CONTROLS.keystrokePosition, + keystrokeSize: DEFAULT_EDITOR_CONTROLS.keystrokeSize, cursorSize: DEFAULT_EDITOR_CONTROLS.cursorSize, cursorSmoothing: DEFAULT_EDITOR_CONTROLS.cursorSmoothing, cursorSpringStiffnessMultiplier: DEFAULT_EDITOR_CONTROLS.cursorSpringStiffnessMultiplier, @@ -304,6 +310,9 @@ function normalizeEditorControls( showCursor: sanitizedRaw.showCursor ?? fallback.showCursor, loopCursor: sanitizedRaw.loopCursor ?? fallback.loopCursor, cursorStyle: sanitizedRaw.cursorStyle ?? fallback.cursorStyle, + showKeystrokes: sanitizedRaw.showKeystrokes ?? fallback.showKeystrokes, + keystrokePosition: sanitizedRaw.keystrokePosition ?? fallback.keystrokePosition, + keystrokeSize: sanitizedRaw.keystrokeSize ?? fallback.keystrokeSize, cursorSize: sanitizedRaw.cursorSize ?? fallback.cursorSize, cursorSmoothing: sanitizedRaw.cursorSmoothing ?? fallback.cursorSmoothing, cursorSpringStiffnessMultiplier: @@ -382,6 +391,9 @@ function normalizeEditorControls( showCursor: normalized.showCursor, loopCursor: normalized.loopCursor, cursorStyle: normalized.cursorStyle, + showKeystrokes: normalized.showKeystrokes, + keystrokePosition: normalized.keystrokePosition, + keystrokeSize: normalized.keystrokeSize, cursorSize: normalized.cursorSize, cursorSmoothing: normalized.cursorSmoothing, cursorSpringStiffnessMultiplier: normalized.cursorSpringStiffnessMultiplier, diff --git a/src/components/video-editor/projectPersistence.ts b/src/components/video-editor/projectPersistence.ts index 9810d5fb3..3406b23bb 100644 --- a/src/components/video-editor/projectPersistence.ts +++ b/src/components/video-editor/projectPersistence.ts @@ -106,6 +106,9 @@ export interface ProjectEditorState { showCursor: boolean; loopCursor: boolean; cursorStyle: CursorStyle; + showKeystrokes: boolean; + keystrokePosition: "bottom-center" | "bottom-left" | "bottom-right" | "top-center"; + keystrokeSize: number; cursorClickEffect: CursorClickEffectStyle; cursorClickEffectColor: string; cursorClickEffectScale: number; @@ -918,6 +921,14 @@ export function normalizeProjectEditor(editor: Partial): Pro ), showCursor: typeof editor.showCursor === "boolean" ? editor.showCursor : true, loopCursor: typeof editor.loopCursor === "boolean" ? editor.loopCursor : false, + showKeystrokes: typeof editor.showKeystrokes === "boolean" ? editor.showKeystrokes : false, + keystrokePosition: + editor.keystrokePosition === "bottom-left" || + editor.keystrokePosition === "bottom-right" || + editor.keystrokePosition === "top-center" + ? editor.keystrokePosition + : "bottom-center", + keystrokeSize: isFiniteNumber(editor.keystrokeSize) ? clamp(editor.keystrokeSize, 0.5, 2) : 1, cursorStyle: normalizedCursorStyle, cursorClickEffect: normalizeCursorClickEffectStyle( (editor as Partial).cursorClickEffect, diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/PixiKeystrokeOverlay.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/PixiKeystrokeOverlay.ts new file mode 100644 index 000000000..7087108c0 --- /dev/null +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/PixiKeystrokeOverlay.ts @@ -0,0 +1,222 @@ +// PixiKeystrokeOverlay — renders coalesced keystroke keycaps as a PIXI layer. +// +// Mirrors PixiCursorOverlay's contract: a parent-agnostic `container` built in +// the constructor (the caller attaches it), a per-frame `update(...)`, and +// `reset()` / `destroy()`. All *display logic* (which keys are visible, chord +// grouping, fade) comes from the pure keystrokeCoalescing core, so this file +// only turns groups into pooled Graphics + Text — keeping the hard parts unit- +// tested and the untestable Pixi parts thin. + +import { Container, Graphics, Text, TextStyle } from "pixi.js"; +import type { CursorViewportRect } from "../cursorViewport"; +import { detectGlyphStyle } from "./keyLabels"; +import { buildKeycapGroups } from "./keystrokeCoalescing"; +import { + DEFAULT_KEYSTROKE_OVERLAY_POLICY, + type KeycapGroup, + type KeystrokeEvent, + type KeystrokeOverlayPolicy, + type KeystrokeOverlayPosition, + type ModifierGlyphStyle, +} from "./keystrokeTypes"; + +export interface KeystrokeRenderConfig { + /** Modifier glyph style (⌘⌥⇧⌃ vs Ctrl/Alt/Win/Super). */ + glyphStyle: ModifierGlyphStyle; + /** Screen anchor for the keycap stack. */ + position: KeystrokeOverlayPosition; + /** User size multiplier (settings `keystrokeSize`, ~0.5..2). */ + sizeScale: number; + /** Coalescing / fade policy shared with the pure core. */ + policy: KeystrokeOverlayPolicy; +} + +export const DEFAULT_KEYSTROKE_CONFIG: KeystrokeRenderConfig = { + glyphStyle: detectGlyphStyle(), + position: "bottom-center", + sizeScale: 1, + policy: DEFAULT_KEYSTROKE_OVERLAY_POLICY, +}; + +interface PooledKeycap { + background: Graphics; + label: Text; +} + +const KEYCAP_FILL = 0x1b1b1f; +const KEYCAP_STROKE = 0x3a3a42; +const KEYCAP_TEXT = 0xf4f4f5; + +export class PixiKeystrokeOverlay { + public readonly container: Container; + private config: KeystrokeRenderConfig; + private readonly pool: PooledKeycap[] = []; + + constructor(config: Partial = {}) { + this.config = { ...DEFAULT_KEYSTROKE_CONFIG, ...config }; + this.container = new Container(); + this.container.label = "keystroke-overlay"; + } + + setConfig(config: Partial): void { + this.config = { ...this.config, ...config }; + } + + /** + * Per-frame render. Signature mirrors PixiCursorOverlay.update: samples must + * be sorted ascending by timeMs; `viewport` is the base frame rect; `visible` + * gates the whole layer. `freeze` is accepted for call-site parity but unused + * — the overlay is a pure function of `timeMs`, so scrubbing already snaps to + * the correct state without stepping any animation. + */ + update( + samples: KeystrokeEvent[], + timeMs: number, + viewport: CursorViewportRect, + visible: boolean, + _freeze = false, + ): void { + if (!visible || samples.length === 0 || viewport.width <= 0 || viewport.height <= 0) { + this.hide(); + return; + } + + const groups = buildKeycapGroups(samples, timeMs, this.config.glyphStyle, this.config.policy); + if (groups.length === 0) { + this.hide(); + return; + } + + this.container.visible = true; + this.render(groups, viewport); + } + + private hide(): void { + this.container.visible = false; + for (const keycap of this.pool) { + keycap.background.visible = false; + keycap.label.visible = false; + } + } + + private obtainKeycap(index: number): PooledKeycap { + let keycap = this.pool[index]; + if (!keycap) { + const background = new Graphics(); + const label = new Text({ + text: "", + style: new TextStyle({ + fontFamily: + "ui-monospace, SFMono-Regular, Menlo, Consolas, 'Liberation Mono', monospace", + fontWeight: "600", + fill: KEYCAP_TEXT, + }), + }); + label.anchor.set(0.5); + keycap = { background, label }; + this.pool[index] = keycap; + this.container.addChild(background, label); + } + keycap.background.visible = true; + keycap.label.visible = true; + return keycap; + } + + private render(groups: KeycapGroup[], viewport: CursorViewportRect): void { + // Size everything relative to viewport height so the overlay scales with + // the rendered video, then apply the user's size multiplier. + const fontSize = Math.max(10, Math.min(64, viewport.height * 0.032 * this.config.sizeScale)); + const padX = fontSize * 0.55; + const capHeight = fontSize * 1.7; + const capGap = fontSize * 0.32; // gap between keys within a group + const rowGap = fontSize * 0.4; // gap between stacked groups + const edgeMargin = fontSize * 1.1; + const cornerRadius = fontSize * 0.32; + + // Lay out each group as a row of pill-shaped keycaps. + let poolIndex = 0; + const rows: { width: number; caps: { index: number; width: number }[] }[] = []; + + for (const group of groups) { + const caps: { index: number; width: number }[] = []; + let rowWidth = 0; + for (const text of group.labels) { + const index = poolIndex++; + const keycap = this.obtainKeycap(index); + keycap.label.style.fontSize = fontSize; + keycap.label.text = text; + const capWidth = Math.max(capHeight, keycap.label.width + padX * 2); + caps.push({ index, width: capWidth }); + rowWidth += capWidth + capGap; + } + rows.push({ width: Math.max(0, rowWidth - capGap), caps }); + } + + // Hide pooled keycaps not used this frame. + for (let i = poolIndex; i < this.pool.length; i++) { + this.pool[i].background.visible = false; + this.pool[i].label.visible = false; + } + + const totalHeight = rows.length * capHeight + Math.max(0, rows.length - 1) * rowGap; + const topY = this.resolveTopY(viewport, totalHeight, edgeMargin); + + rows.forEach((row, rowIndex) => { + const rowY = topY + rowIndex * (capHeight + rowGap); + const opacity = groups[rowIndex].opacity; + let x = this.resolveRowStartX(viewport, row.width, edgeMargin); + + for (const cap of row.caps) { + const keycap = this.pool[cap.index]; + keycap.background.clear(); + keycap.background + .roundRect(x, rowY, cap.width, capHeight, cornerRadius) + .fill({ color: KEYCAP_FILL, alpha: 0.82 * opacity }) + .stroke({ color: KEYCAP_STROKE, width: 1, alpha: 0.9 * opacity }); + keycap.label.position.set(x + cap.width / 2, rowY + capHeight / 2); + keycap.label.alpha = opacity; + x += cap.width + capGap; + } + }); + } + + private resolveTopY( + viewport: CursorViewportRect, + totalHeight: number, + edgeMargin: number, + ): number { + if (this.config.position === "top-center") { + return viewport.y + edgeMargin; + } + // bottom-* anchors: stack upward from near the bottom edge. + return viewport.y + viewport.height - edgeMargin - totalHeight; + } + + private resolveRowStartX( + viewport: CursorViewportRect, + rowWidth: number, + edgeMargin: number, + ): number { + switch (this.config.position) { + case "bottom-left": + return viewport.x + edgeMargin; + case "bottom-right": + return viewport.x + viewport.width - edgeMargin - rowWidth; + default: + return viewport.x + (viewport.width - rowWidth) / 2; + } + } + + reset(): void { + this.hide(); + } + + destroy(): void { + for (const keycap of this.pool) { + keycap.background.destroy(); + keycap.label.destroy(); + } + this.pool.length = 0; + this.container.destroy({ children: true }); + } +} diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts index df3eab0f7..29811929a 100644 --- a/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keyLabels.ts @@ -5,6 +5,25 @@ import type { KeystrokeEvent, ModifierGlyphStyle } from "./keystrokeTypes"; +/** + * Best-effort platform glyph style for modifier labels. Guards `navigator` so + * the pure label functions stay usable under Node (tests) — defaults to + * "windows" when there is no navigator. + */ +export function detectGlyphStyle(): ModifierGlyphStyle { + if (typeof navigator === "undefined") { + return "windows"; + } + const probe = `${navigator.platform ?? ""} ${navigator.userAgent ?? ""}`.toLowerCase(); + if (probe.includes("mac")) { + return "mac"; + } + if (probe.includes("linux") && !probe.includes("android")) { + return "linux"; + } + return "windows"; +} + /** * Tokens that represent a modifier or lock key on their own. A lone press of * one of these should NOT produce a keycap — modifiers only appear as part of a diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts index 40b65fcfb..fa339aa68 100644 --- a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeTypes.ts @@ -20,6 +20,12 @@ export interface KeystrokeEvent { export type ModifierGlyphStyle = "mac" | "windows" | "linux"; +export type KeystrokeOverlayPosition = + | "bottom-center" + | "bottom-left" + | "bottom-right" + | "top-center"; + export interface KeystrokeOverlayPolicy { /** Consecutive plain keystrokes within this gap merge into one keycap group. */ groupWindowMs: number; From b21df2e0c2f491b6f62d00f8409885f4d1fb4322 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 13:23:40 +0530 Subject: [PATCH 04/10] feat(editor): keystroke telemetry sidecar persistence primitives Add the main-process persistence layer for keystroke telemetry, mirroring the cursor .cursor.json sidecar exactly: - utils: getKeystrokePathForVideo -> `${videoPath}.keystrokes.json` - telemetry: normalizeKeystrokeTelemetrySamples, writeKeystrokeTelemetry, persistPendingKeystrokeTelemetry, snapshotKeystrokeTelemetryForPersistence (use KEYSTROKE_TELEMETRY_VERSION / MAX_KEYSTROKE_SAMPLES; skip/remove the sidecar when empty, matching the cursor path). tsc (node) + biome clean. Wiring these into set-recording-state, the stop paths, a get-keystroke-telemetry IPC channel, and the renderer load path is the remaining follow-up (see PR description checklist). Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/ipc/cursor/telemetry.ts | 91 +++++++++++++++++++++++++++++++- electron/ipc/utils.ts | 5 +- 2 files changed, 94 insertions(+), 2 deletions(-) diff --git a/electron/ipc/cursor/telemetry.ts b/electron/ipc/cursor/telemetry.ts index b8695be04..0f7168cc8 100644 --- a/electron/ipc/cursor/telemetry.ts +++ b/electron/ipc/cursor/telemetry.ts @@ -2,6 +2,7 @@ import fs from "node:fs/promises"; import { CURSOR_SAMPLE_INTERVAL_MS, CURSOR_TELEMETRY_VERSION, + KEYSTROKE_TELEMETRY_VERSION, MAX_CURSOR_SAMPLES, MAX_KEYSTROKE_SAMPLES, } from "../constants"; @@ -16,6 +17,7 @@ import { isCursorCaptureActive, linuxCursorScreenPoint, pendingCursorSamples, + pendingKeystrokeSamples, selectedSource, selectedWindowBounds, setActiveCursorSamples, @@ -23,6 +25,7 @@ import { setCursorCaptureInterval, setCursorCapturePauseStartedAtMs, setPendingCursorSamples, + setPendingKeystrokeSamples, } from "../state"; import type { CursorInteractionType, @@ -30,7 +33,7 @@ import type { CursorVisualType, KeystrokeTelemetryPoint, } from "../types"; -import { getScreen, getTelemetryPathForVideo } from "../utils"; +import { getKeystrokePathForVideo, getScreen, getTelemetryPathForVideo } from "../utils"; export function clamp(value: number, min: number, max: number) { return Math.min(max, Math.max(min, value)); @@ -279,6 +282,92 @@ export function pushKeystrokeSample(sample: KeystrokeTelemetryPoint) { } } +export function normalizeKeystrokeTelemetrySamples(rawSamples: unknown): KeystrokeTelemetryPoint[] { + const samples = Array.isArray(rawSamples) + ? rawSamples + : Array.isArray((rawSamples as { samples?: unknown[] } | null | undefined)?.samples) + ? ((rawSamples as { samples: unknown[] }).samples ?? []) + : []; + const boundedSamples = samples.slice(0, MAX_KEYSTROKE_SAMPLES); + + return boundedSamples + .filter((sample: unknown) => Boolean(sample && typeof sample === "object")) + .map((sample: unknown) => { + const point = sample as Partial; + return { + timeMs: + typeof point.timeMs === "number" && Number.isFinite(point.timeMs) + ? Math.max(0, point.timeMs) + : 0, + key: typeof point.key === "string" ? point.key : "", + ctrl: point.ctrl === true ? true : undefined, + alt: point.alt === true ? true : undefined, + shift: point.shift === true ? true : undefined, + meta: point.meta === true ? true : undefined, + }; + }) + .filter((point) => point.key.length > 0) + .sort((a, b) => a.timeMs - b.timeMs); +} + +export async function writeKeystrokeTelemetry(videoPath: string, samples: unknown) { + const keystrokePath = getKeystrokePathForVideo(videoPath); + const normalizedSamples = normalizeKeystrokeTelemetrySamples(samples); + + if (normalizedSamples.length === 0) { + await fs.rm(keystrokePath, { force: true }); + return normalizedSamples; + } + + await fs.writeFile( + keystrokePath, + JSON.stringify( + { version: KEYSTROKE_TELEMETRY_VERSION, samples: normalizedSamples }, + null, + 2, + ), + "utf-8", + ); + + return normalizedSamples; +} + +export async function persistPendingKeystrokeTelemetry(videoPath: string) { + const keystrokePath = getKeystrokePathForVideo(videoPath); + if (pendingKeystrokeSamples.length > 0) { + await fs.writeFile( + keystrokePath, + JSON.stringify( + { version: KEYSTROKE_TELEMETRY_VERSION, samples: pendingKeystrokeSamples }, + null, + 2, + ), + "utf-8", + ); + } else { + await fs.rm(keystrokePath, { force: true }); + } + setPendingKeystrokeSamples([]); +} + +export function snapshotKeystrokeTelemetryForPersistence() { + if (activeKeystrokeSamples.length === 0) { + return; + } + + if (pendingKeystrokeSamples.length === 0) { + setPendingKeystrokeSamples([...activeKeystrokeSamples]); + return; + } + + const lastPendingTimeMs = + pendingKeystrokeSamples[pendingKeystrokeSamples.length - 1]?.timeMs ?? -1; + setPendingKeystrokeSamples([ + ...pendingKeystrokeSamples, + ...activeKeystrokeSamples.filter((sample) => sample.timeMs > lastPendingTimeMs), + ]); +} + export function sampleCursorPoint() { const point = getNormalizedCursorPoint(); pushCursorSample(point.cx, point.cy, getCursorCaptureElapsedMs(), "move"); diff --git a/electron/ipc/utils.ts b/electron/ipc/utils.ts index 3f2efb065..6acb767fc 100644 --- a/electron/ipc/utils.ts +++ b/electron/ipc/utils.ts @@ -67,6 +67,10 @@ export function getTelemetryPathForVideo(videoPath: string) { return `${videoPath}.cursor.json`; } +export function getKeystrokePathForVideo(videoPath: string) { + return `${videoPath}.keystrokes.json`; +} + export function isAutoRecordingPath(filePath: string) { return path.basename(filePath).startsWith(AUTO_RECORDING_PREFIX); } @@ -129,4 +133,3 @@ export function approveUserPath(filePath: string | null | undefined): void { // Ignore invalid paths; later reads will surface the underlying error. } } - From cc698e19329aac5c83fc0d5f4d105ce69e7ef63e Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 13:32:19 +0530 Subject: [PATCH 05/10] fix(editor): keep preset snapshot type-complete after adding keystroke prefs Adding showKeystrokes/keystrokePosition/keystrokeSize to PersistedEditorControls left captureEditorPresetSnapshot's EditorPresetSnapshot literal missing them. Add the three fields (placeholder defaults until VideoEditor manages keystroke settings in part 3). Full-project tsc is now clean except one pre-existing unused import (VideoPlayback scalePreviewBorderRadius) that predates this branch. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/components/video-editor/VideoEditor.tsx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 0b23fb692..bd27caccc 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -752,6 +752,11 @@ export default function VideoEditor() { const captureEditorPresetSnapshot = useCallback( (): EditorPresetSnapshot => ({ wallpaper, + // TODO(keystroke-overlay part 3): capture real keystroke overlay settings + // once VideoEditor manages them; placeholders keep presets type-complete. + showKeystrokes: false, + keystrokePosition: "bottom-center", + keystrokeSize: 1, shadowIntensity, backgroundBlur, zoomMotionBlur, From 6823233034eacbfc0d72dd2a108ec59339077111 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 14:04:39 +0530 Subject: [PATCH 06/10] feat(editor): keystroke capture-enable IPC + get-keystroke-telemetry - recording.ts: enable keystroke capture at record-start when showKeystrokes is set (new set-recording-state options arg); disable + snapshot on all stop paths (native-win + set-recording-state); add get-keystroke-telemetry read handler mirroring the cursor one. - preload + electron-env: getKeystrokeTelemetry bridge, setRecordingState options, KeystrokeTelemetryPoint type. tsc (node) clean. Renderer wiring (overlay mount + settings toggle) is the remaining follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- electron/electron-env.d.ts | 20 +- electron/ipc/register/recording.ts | 733 ++++++++++++++++------------- electron/preload.ts | 7 +- 3 files changed, 422 insertions(+), 338 deletions(-) diff --git a/electron/electron-env.d.ts b/electron/electron-env.d.ts index 2aa748b49..3c1ff98fb 100644 --- a/electron/electron-env.d.ts +++ b/electron/electron-env.d.ts @@ -565,7 +565,10 @@ interface Window { startDelayMsByPath?: Record; error?: string; }>; - setRecordingState: (recording: boolean) => Promise; + setRecordingState: ( + recording: boolean, + options?: { showKeystrokes?: boolean }, + ) => Promise; getCursorTelemetry: (videoPath?: string) => Promise<{ success: boolean; samples: CursorTelemetryPoint[]; @@ -581,6 +584,12 @@ interface Window { message?: string; error?: string; }>; + getKeystrokeTelemetry: (videoPath?: string) => Promise<{ + success: boolean; + samples: KeystrokeTelemetryPoint[]; + message?: string; + error?: string; + }>; getSystemCursorAssets: () => Promise<{ success: boolean; cursors: Record; @@ -952,6 +961,15 @@ interface ProcessedDesktopSource { windowTitle?: string; } +interface KeystrokeTelemetryPoint { + timeMs: number; + key: string; + ctrl?: boolean; + alt?: boolean; + shift?: boolean; + meta?: boolean; +} + interface CursorTelemetryPoint { timeMs: number; cx: number; diff --git a/electron/ipc/register/recording.ts b/electron/ipc/register/recording.ts index b13453c74..da2f72e9e 100644 --- a/electron/ipc/register/recording.ts +++ b/electron/ipc/register/recording.ts @@ -13,24 +13,27 @@ import { systemPreferences, } from "electron"; import { showCursor } from "../../cursorHider"; -import { getMonitorHandles } from "../monitorResolver"; import { ALLOW_RECORDLY_WINDOW_CAPTURE } from "../constants"; import { startWindowBoundsCapture, stopWindowBoundsCapture } from "../cursor/bounds"; import { startInteractionCapture, stopInteractionCapture } from "../cursor/interaction"; import { startNativeCursorMonitor, stopNativeCursorMonitor } from "../cursor/monitor"; import { normalizeCursorTelemetrySamples, + normalizeKeystrokeTelemetrySamples, pauseCursorCaptureAtBoundary, persistPendingCursorTelemetry, + persistPendingKeystrokeTelemetry, resetCursorCaptureClock, resumeCursorCapture, sampleCursorPoint, snapshotCursorTelemetryForPersistence, + snapshotKeystrokeTelemetryForPersistence, startCursorSampling, stopCursorCapture, writeCursorTelemetry, } from "../cursor/telemetry"; import { getFfmpegBinaryPath } from "../ffmpeg/binary"; +import { getMonitorHandles } from "../monitorResolver"; import { ensureNativeCaptureHelperBinary, ensureSwiftHelperBinary, @@ -97,6 +100,7 @@ import { nativeScreenRecordingActive, selectedSource, setActiveCursorSamples, + setActiveKeystrokeSamples, setCachedSystemCursorAssets, setCachedSystemCursorAssetsSourceMtimeMs, setCursorCaptureStartTimeMs, @@ -105,6 +109,7 @@ import { setFfmpegCaptureTargetPath, setFfmpegScreenRecordingActive, setIsCursorCaptureActive, + setIsKeystrokeCaptureActive, setLastLeftClick, setLinuxCursorScreenPoint, setNativeCaptureMicrophonePath, @@ -116,6 +121,7 @@ import { setNativeCaptureTargetPath, setNativeScreenRecordingActive, setPendingCursorSamples, + setPendingKeystrokeSamples, setWindowsCaptureOutputBuffer, setWindowsCapturePaused, setWindowsCaptureProcess, @@ -138,6 +144,7 @@ import { } from "../state"; import type { CursorTelemetryPoint, NativeMacRecordingOptions, SelectedSource } from "../types"; import { + getKeystrokePathForVideo, getMacPrivacySettingsUrl, getRecordingsDir, getScreen, @@ -433,13 +440,16 @@ export function registerRecordingHandlers( const recordingsDir = await getRecordingsDir(); const timestamp = Date.now(); const outputPath = path.join(recordingsDir, `recording-${timestamp}.mp4`); - tempVideoPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mp4`); - + tempVideoPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mp4`, + ); + let captureOutput = ""; let systemAudioPath: string | null = null; let microphonePath: string | null = null; let orphanedMicAudioPath: string | null = null; - + const browserMicFallbackRequested = shouldStartWindowsBrowserMicrophoneFallback(options); const captureTarget = resolveWindowsCaptureTarget( @@ -482,7 +492,7 @@ export function registerRecordingHandlers( // Fallback to coordinate-based matching if handle resolution fails config.displayId = captureTarget.displayId; } - + config.displayX = Math.round(captureTarget.bounds.x); config.displayY = Math.round(captureTarget.bounds.y); config.displayW = Math.round(captureTarget.bounds.width); @@ -507,7 +517,10 @@ export function registerRecordingHandlers( if (options?.capturesMicrophone && !browserMicFallbackRequested) { microphonePath = path.join(recordingsDir, `recording-${timestamp}.mic.wav`); - tempMicPath = path.join(app.getPath("temp"), `recordly-native-${timestamp}.mic.wav`); + tempMicPath = path.join( + app.getPath("temp"), + `recordly-native-${timestamp}.mic.wav`, + ); config.captureMic = true; config.micOutputPath = tempMicPath; if (options.microphoneLabel) { @@ -905,333 +918,346 @@ export function registerRecordingHandlers( const start = Date.now(); console.log("[PERF:MAIN] Handler: stop-native-screen-recording: STARTED"); try { - // Windows native capture stop path - if (process.platform === "win32" && windowsNativeCaptureActive) { - let stagedTempVideoPath: string | null = null; - let stagedTempSystemAudioPath: string | null = null; - let stagedTempMicAudioPath: string | null = null; - try { - if (!windowsCaptureProcess) { - throw new Error("Native Windows capture process is not running"); - } - - const proc = windowsCaptureProcess; - const preferredVideoPath = windowsCaptureTargetPath; - const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = windowsSystemAudioPath; - const diagnosticsMicAudioPath = windowsMicAudioPath; - setWindowsCaptureStopRequested(true); - proc.stdin.write("stop\n"); - const tempVideoPath = await waitForWindowsCaptureStop(proc); - stagedTempVideoPath = tempVideoPath; - const finalVideoPath = preferredVideoPath ?? tempVideoPath; + // Windows native capture stop path + if (process.platform === "win32" && windowsNativeCaptureActive) { + let stagedTempVideoPath: string | null = null; + let stagedTempSystemAudioPath: string | null = null; + let stagedTempMicAudioPath: string | null = null; + try { + if (!windowsCaptureProcess) { + throw new Error("Native Windows capture process is not running"); + } - // Native Windows capture results are initially written to a safe temporary path - // (to avoid encoding failures with non-ASCII characters). We move them to the final - // destination now using Node.js, which handles Unicode paths correctly. - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + const proc = windowsCaptureProcess; + const preferredVideoPath = windowsCaptureTargetPath; + const preferredOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = windowsSystemAudioPath; + const diagnosticsMicAudioPath = windowsMicAudioPath; + setWindowsCaptureStopRequested(true); + proc.stdin.write("stop\n"); + const tempVideoPath = await waitForWindowsCaptureStop(proc); + stagedTempVideoPath = tempVideoPath; + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + + // Native Windows capture results are initially written to a safe temporary path + // (to avoid encoding failures with non-ASCII characters). We move them to the final + // destination now using Node.js, which handles Unicode paths correctly. + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } - if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); - stagedTempSystemAudioPath = tempAudioPath; - const finalAudioPath = windowsSystemAudioPath; - if (await pathExists(tempAudioPath)) { - await moveFileWithOverwrite(tempAudioPath, finalAudioPath); - const tempJson = tempAudioPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + if (windowsSystemAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempAudioPath = tempVideoPath.replace(".mp4", ".system.wav"); + stagedTempSystemAudioPath = tempAudioPath; + const finalAudioPath = windowsSystemAudioPath; + if (await pathExists(tempAudioPath)) { + await moveFileWithOverwrite(tempAudioPath, finalAudioPath); + const tempJson = tempAudioPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalAudioPath + ".json"); + } } } - } - if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { - const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); - stagedTempMicAudioPath = tempMicPath; - const finalMicPath = windowsMicAudioPath; - if (await pathExists(tempMicPath)) { - await moveFileWithOverwrite(tempMicPath, finalMicPath); - const tempJson = tempMicPath + ".json"; - if (await pathExists(tempJson)) { - await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + if (windowsMicAudioPath && tempVideoPath.endsWith(".mp4")) { + const tempMicPath = tempVideoPath.replace(".mp4", ".mic.wav"); + stagedTempMicAudioPath = tempMicPath; + const finalMicPath = windowsMicAudioPath; + if (await pathExists(tempMicPath)) { + await moveFileWithOverwrite(tempMicPath, finalMicPath); + const tempJson = tempMicPath + ".json"; + if (await pathExists(tempJson)) { + await moveFileWithOverwrite(tempJson, finalMicPath + ".json"); + } } } - } - const validation = await validateRecordedVideo(finalVideoPath); + const validation = await validateRecordedVideo(finalVideoPath); - setWindowsCaptureProcess(null); - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); - await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); - setWindowsPendingVideoPath(finalVideoPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - }); - await writeWindowsRecordingDiagnostics(finalVideoPath, { - phase: "stop", - outputPath: finalVideoPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - details: { + setWindowsCaptureProcess(null); + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); + await cleanupWindowsOrphanedMicAudioPath(preferredOrphanedMicAudioPath); + setWindowsPendingVideoPath(finalVideoPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - }, - }); + }); + await writeWindowsRecordingDiagnostics(finalVideoPath, { + phase: "stop", + outputPath: finalVideoPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + }, + }); - // Persist cursor telemetry before returning so the editor can find it immediately - snapshotCursorTelemetryForPersistence(); - try { - await persistPendingCursorTelemetry(finalVideoPath); - } catch (error) { - console.warn("Failed to persist cursor telemetry during native stop:", error); - } + // Persist cursor telemetry before returning so the editor can find it immediately + snapshotCursorTelemetryForPersistence(); + try { + await persistPendingCursorTelemetry(finalVideoPath); + } catch (error) { + console.warn( + "Failed to persist cursor telemetry during native stop:", + error, + ); + } + snapshotKeystrokeTelemetryForPersistence(); + try { + await persistPendingKeystrokeTelemetry(finalVideoPath); + } catch (error) { + console.warn( + "Failed to persist keystroke telemetry during native stop:", + error, + ); + } - return { success: true, path: finalVideoPath }; - } catch (error) { - console.error("Failed to stop native Windows capture:", error); - const fallbackPath = await resolveExistingPath( - windowsCaptureTargetPath, - stagedTempVideoPath, - ); - const recoveredSystemAudioPath = await resolveExistingPath( - windowsSystemAudioPath, - stagedTempSystemAudioPath, - ); - const recoveredMicAudioPath = await resolveExistingPath( - windowsMicAudioPath, - stagedTempMicAudioPath, - ); - const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; - const diagnosticsSystemAudioPath = recoveredSystemAudioPath ?? windowsSystemAudioPath; - const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; - setWindowsNativeCaptureActive(false); - setNativeScreenRecordingActive(false); - setWindowsCaptureProcess(null); - setWindowsCaptureTargetPath(null); - setWindowsCaptureStopRequested(false); - setWindowsCapturePaused(false); - setWindowsOrphanedMicAudioPath(null); + return { success: true, path: finalVideoPath }; + } catch (error) { + console.error("Failed to stop native Windows capture:", error); + const fallbackPath = await resolveExistingPath( + windowsCaptureTargetPath, + stagedTempVideoPath, + ); + const recoveredSystemAudioPath = await resolveExistingPath( + windowsSystemAudioPath, + stagedTempSystemAudioPath, + ); + const recoveredMicAudioPath = await resolveExistingPath( + windowsMicAudioPath, + stagedTempMicAudioPath, + ); + const fallbackOrphanedMicAudioPath = windowsOrphanedMicAudioPath; + const diagnosticsSystemAudioPath = + recoveredSystemAudioPath ?? windowsSystemAudioPath; + const diagnosticsMicAudioPath = recoveredMicAudioPath ?? windowsMicAudioPath; + setWindowsNativeCaptureActive(false); + setNativeScreenRecordingActive(false); + setWindowsCaptureProcess(null); + setWindowsCaptureTargetPath(null); + setWindowsCaptureStopRequested(false); + setWindowsCapturePaused(false); + setWindowsOrphanedMicAudioPath(null); - if (fallbackPath) { - try { - const validation = await validateRecordedVideo(fallbackPath); - setWindowsPendingVideoPath(fallbackPath); - setWindowsSystemAudioPath(recoveredSystemAudioPath); - setWindowsMicAudioPath(recoveredMicAudioPath); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: validation.fileSizeBytes, - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + if (fallbackPath) { + try { + const validation = await validateRecordedVideo(fallbackPath); + setWindowsPendingVideoPath(fallbackPath); + setWindowsSystemAudioPath(recoveredSystemAudioPath); + setWindowsMicAudioPath(recoveredMicAudioPath); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: validation.fileSizeBytes, - durationSeconds: validation.durationSeconds, - recoveredAfterStopFailure: true, - }, - }); - return { success: true, path: fallbackPath }; - } catch { - // File is absent or failed validation. + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: validation.fileSizeBytes, + durationSeconds: validation.durationSeconds, + recoveredAfterStopFailure: true, + }, + }); + return { success: true, path: fallbackPath }; + } catch { + // File is absent or failed validation. + } } - } - setWindowsSystemAudioPath(null); - setWindowsMicAudioPath(null); - setWindowsPendingVideoPath(null); - await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); + setWindowsSystemAudioPath(null); + setWindowsMicAudioPath(null); + setWindowsPendingVideoPath(null); + await cleanupWindowsOrphanedMicAudioPath(fallbackOrphanedMicAudioPath); - recordNativeCaptureDiagnostics({ - backend: "windows-wgc", - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - error: String(error), - }); - await writeWindowsRecordingDiagnostics(fallbackPath, { - phase: "stop", - outputPath: fallbackPath, - systemAudioPath: diagnosticsSystemAudioPath, - microphonePath: diagnosticsMicAudioPath, - processOutput: windowsCaptureOutputBuffer.trim() || undefined, - error: String(error), - details: { + recordNativeCaptureDiagnostics({ + backend: "windows-wgc", + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, fileSizeBytes: await getFileSizeIfPresent(fallbackPath), - }, - }); + error: String(error), + }); + await writeWindowsRecordingDiagnostics(fallbackPath, { + phase: "stop", + outputPath: fallbackPath, + systemAudioPath: diagnosticsSystemAudioPath, + microphonePath: diagnosticsMicAudioPath, + processOutput: windowsCaptureOutputBuffer.trim() || undefined, + error: String(error), + details: { + fileSizeBytes: await getFileSizeIfPresent(fallbackPath), + }, + }); + + return { + success: false, + message: "Failed to stop native Windows capture", + error: String(error), + }; + } + } + if (process.platform !== "darwin") { return { success: false, - message: "Failed to stop native Windows capture", - error: String(error), + message: "Native screen recording is only available on macOS.", }; } - } - if (process.platform !== "darwin") { - return { - success: false, - message: "Native screen recording is only available on macOS.", - }; - } + if (!nativeScreenRecordingActive) { + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } - if (!nativeScreenRecordingActive) { - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; + return { success: false, message: "No native screen recording is active." }; } - return { success: false, message: "No native screen recording is active." }; - } - - try { - if (!nativeCaptureProcess) { - throw new Error("Native capture helper process is not running"); - } - - const process = nativeCaptureProcess; - const preferredVideoPath = nativeCaptureTargetPath; - const preferredSystemAudioPath = nativeCaptureSystemAudioPath; - const preferredMicrophonePath = nativeCaptureMicrophonePath; - console.log( - "[stop-native] Audio paths — system:", - preferredSystemAudioPath, - "mic:", - preferredMicrophonePath, - ); - setNativeCaptureStopRequested(true); - process.stdin.write("stop\n"); - const tempVideoPath = await waitForNativeCaptureStop(process); - console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); - setNativeCaptureProcess(null); - setNativeScreenRecordingActive(false); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); - - const finalVideoPath = preferredVideoPath ?? tempVideoPath; - if (tempVideoPath !== finalVideoPath) { - await moveFileWithOverwrite(tempVideoPath, finalVideoPath); - } + try { + if (!nativeCaptureProcess) { + throw new Error("Native capture helper process is not running"); + } - if (preferredSystemAudioPath || preferredMicrophonePath) { + const process = nativeCaptureProcess; + const preferredVideoPath = nativeCaptureTargetPath; + const preferredSystemAudioPath = nativeCaptureSystemAudioPath; + const preferredMicrophonePath = nativeCaptureMicrophonePath; console.log( - "[stop-native] Attempting audio mux (merging separate tracks) into:", - finalVideoPath, + "[stop-native] Audio paths — system:", + preferredSystemAudioPath, + "mic:", + preferredMicrophonePath, ); - try { - await muxNativeMacRecordingWithAudio( + setNativeCaptureStopRequested(true); + process.stdin.write("stop\n"); + const tempVideoPath = await waitForNativeCaptureStop(process); + console.log("[stop-native] Helper stopped, tempVideoPath:", tempVideoPath); + setNativeCaptureProcess(null); + setNativeScreenRecordingActive(false); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); + + const finalVideoPath = preferredVideoPath ?? tempVideoPath; + if (tempVideoPath !== finalVideoPath) { + await moveFileWithOverwrite(tempVideoPath, finalVideoPath); + } + + if (preferredSystemAudioPath || preferredMicrophonePath) { + console.log( + "[stop-native] Attempting audio mux (merging separate tracks) into:", finalVideoPath, - preferredSystemAudioPath, - preferredMicrophonePath, - ); - console.log("[stop-native] Audio mux completed successfully"); - } catch (error) { - console.warn( - "[stop-native] Audio mux failed (video still has inline audio):", - error, ); + try { + await muxNativeMacRecordingWithAudio( + finalVideoPath, + preferredSystemAudioPath, + preferredMicrophonePath, + ); + console.log("[stop-native] Audio mux completed successfully"); + } catch (error) { + console.warn( + "[stop-native] Audio mux failed (video still has inline audio):", + error, + ); + } + } else { + console.log("[stop-native] No separate audio tracks to mux"); } - } else { - console.log("[stop-native] No separate audio tracks to mux"); - } - return await finalizeStoredVideo(finalVideoPath); - } catch (error) { - console.error("Failed to stop native ScreenCaptureKit recording:", error); - const fallbackPath = nativeCaptureTargetPath; - const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; - const fallbackMicrophonePath = nativeCaptureMicrophonePath; - const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); - setNativeScreenRecordingActive(false); - setNativeCaptureProcess(null); - setNativeCaptureTargetPath(null); - setNativeCaptureSystemAudioPath(null); - setNativeCaptureMicrophonePath(null); - setNativeCaptureStopRequested(false); - setNativeCapturePaused(false); + return await finalizeStoredVideo(finalVideoPath); + } catch (error) { + console.error("Failed to stop native ScreenCaptureKit recording:", error); + const fallbackPath = nativeCaptureTargetPath; + const fallbackSystemAudioPath = nativeCaptureSystemAudioPath; + const fallbackMicrophonePath = nativeCaptureMicrophonePath; + const fallbackFileSizeBytes = await getFileSizeIfPresent(fallbackPath); + setNativeScreenRecordingActive(false); + setNativeCaptureProcess(null); + setNativeCaptureTargetPath(null); + setNativeCaptureSystemAudioPath(null); + setNativeCaptureMicrophonePath(null); + setNativeCaptureStopRequested(false); + setNativeCapturePaused(false); - recordNativeCaptureDiagnostics({ - backend: "mac-screencapturekit", - phase: "stop", - sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, - sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", - displayId: lastNativeCaptureDiagnostics?.displayId ?? null, - displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, - windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, - helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, - outputPath: fallbackPath, - systemAudioPath: fallbackSystemAudioPath, - microphonePath: fallbackMicrophonePath, - osRelease: lastNativeCaptureDiagnostics?.osRelease, - supported: lastNativeCaptureDiagnostics?.supported, - helperExists: lastNativeCaptureDiagnostics?.helperExists, - processOutput: nativeCaptureOutputBuffer.trim() || undefined, - fileSizeBytes: fallbackFileSizeBytes, - error: String(error), - }); + recordNativeCaptureDiagnostics({ + backend: "mac-screencapturekit", + phase: "stop", + sourceId: lastNativeCaptureDiagnostics?.sourceId ?? null, + sourceType: lastNativeCaptureDiagnostics?.sourceType ?? "unknown", + displayId: lastNativeCaptureDiagnostics?.displayId ?? null, + displayBounds: lastNativeCaptureDiagnostics?.displayBounds ?? null, + windowHandle: lastNativeCaptureDiagnostics?.windowHandle ?? null, + helperPath: lastNativeCaptureDiagnostics?.helperPath ?? null, + outputPath: fallbackPath, + systemAudioPath: fallbackSystemAudioPath, + microphonePath: fallbackMicrophonePath, + osRelease: lastNativeCaptureDiagnostics?.osRelease, + supported: lastNativeCaptureDiagnostics?.supported, + helperExists: lastNativeCaptureDiagnostics?.helperExists, + processOutput: nativeCaptureOutputBuffer.trim() || undefined, + fileSizeBytes: fallbackFileSizeBytes, + error: String(error), + }); - // Try to recover: if the target file exists on disk, finalize with it - if (fallbackPath) { - try { - await fs.access(fallbackPath); - console.log( - "[stop-native-screen-recording] Recovering with fallback path:", - fallbackPath, - ); - if (fallbackSystemAudioPath || fallbackMicrophonePath) { - try { - await muxNativeMacRecordingWithAudio( - fallbackPath, - fallbackSystemAudioPath, - fallbackMicrophonePath, - ); - } catch (muxError) { - console.warn( - "Failed to mux recovered native macOS audio into capture:", - muxError, - ); + // Try to recover: if the target file exists on disk, finalize with it + if (fallbackPath) { + try { + await fs.access(fallbackPath); + console.log( + "[stop-native-screen-recording] Recovering with fallback path:", + fallbackPath, + ); + if (fallbackSystemAudioPath || fallbackMicrophonePath) { + try { + await muxNativeMacRecordingWithAudio( + fallbackPath, + fallbackSystemAudioPath, + fallbackMicrophonePath, + ); + } catch (muxError) { + console.warn( + "Failed to mux recovered native macOS audio into capture:", + muxError, + ); + } } + return await finalizeStoredVideo(fallbackPath); + } catch { + // File doesn't exist or isn't accessible } - return await finalizeStoredVideo(fallbackPath); - } catch { - // File doesn't exist or isn't accessible } - } - const recovered = await recoverNativeMacCaptureOutput(); - if (recovered) { - return recovered; - } + const recovered = await recoverNativeMacCaptureOutput(); + if (recovered) { + return recovered; + } return { success: false, @@ -1810,49 +1836,58 @@ export function registerRecordingHandlers( } }); - ipcMain.handle("set-recording-state", (_, recording: boolean) => { - if (recording) { - stopCursorCapture(); - stopInteractionCapture(); - startWindowBoundsCapture(); - void startNativeCursorMonitor(); - setIsCursorCaptureActive(true); - setActiveCursorSamples([]); - setPendingCursorSamples([]); - setCursorCaptureStartTimeMs(Date.now()); - resetCursorCaptureClock(); - setLinuxCursorScreenPoint(null); - setLastLeftClick(null); - sampleCursorPoint(); - startCursorSampling(); - void startInteractionCapture(); - } else { - setIsCursorCaptureActive(false); - stopCursorCapture(); - stopInteractionCapture(); - stopWindowBoundsCapture(); - stopNativeCursorMonitor(); - showCursor(); - setLinuxCursorScreenPoint(null); - resetCursorCaptureClock(); - snapshotCursorTelemetryForPersistence(); - setActiveCursorSamples([]); - } - - const source = selectedSource || { name: "Screen" }; - BrowserWindow.getAllWindows().forEach((window) => { - if (!window.isDestroyed()) { - window.webContents.send("recording-state-changed", { - recording, - sourceName: source.name, - }); + ipcMain.handle( + "set-recording-state", + (_, recording: boolean, options?: { showKeystrokes?: boolean }) => { + if (recording) { + stopCursorCapture(); + stopInteractionCapture(); + startWindowBoundsCapture(); + void startNativeCursorMonitor(); + setIsCursorCaptureActive(true); + setIsKeystrokeCaptureActive(Boolean(options?.showKeystrokes)); + setActiveCursorSamples([]); + setPendingCursorSamples([]); + setActiveKeystrokeSamples([]); + setPendingKeystrokeSamples([]); + setCursorCaptureStartTimeMs(Date.now()); + resetCursorCaptureClock(); + setLinuxCursorScreenPoint(null); + setLastLeftClick(null); + sampleCursorPoint(); + startCursorSampling(); + void startInteractionCapture(); + } else { + setIsCursorCaptureActive(false); + setIsKeystrokeCaptureActive(false); + stopCursorCapture(); + stopInteractionCapture(); + stopWindowBoundsCapture(); + stopNativeCursorMonitor(); + showCursor(); + setLinuxCursorScreenPoint(null); + resetCursorCaptureClock(); + snapshotCursorTelemetryForPersistence(); + snapshotKeystrokeTelemetryForPersistence(); + setActiveCursorSamples([]); + setActiveKeystrokeSamples([]); } - }); - if (onRecordingStateChange) { - onRecordingStateChange(recording, source.name); - } - }); + const source = selectedSource || { name: "Screen" }; + BrowserWindow.getAllWindows().forEach((window) => { + if (!window.isDestroyed()) { + window.webContents.send("recording-state-changed", { + recording, + sourceName: source.name, + }); + } + }); + + if (onRecordingStateChange) { + onRecordingStateChange(recording, source.name); + } + }, + ); ipcMain.handle("pause-cursor-capture", (_, pausedAtMs?: unknown) => { pauseCursorCaptureAtBoundary(normalizeRendererTimestampMs(pausedAtMs)); @@ -1865,6 +1900,34 @@ export function registerRecordingHandlers( return { success: true }; }); + ipcMain.handle("get-keystroke-telemetry", async (_, videoPath?: string) => { + const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); + if (!targetVideoPath) { + return { success: true, samples: [] }; + } + + const keystrokePath = getKeystrokePathForVideo(targetVideoPath); + try { + const content = await fs.readFile(keystrokePath, "utf-8"); + const parsed = parseJsonWithByteOrderMark(content); + const samples = normalizeKeystrokeTelemetrySamples(parsed); + + return { success: true, samples }; + } catch (error) { + const nodeError = error as NodeJS.ErrnoException; + if (nodeError.code === "ENOENT") { + return { success: true, samples: [] }; + } + console.error("Failed to load keystroke telemetry:", error); + return { + success: false, + message: "Failed to load keystroke telemetry", + error: String(error), + samples: [], + }; + } + }); + ipcMain.handle("get-cursor-telemetry", async (_, videoPath?: string) => { const targetVideoPath = normalizeVideoSourcePath(videoPath ?? currentVideoPath); if (!targetVideoPath) { diff --git a/electron/preload.ts b/electron/preload.ts index 9a15edb99..8d3a3eb6f 100644 --- a/electron/preload.ts +++ b/electron/preload.ts @@ -572,8 +572,8 @@ contextBridge.exposeInMainWorld("electronAPI", { getRecordedVideoPath: () => { return ipcRenderer.invoke("get-recorded-video-path"); }, - setRecordingState: (recording: boolean) => { - return ipcRenderer.invoke("set-recording-state", recording); + setRecordingState: (recording: boolean, options?: { showKeystrokes?: boolean }) => { + return ipcRenderer.invoke("set-recording-state", recording, options); }, setCursorScale: (scale: number) => { return ipcRenderer.invoke("set-cursor-scale", scale); @@ -584,6 +584,9 @@ contextBridge.exposeInMainWorld("electronAPI", { setCursorTelemetry: (videoPath: string | undefined, samples: CursorTelemetryPoint[]) => { return ipcRenderer.invoke("set-cursor-telemetry", videoPath, samples); }, + getKeystrokeTelemetry: (videoPath?: string) => { + return ipcRenderer.invoke("get-keystroke-telemetry", videoPath); + }, getSystemCursorAssets: () => { return ipcRenderer.invoke("get-system-cursor-assets"); }, From 4b39df16373ae4d2c6618ed4190e1d1bab770867 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 18:34:19 +0530 Subject: [PATCH 07/10] fix(keystroke-overlay): drop redundant open guard in coalescing (CodeRabbit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit canExtend already includes the `open` truthiness check, and TS 4.4+ aliased- condition narrowing keeps `open` narrowed under `if (canExtend)`, so the extra `open &&` was redundant. Behaviour unchanged; 25 tests pass. CodeRabbit also suggested deriving the keycap id from the relevant-array index; skipped intentionally — the lookback window slides, so that index is not stable across frames and would reintroduce the remount/flicker fixed in d741c80. True duplicate events (same time+key+modifiers) are already prevented upstream by the capture-layer auto-repeat collapse. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts index 03c1c6af9..2b01ecc6c 100644 --- a/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts +++ b/src/components/video-editor/videoPlayback/keystrokeOverlay/keystrokeCoalescing.ts @@ -90,7 +90,7 @@ export function buildKeycapGroups( event.timeMs - open.lastMs <= policy.groupWindowMs && open.labels.length < policy.maxKeysPerGroup; - if (open && canExtend) { + if (canExtend) { open.labels.push(keyLabel); open.lastMs = event.timeMs; } else { From 161354ca98adb6993942950f2da3b41b1e002583 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 18:50:53 +0530 Subject: [PATCH 08/10] feat(editor): mount keystroke overlay + settings toggle (Part 3/3 renderer) Completes the keystroke overlay end to end: - VideoEditor: keystrokeTelemetry state + load effect (get-keystroke-telemetry), showKeystrokes/position/size state, props to VideoPlayback + SettingsPanel. - VideoPlayback: mount PixiKeystrokeOverlay in a sibling container, per-frame update (outside the cursor-overlay branch so it renders independently), plus attach/destroy lifecycle; drop a pre-existing unused import. - SettingsPanel: 'Show Keystrokes' toggle + size slider. - useScreenRecorder: pass showKeystrokes to setRecordingState at record start. - i18n (en): effects.showKeystrokes / effects.keystrokesSize. Full-project tsc clean (both configs); 25 keystroke tests pass; Electron dev build boots and renderer renders with zero console errors. Position Select and the other 9 locale strings are trivial follow-ups. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/components/video-editor/SettingsPanel.tsx | 33 +++++++++ src/components/video-editor/VideoEditor.tsx | 51 +++++++++++++ src/components/video-editor/VideoPlayback.tsx | 74 ++++++++++++++++++- src/hooks/useScreenRecorder.ts | 14 ++-- src/i18n/locales/en/settings.json | 2 + 5 files changed, 164 insertions(+), 10 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index a90028e2b..7d1ddb0d7 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -762,6 +762,14 @@ interface SettingsPanelProps { onShowCursorChange?: (enabled: boolean) => void; loopCursor?: boolean; onLoopCursorChange?: (enabled: boolean) => void; + showKeystrokes?: boolean; + onShowKeystrokesChange?: (enabled: boolean) => void; + keystrokesPosition?: "bottom-center" | "bottom-left" | "bottom-right" | "top-center"; + onKeystrokesPositionChange?: ( + position: "bottom-center" | "bottom-left" | "bottom-right" | "top-center", + ) => void; + keystrokesSize?: number; + onKeystrokesSizeChange?: (size: number) => void; cursorStyle?: CursorStyle; onCursorStyleChange?: (style: CursorStyle) => void; cursorSize?: number; @@ -1206,6 +1214,10 @@ export function SettingsPanel({ onShowCursorChange, loopCursor = false, onLoopCursorChange, + showKeystrokes = false, + onShowKeystrokesChange, + keystrokesSize = 1, + onKeystrokesSizeChange, cursorStyle = DEFAULT_CURSOR_STYLE, onCursorStyleChange, cursorSize = 5, @@ -3686,6 +3698,14 @@ export function SettingsPanel({ className="data-[state=checked]:bg-[#2563EB] scale-75" /> +
@@ -3735,6 +3755,19 @@ export function SettingsPanel({ formatValue={(v) => `${v.toFixed(2)}×`} parseInput={(text) => parseFloat(text.replace(/×$/, ""))} /> + {showKeystrokes ? ( + onKeystrokesSizeChange?.(v)} + formatValue={(v) => `${v.toFixed(2)}×`} + parseInput={(text) => parseFloat(text.replace(/×$/, ""))} + /> + ) : null} ( + initialEditorPreferences.keystrokePosition, + ); + const [keystrokeSize, setKeystrokeSize] = useState(initialEditorPreferences.keystrokeSize); const [loopCursor, setLoopCursor] = useState(initialEditorPreferences.loopCursor); const [cursorStyle, setCursorStyle] = useState( initialEditorPreferences.cursorStyle ?? DEFAULT_CURSOR_STYLE, @@ -536,6 +545,7 @@ export default function VideoEditor() { const [resolvedWebcamVideoUrl, setResolvedWebcamVideoUrl] = useState(null); const [zoomRegions, setZoomRegions] = useState([]); const [cursorTelemetry, setCursorTelemetry] = useState([]); + const [keystrokeTelemetry, setKeystrokeTelemetry] = useState([]); // Tracks the videoSourcePath for which the cursor telemetry IPC has already // resolved. The smoke-export auto-trigger waits on this so long recordings // still bake cursor/zoom animations into the output — without it, the @@ -3431,6 +3441,37 @@ export default function VideoEditor() { }; }, [videoPath, videoSourcePath]); + useEffect(() => { + let mounted = true; + + async function loadKeystrokeTelemetry() { + if (!videoPath || !videoSourcePath) { + if (mounted) { + setKeystrokeTelemetry([]); + } + return; + } + + try { + const result = await window.electronAPI.getKeystrokeTelemetry(videoSourcePath); + if (mounted) { + setKeystrokeTelemetry(result.success ? result.samples : []); + } + } catch (keystrokeError) { + console.warn("Unable to load keystroke telemetry:", keystrokeError); + if (mounted) { + setKeystrokeTelemetry([]); + } + } + } + + loadKeystrokeTelemetry(); + + return () => { + mounted = false; + }; + }, [videoPath, videoSourcePath]); + const normalizedCursorTelemetry = useMemo(() => { if (cursorTelemetry.length === 0) { return [] as CursorTelemetryPoint[]; @@ -5568,6 +5609,10 @@ export default function VideoEditor() { onAnnotationPositionChange={handleAnnotationPositionChange} onAnnotationSizeChange={handleAnnotationSizeChange} cursorTelemetry={effectiveCursorTelemetry} + keystrokeTelemetry={keystrokeTelemetry} + showKeystrokes={showKeystrokes} + keystrokePosition={keystrokePosition} + keystrokeSize={keystrokeSize} showCursor={effectiveShowCursor} cursorStyle={cursorStyle} cursorSize={cursorSize} @@ -6441,6 +6486,12 @@ export default function VideoEditor() { onConnectedZoomEasingChange={setConnectedZoomEasing} showCursor={effectiveShowCursor} onShowCursorChange={handleShowCursorChange} + showKeystrokes={showKeystrokes} + onShowKeystrokesChange={setShowKeystrokes} + keystrokesPosition={keystrokePosition} + onKeystrokesPositionChange={setKeystrokePosition} + keystrokesSize={keystrokeSize} + onKeystrokesSizeChange={setKeystrokeSize} loopCursor={loopCursor} onLoopCursorChange={setLoopCursor} cursorStyle={cursorStyle} diff --git a/src/components/video-editor/VideoPlayback.tsx b/src/components/video-editor/VideoPlayback.tsx index 499179f39..0ce030b01 100644 --- a/src/components/video-editor/VideoPlayback.tsx +++ b/src/components/video-editor/VideoPlayback.tsx @@ -56,6 +56,11 @@ import { PixiCursorOverlay, preloadCursorAssets, } from "./videoPlayback/cursorRenderer"; +import type { + KeystrokeEvent, + KeystrokeOverlayPosition, +} from "./videoPlayback/keystrokeOverlay/keystrokeTypes"; +import { PixiKeystrokeOverlay } from "./videoPlayback/keystrokeOverlay/PixiKeystrokeOverlay"; import { clamp01 } from "./videoPlayback/mathUtils"; import { createSpringState, @@ -167,10 +172,7 @@ import { SNAP_TO_EDGES_RATIO_AUTO, } from "./videoPlayback/cursorFollowCamera"; import { clampFocusToStage as clampFocusToStageUtil } from "./videoPlayback/focusUtils"; -import { - layoutVideoContent as layoutVideoContentUtil, - scalePreviewBorderRadius, -} from "./videoPlayback/layoutUtils"; +import { layoutVideoContent as layoutVideoContentUtil } from "./videoPlayback/layoutUtils"; import { updateOverlayIndicator } from "./videoPlayback/overlayUtils"; import { createVideoEventHandlers } from "./videoPlayback/videoEventHandlers"; import { getWebcamMediaTargetTimeSeconds, shouldSeekWebcamMedia } from "./videoPlayback/webcamSync"; @@ -386,6 +388,10 @@ interface VideoPlaybackProps { onAnnotationPositionChange?: (id: string, position: { x: number; y: number }) => void; onAnnotationSizeChange?: (id: string, size: { width: number; height: number }) => void; cursorTelemetry?: CursorTelemetryPoint[]; + keystrokeTelemetry?: KeystrokeEvent[]; + showKeystrokes?: boolean; + keystrokePosition?: KeystrokeOverlayPosition; + keystrokeSize?: number; showCursor?: boolean; cursorStyle?: CursorStyle; cursorSize?: number; @@ -471,6 +477,10 @@ const VideoPlayback = forwardRef( onAnnotationPositionChange, onAnnotationSizeChange, cursorTelemetry = [], + keystrokeTelemetry = [], + showKeystrokes = false, + keystrokePosition = "bottom-center", + keystrokeSize = 1, showCursor = false, cursorStyle = DEFAULT_CURSOR_STYLE, cursorSize = DEFAULT_CURSOR_SIZE, @@ -623,6 +633,12 @@ const VideoPlayback = forwardRef( const cursorTelemetryRef = useRef([]); const showCursorRef = useRef(showCursor); const cursorSizeRef = useRef(cursorSize); + const keystrokeTelemetryRef = useRef([]); + const showKeystrokesRef = useRef(showKeystrokes); + const keystrokePositionRef = useRef(keystrokePosition); + const keystrokeSizeRef = useRef(keystrokeSize); + const keystrokeContainerRef = useRef(null); + const keystrokeOverlayRef = useRef(null); const cursorStyleRef = useRef(cursorStyle); const cursorSmoothingRef = useRef(cursorSmoothing); const cursorSpringStiffnessMultiplierRef = useRef(cursorSpringStiffnessMultiplier); @@ -1755,6 +1771,22 @@ const VideoPlayback = forwardRef( cursorSizeRef.current = cursorSize; }, [cursorSize]); + useEffect(() => { + keystrokeTelemetryRef.current = keystrokeTelemetry; + }, [keystrokeTelemetry]); + + useEffect(() => { + showKeystrokesRef.current = showKeystrokes; + }, [showKeystrokes]); + + useEffect(() => { + keystrokePositionRef.current = keystrokePosition; + }, [keystrokePosition]); + + useEffect(() => { + keystrokeSizeRef.current = keystrokeSize; + }, [keystrokeSize]); + useEffect(() => { cursorSmoothingRef.current = cursorSmoothing; }, [cursorSmoothing]); @@ -2170,6 +2202,10 @@ const VideoPlayback = forwardRef( cursorContainerRef.current = cursorContainer; cameraContainer.addChild(cursorContainer); + const keystrokeContainer = new Container(); + keystrokeContainerRef.current = keystrokeContainer; + cameraContainer.addChild(keystrokeContainer); + // Cursor overlay - rendered above the masked video so it can sit in front // of the content without getting clipped. if (cursorOverlayEnabled) { @@ -2199,6 +2235,13 @@ const VideoPlayback = forwardRef( cursorOverlayRef.current = null; } + const keystrokeOverlay = new PixiKeystrokeOverlay({ + position: keystrokePositionRef.current, + sizeScale: keystrokeSizeRef.current, + }); + keystrokeOverlayRef.current = keystrokeOverlay; + keystrokeContainer.addChild(keystrokeOverlay.container); + setPixiReady(true); })().catch((error) => { const errorMessage = @@ -2223,6 +2266,10 @@ const VideoPlayback = forwardRef( cursorOverlayRef.current.destroy(); cursorOverlayRef.current = null; } + if (keystrokeOverlayRef.current) { + keystrokeOverlayRef.current.destroy(); + keystrokeOverlayRef.current = null; + } zoomBlurFilterRef.current?.destroy(); motionBlurFilterRef.current?.destroy(); zoomBlurFilterRef.current = null; @@ -2241,6 +2288,7 @@ const VideoPlayback = forwardRef( frameContainerRef.current = null; frameSpriteRef.current = null; cursorContainerRef.current = null; + keystrokeContainerRef.current = null; videoSpriteRef.current = null; }; }, [initializePixiRenderer, onError]); @@ -2296,6 +2344,9 @@ const VideoPlayback = forwardRef( if (cursorOverlayRef.current) { cursorContainer.addChild(cursorOverlayRef.current.container); } + if (keystrokeOverlayRef.current && keystrokeContainerRef.current) { + keystrokeContainerRef.current.addChild(keystrokeOverlayRef.current.container); + } animationStateRef.current = createPlaybackAnimationState(); @@ -2541,6 +2592,21 @@ const VideoPlayback = forwardRef( } | null = null; // Update cursor overlay + emit cursor events + const keystrokeOverlay = keystrokeOverlayRef.current; + if (keystrokeOverlay) { + keystrokeOverlay.setConfig({ + position: keystrokePositionRef.current, + sizeScale: keystrokeSizeRef.current, + }); + keystrokeOverlay.update( + keystrokeTelemetryRef.current, + timeMs, + baseMaskRef.current, + showKeystrokesRef.current, + !isPlayingRef.current || isSeekingRef.current, + ); + } + const cursorOverlay = cursorOverlayRef.current; if (cursorOverlay) { const telemetry = cursorTelemetryRef.current; diff --git a/src/hooks/useScreenRecorder.ts b/src/hooks/useScreenRecorder.ts index c5cd70056..d8da17eb4 100644 --- a/src/hooks/useScreenRecorder.ts +++ b/src/hooks/useScreenRecorder.ts @@ -1,6 +1,7 @@ import { fixWebmDuration } from "@fix-webm-duration/fix"; import { useCallback, useEffect, useRef, useState } from "react"; import { toast } from "sonner"; +import { loadEditorPreferences } from "@/components/video-editor/editorPreferences"; import { getEffectiveRecordingDurationMs } from "@/lib/mediaTiming"; import { getVideoExtensionForMimeType, @@ -213,10 +214,7 @@ export function resolveBrowserCaptureCursorPolicy({ export function shouldUseNativeWindowsCaptureForSource( source: Pick | null | undefined, ): boolean { - return ( - source?.id?.startsWith("screen:") === true || - source?.id?.startsWith("window:") === true - ); + return source?.id?.startsWith("screen:") === true || source?.id?.startsWith("window:") === true; } export function createProcessedMicrophoneConstraints( @@ -1569,7 +1567,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { setRecording(true); try { - await window.electronAPI?.setRecordingState(true); + await window.electronAPI?.setRecordingState(true, { + showKeystrokes: loadEditorPreferences().showKeystrokes, + }); } catch (stateError) { console.warn( "Failed to notify main process that native recording started:", @@ -1909,7 +1909,9 @@ export function useScreenRecorder(): UseScreenRecorderReturn { recorder.start(RECORDER_TIMESLICE_MS); setRecording(true); try { - await window.electronAPI?.setRecordingState(true); + await window.electronAPI?.setRecordingState(true, { + showKeystrokes: loadEditorPreferences().showKeystrokes, + }); } catch (stateError) { console.warn("Failed to notify main process that recording started:", stateError); } diff --git a/src/i18n/locales/en/settings.json b/src/i18n/locales/en/settings.json index 60aa86221..6cc1abdd2 100644 --- a/src/i18n/locales/en/settings.json +++ b/src/i18n/locales/en/settings.json @@ -30,6 +30,7 @@ "show": "Show", "showCursor": "Show Cursor", "loopCursor": "Loop cursor", + "showKeystrokes": "Show Keystrokes", "cursorStyle": "Cursor Style", "cursorStyleOptions": { "macos": "macOS", @@ -92,6 +93,7 @@ "linear": "Linear" }, "cursorSize": "Cursor Size", + "keystrokesSize": "Keystrokes Size", "cursorSmoothing": "Cursor Smoothing", "cursorSpringStiffness": "Cursor Spring Stiffness", "cursorSpringDamping": "Cursor Spring Damping", From 867c1a90793e21b6009d50dab74839b9a6c10958 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 18:53:48 +0530 Subject: [PATCH 09/10] feat(editor): persist keystroke overlay settings so capture enables next recording Add showKeystrokes/keystrokePosition/keystrokeSize to the saveEditorPreferences bundle (+ effect deps). Without this, toggling 'Show Keystrokes' only affected the in-editor overlay; capture is gated at record-start on the persisted pref (loadEditorPreferences().showKeystrokes), so the toggle must persist to actually capture keystrokes on the next recording. Closes the end-to-end loop. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/components/video-editor/VideoEditor.tsx | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/components/video-editor/VideoEditor.tsx b/src/components/video-editor/VideoEditor.tsx index 6a4bbf5cb..e486954a8 100644 --- a/src/components/video-editor/VideoEditor.tsx +++ b/src/components/video-editor/VideoEditor.tsx @@ -2610,6 +2610,9 @@ export default function VideoEditor() { useEffect(() => { saveEditorPreferences({ + showKeystrokes, + keystrokePosition, + keystrokeSize, wallpaper, shadowIntensity, backgroundBlur, @@ -2685,6 +2688,9 @@ export default function VideoEditor() { zoomOutEasing, connectedZoomEasing, showCursor, + showKeystrokes, + keystrokePosition, + keystrokeSize, loopCursor, cursorStyle, cursorSize, From 5481d53ad691a87f402f1d4b2ced1ff66d3462e4 Mon Sep 17 00:00:00 2001 From: starkdcc Date: Sat, 11 Jul 2026 21:54:52 +0530 Subject: [PATCH 10/10] fix(editor): declutter cursor panel + clearer Show Keystrokes toggle Move 'Show Keystrokes' out of the cramped 3-toggle header row into its own full-width labeled row at the top of the cursor panel body, with a full-size switch (was scale-75) so its on/off state is obvious. Header returns to Show Cursor + Loop cursor. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/components/video-editor/SettingsPanel.tsx | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/src/components/video-editor/SettingsPanel.tsx b/src/components/video-editor/SettingsPanel.tsx index 7d1ddb0d7..9828f4137 100644 --- a/src/components/video-editor/SettingsPanel.tsx +++ b/src/components/video-editor/SettingsPanel.tsx @@ -3698,17 +3698,19 @@ export function SettingsPanel({ className="data-[state=checked]:bg-[#2563EB] scale-75" /> -
+