Playback performance and sync

apps/desktop/src-tauri/src/lib.rs

@@ -1915,11 +1915,29 @@ async fn set_playhead_position(
     editor_instance: WindowEditorInstance,
     frame_number: u32,
 ) -> Result<(), String> {
-    editor_instance
-        .modify_and_emit_state(|state| {
-            state.playhead_position = frame_number;
-        })
-        .await;
+    let state_changed = {
+        let state = editor_instance.state.lock().await;
+        state.playhead_position != frame_number
+    };
+
+    if state_changed {
+        editor_instance
+            .modify_and_emit_state(|state| {
+                state.playhead_position = frame_number;
+            })
+            .await;
+    }
+
+    let playback_handle = if state_changed {
+        let state = editor_instance.state.lock().await;
+        state.playback_task.clone()
+    } else {
+        None
+    };
+
+    if let Some(handle) = playback_handle {
+        handle.seek(frame_number);
+    }
 
     Ok(())
 }
@@ -2539,11 +2557,29 @@ async fn is_camera_window_open(app: AppHandle) -> bool {
 #[specta::specta]
 #[instrument(skip(editor_instance))]
 async fn seek_to(editor_instance: WindowEditorInstance, frame_number: u32) -> Result<(), String> {
-    editor_instance
-        .modify_and_emit_state(|state| {
-            state.playhead_position = frame_number;
-        })
-        .await;
+    let state_changed = {
+        let state = editor_instance.state.lock().await;
+        state.playhead_position != frame_number
+    };
+
+    if state_changed {
+        editor_instance
+            .modify_and_emit_state(|state| {
+                state.playhead_position = frame_number;
+            })
+            .await;
+    }
+
+    let playback_handle = if state_changed {
+        let state = editor_instance.state.lock().await;
+        state.playback_task.clone()
+    } else {
+        None
+    };
+
+    if let Some(handle) = playback_handle {
+        handle.seek(frame_number);
+    }
 
     Ok(())
 }

apps/desktop/src/routes/editor/Timeline/index.tsx

@@ -7,6 +7,7 @@ import {
 	createSignal,
 	Index,
 	type JSX,
+	onCleanup,
 	onMount,
 	Show,
 } from "solid-js";
@@ -89,7 +90,6 @@ export function Timeline() {
 		editorState,
 		projectActions,
 		meta,
-		previewResolutionBase,
 	} = useEditorContext();
 
 	const duration = () => editorInstance.recordingDuration;
@@ -221,6 +221,17 @@ export function Timeline() {
 
 	let pendingScrollDelta = 0;
 	let scrollRafId: number | null = null;
+	let pendingSeekFrame: number | null = null;
+	let seekRafId: number | null = null;
+	let seekInFlight = false;
+	let inFlightSeekFrame: number | null = null;
+	let lastCompletedSeekFrame: number | null = null;
+
+	onCleanup(() => {
+		if (zoomRafId !== null) cancelAnimationFrame(zoomRafId);
+		if (scrollRafId !== null) cancelAnimationFrame(scrollRafId);
+		if (seekRafId !== null) cancelAnimationFrame(seekRafId);
+	});
 
 	function flushPendingZoom() {
 		if (pendingZoomDelta === 0 || pendingZoomOrigin === null) {
@@ -266,41 +277,65 @@ export function Timeline() {
 		}
 	}
 
-	async function handleUpdatePlayhead(e: MouseEvent) {
+	function scheduleSeek(frameNumber: number) {
+		if (
+			frameNumber === pendingSeekFrame ||
+			frameNumber === inFlightSeekFrame ||
+			frameNumber === lastCompletedSeekFrame
+		) {
+			return;
+		}
+		pendingSeekFrame = frameNumber;
+		if (seekRafId === null) {
+			seekRafId = requestAnimationFrame(flushPendingSeek);
+		}
+	}
+
+	async function flushPendingSeek() {
+		seekRafId = null;
+
+		if (seekInFlight || pendingSeekFrame === null) {
+			if (pendingSeekFrame !== null && seekRafId === null) {
+				seekRafId = requestAnimationFrame(flushPendingSeek);
+			}
+			return;
+		}
+
+		const frameNumber = pendingSeekFrame;
+		pendingSeekFrame = null;
+		seekInFlight = true;
+		inFlightSeekFrame = frameNumber;
+
+		try {
+			await commands.seekTo(frameNumber);
+			lastCompletedSeekFrame = frameNumber;
+		} catch (err) {
+			console.error("Failed to seek timeline playhead:", err);
+		} finally {
+			seekInFlight = false;
+			inFlightSeekFrame = null;
+			if (pendingSeekFrame !== null && seekRafId === null) {
+				seekRafId = requestAnimationFrame(flushPendingSeek);
+			}
+		}
+	}
+
+	function handleUpdatePlayhead(e: MouseEvent) {
 		const { left } = timelineBounds;
 		if (
 			zoomSegmentDragState.type !== "moving" &&
 			sceneSegmentDragState.type !== "moving" &&
 			maskSegmentDragState.type !== "moving" &&
 			textSegmentDragState.type !== "moving"
 		) {
-			// Guard against missing bounds and clamp computed time to [0, totalDuration()]
 			if (left == null) return;
 			const rawTime =
 				secsPerPixel() * (e.clientX - left) + transform().position;
 			const newTime = Math.min(Math.max(0, rawTime), totalDuration());
-
-			// If playing, some backends require restart to seek reliably
-			if (editorState.playing) {
-				try {
-					await commands.stopPlayback();
-
-					// Round to nearest frame to prevent off-by-one drift
-					const targetFrame = Math.round(newTime * FPS);
-					await commands.seekTo(targetFrame);
-
-					// If the user paused during these async ops, bail out without restarting
-					if (!editorState.playing) {
-						setEditorState("playbackTime", newTime);
-						return;
-					}
-
-					await commands.startPlayback(FPS, previewResolutionBase());
-					setEditorState("playing", true);
-				} catch (err) {
-					console.error("Failed to seek during playback:", err);
-				}
-			}
+			const total = totalDuration();
+			const maxFrame = Math.max(0, Math.ceil(total * FPS) - 1);
+			const targetFrame = Math.min(Math.round(newTime * FPS), maxFrame);
+			scheduleSeek(targetFrame);
 
 			setEditorState("playbackTime", newTime);
 		}

crates/editor/Cargo.toml

@@ -23,6 +23,7 @@ axum = { version = "0.7.5", features = ["ws"] }
 ffmpeg.workspace = true
 specta.workspace = true
 serde = { workspace = true }
+serde_json = "1"
 sentry.workspace = true
 futures = { workspace = true }
 tracing.workspace = true

crates/editor/PLAYBACK-BENCHMARKS.md

@@ -10,8 +10,10 @@ This document tracks performance benchmarks for Cap's playback and decoding syst
 |--------|--------|-----------|
 | Decoder Init | <200ms | - |
 | Decode Latency (p95) | <50ms | - |
+| Startup to First Frame | <250ms | configurable |
 | Effective FPS | ≥30 fps | ±2 fps |
 | Decode Jitter | <10ms | - |
+| Scrub Seek Latency (p95) | <40ms | - |
 | A/V Sync (mic↔video) | <100ms | - |
 | A/V Sync (system↔video) | <100ms | - |
 | Camera-Display Drift | <100ms | - |
@@ -20,6 +22,7 @@ This document tracks performance benchmarks for Cap's playback and decoding syst
 
 - **Decoder Tests**: Init time, hardware acceleration detection, fallback handling
 - **Playback Tests**: Sequential decode, frame retrieval, latency percentiles
+- **Scrub Tests**: Random access seek decode latency and seek failure rate
 - **Audio Sync Tests**: Mic-video sync, system audio-video sync
 - **Camera Sync Tests**: Camera-display drift, frame count alignment
 - **Decode Benchmark**: Creation, sequential, seek, and random access performance
@@ -40,10 +43,12 @@ This document tracks performance benchmarks for Cap's playback and decoding syst
 ```bash
 # Run full playback validation on recordings from real-device-test-runner
 cargo run -p cap-recording --example playback-test-runner -- full
+cargo run -p cap-recording --example playback-test-runner -- full --startup-threshold-ms 250
 
 # Run specific test categories
 cargo run -p cap-recording --example playback-test-runner -- decoder
 cargo run -p cap-recording --example playback-test-runner -- playback
+cargo run -p cap-recording --example playback-test-runner -- scrub
 cargo run -p cap-recording --example playback-test-runner -- audio-sync
 cargo run -p cap-recording --example playback-test-runner -- camera-sync
 
@@ -52,6 +57,69 @@ cargo run -p cap-recording --example playback-test-runner -- --recording-path /p
 
 # List available recordings
 cargo run -p cap-recording --example playback-test-runner -- list
+
+# Emit machine-readable JSON report
+cargo run -p cap-recording --example playback-test-runner -- full --json-output /tmp/playback-benchmark.json
+```
+
+Aggregate JSON outputs from multiple machines:
+
+```bash
+node scripts/aggregate-playback-benchmarks.js --input /path/to/json-results --output /tmp/playback-benchmark-aggregate.md
+node scripts/build-playback-matrix-report.js --input /path/to/json-results --output /tmp/playback-matrix-status.md
+```
+
+Validate matrix coverage and required formats:
+
+```bash
+node scripts/validate-playback-matrix.js --input /path/to/json-results --require-formats mp4,fragmented
+node scripts/validate-playback-matrix.js --input /path/to/json-results --require-formats mp4,fragmented --output-json /tmp/playback-matrix-validation.json
+
+# Finalize aggregate + status + validation artifacts
+node scripts/finalize-playback-matrix.js --input /path/to/json-results --output-dir /tmp/playback-matrix-final --require-formats mp4,fragmented
+node scripts/finalize-playback-matrix.js --input /path/to/json-results --output-dir /tmp/playback-matrix-final --require-formats mp4,fragmented --target-fps 60 --max-scrub-p95-ms 40 --max-startup-ms 250
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --allow-fps-drop 2 --allow-startup-increase-ms 25 --allow-scrub-p95-increase-ms 5
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --allow-missing-candidate
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --fail-on-candidate-only
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --min-samples-per-row 3
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --fail-on-parse-errors
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --fail-on-zero-compared
+node scripts/finalize-playback-matrix.js --input /path/to/candidate-results --output-dir /tmp/playback-matrix-final --compare-baseline /path/to/baseline-results --fail-on-skipped-files
+node scripts/finalize-playback-matrix.js --input /path/to/json-results --output-dir /tmp/playback-matrix-final --output-json /tmp/playback-matrix-final/playback-finalize-summary.json
+node scripts/finalize-playback-matrix.js --input /path/to/json-results --output-dir /tmp/playback-matrix-final --require-formats mp4,fragmented --target-fps 60 --max-scrub-p95-ms 40 --max-startup-ms 250 --publish-target /workspace/crates/editor/PLAYBACK-BENCHMARKS.md
+
+Finalize summary JSON includes generated timestamp, artifact paths, settings, pass/fail flags, and git branch/commit metadata when available.
+Finalize summary JSON also carries comparison failure reasons and gate outcomes when comparison is enabled.
+Finalize summary JSON includes comparison file stats (including parse error counts) when comparison is enabled.
+Finalize summary JSON includes comparison summary counts (compared rows, regressions, missing/candidate-only/insufficient-sample counts) when comparison is enabled.
+
+# Publish matrix artifacts into this benchmark history
+node scripts/publish-playback-matrix-summary.js --aggregate-md /tmp/playback-matrix-final/playback-benchmark-aggregate.md --status-md /tmp/playback-matrix-final/playback-matrix-status.md --validation-json /tmp/playback-matrix-final/playback-matrix-validation.json --bottlenecks-md /tmp/playback-matrix-final/playback-bottlenecks.md
+node scripts/publish-playback-matrix-summary.js --aggregate-md /tmp/playback-matrix-final/playback-benchmark-aggregate.md --status-md /tmp/playback-matrix-final/playback-matrix-status.md --validation-json /tmp/playback-matrix-final/playback-matrix-validation.json --comparison-md /tmp/playback-matrix-final/playback-comparison.md --comparison-json /tmp/playback-matrix-final/playback-comparison.json
+node scripts/publish-playback-matrix-summary.js --aggregate-md /tmp/playback-matrix-final/playback-benchmark-aggregate.md --status-md /tmp/playback-matrix-final/playback-matrix-status.md --validation-json /tmp/playback-matrix-final/playback-matrix-validation.json --finalize-summary-json /tmp/playback-matrix-final/playback-finalize-summary.json
+
+# Analyze bottlenecks from matrix results
+node scripts/analyze-playback-matrix-bottlenecks.js --input /path/to/json-results --output /tmp/playback-bottlenecks.md --target-fps 60 --max-scrub-p95-ms 40 --max-startup-ms 250
+node scripts/analyze-playback-matrix-bottlenecks.js --input /path/to/json-results --output /tmp/playback-bottlenecks.md --output-json /tmp/playback-bottlenecks.json --target-fps 60 --max-scrub-p95-ms 40 --max-startup-ms 250
+
+# Compare candidate against baseline and flag regressions
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --output /tmp/playback-comparison.md --allow-fps-drop 2 --allow-startup-increase-ms 25 --allow-scrub-p95-increase-ms 5
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results-a --baseline /path/to/baseline-results-b --candidate /path/to/candidate-results-a --candidate /path/to/candidate-results-b --output /tmp/playback-comparison.md
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --allow-missing-candidate
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --fail-on-candidate-only
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --min-samples-per-row 3
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --fail-on-parse-errors
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --fail-on-zero-compared
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --fail-on-skipped-files
+node scripts/compare-playback-benchmark-runs.js --baseline /path/to/baseline-results --candidate /path/to/candidate-results --output /tmp/playback-comparison.md --output-json /tmp/playback-comparison.json
+
+Comparison output reports both baseline rows missing in candidate and candidate-only rows not present in baseline.
+Comparison table also reports baseline/candidate run counts per row when multiple JSON inputs contribute to the same key.
+Comparison JSON summary includes `failureReasons` and `gateOutcomes` fields for automation-friendly gate diagnostics.
+Minimum sample gating is applied against metrics that are actually comparable for the row (for example, scrub samples are not required for rows with no scrub metric comparison).
+Comparison JSON also includes baseline/candidate file parsing stats and optional parse-error gating support.
+Comparison can optionally gate on skipped input files via `--fail-on-skipped-files`.
+Comparison file stats now include skipped-file breakdown for `no_reports` and `no_usable_metrics`.
 ```
 
 #### Decode Performance Benchmark
@@ -62,6 +130,12 @@ cargo run -p cap-editor --example decode-benchmark -- --video /path/to/video.mp4
 
 # With custom FPS and iterations
 cargo run -p cap-editor --example decode-benchmark -- --video /path/to/video.mp4 --fps 60 --iterations 50
+
+# Emit machine-readable JSON with startup/scrub metrics
+cargo run -p cap-editor --example decode-benchmark -- --video /path/to/video.mp4 --fps 60 --sequential-frames 180 --random-samples 120 --output-json /tmp/decode-benchmark.json
+
+# Fragmented segment input is supported by passing the display directory
+cargo run -p cap-editor --example decode-benchmark -- --video /path/to/segment/display --fps 60 --output-json /tmp/decode-benchmark-fragmented.json
 ```
 
 #### Combined Workflow (Recording → Playback)
@@ -74,6 +148,30 @@ cargo run -p cap-recording --example real-device-test-runner -- full --keep-outp
 cargo run -p cap-recording --example playback-test-runner -- full
 ```
 
+### Cross-Platform Validation Matrix
+
+Run these scenarios on each required hardware class and append outputs via `--benchmark-output`.
+
+```bash
+cargo run -p cap-recording --example playback-test-runner -- full --fps 60 --benchmark-output --notes "platform=<platform> gpu=<gpu> scenario=full"
+cargo run -p cap-recording --example playback-test-runner -- scrub --fps 60 --benchmark-output --notes "platform=<platform> gpu=<gpu> scenario=scrub"
+```
+
+Automated helper for machine runs:
+
+```bash
+node scripts/run-playback-benchmark-matrix.js --platform macos-13 --gpu apple-silicon --output-dir /tmp/playback-matrix --fps 60 --startup-threshold-ms 250 --input-dir /tmp/cap-real-device-tests
+node scripts/run-playback-benchmark-matrix.js --platform windows-11 --gpu nvidia-discrete --output-dir /tmp/playback-matrix-windows-nvidia --fps 60 --startup-threshold-ms 250 --require-formats mp4,fragmented --input-dir /tmp/cap-real-device-tests
+node scripts/run-playback-benchmark-matrix.js --platform windows-11 --gpu integrated --output-dir /tmp/playback-matrix-windows-integrated --fps 60 --startup-threshold-ms 250 --scenarios scrub --input-dir /tmp/cap-real-device-tests
+```
+
+| Platform | GPU Class | MP4 Full | Fragmented Full | MP4 Scrub | Fragmented Scrub | Notes |
+|----------|-----------|----------|-----------------|-----------|------------------|-------|
+| macOS 13+ | Apple Silicon | ☐ | ☐ | ☐ | ☐ | |
+| Windows 11 | NVIDIA discrete | ☐ | ☐ | ☐ | ☐ | |
+| Windows 11 | AMD discrete | ☐ | ☐ | ☐ | ☐ | |
+| Windows 11 | Integrated baseline | ☐ | ☐ | ☐ | ☐ | |
+
 ---
 
 ## Benchmark History
@@ -106,6 +204,19 @@ cargo run -p cap-recording --example playback-test-runner -- full
 | **P50/P95/P99** | Latency percentiles | Sorted distribution |
 | **Effective FPS** | Actual decode throughput | frames / elapsed_time |
 | **Jitter** | Decode time variance (std dev) | sqrt(variance) |
+| **First Decode** | Decode latency for first successful frame | elapsed from first frame request |
+| **Startup to First** | Time from playback test start to first decoded frame | elapsed since playback test start |
+
+### Scrub Metrics
+
+| Metric | Description | How Measured |
+|--------|-------------|--------------|
+| **Seek Operations** | Total random seek attempts | Fixed operation count per segment |
+| **Successful Seeks** | Seeks returning a decoded frame | Count of non-None seek decodes |
+| **Failed Seeks** | Seeks returning no frame | Count of None seek decodes |
+| **Avg Seek Time** | Mean random seek decode latency | Avg of seek decode times |
+| **P50/P95/P99 Seek** | Seek latency percentiles | Sorted seek time distribution |
+| **Max Seek Time** | Worst seek decode latency | Max of seek decode times |
 
 ### Audio Sync Metrics
 
@@ -188,5 +299,6 @@ When analyzing benchmark results, focus on:
 ## Related Documentation
 
 - [Recording Benchmarks](../recording/BENCHMARKS.md) - Recording performance tracking
+- [Playback Matrix Runbook](./PLAYBACK-MATRIX-RUNBOOK.md) - Cross-platform evidence collection workflow
 - [cap-rendering/decoder](../rendering/src/decoder.rs) - Decoder implementation
 - [cap-video-decode](../video-decode/) - Platform-specific decoders