Skip to content

Commit 4186535

Browse files
committed
Sync system audio start_time; minor rust refactors
1 parent a451f83 commit 4186535

File tree

4 files changed

+114
-18
lines changed

4 files changed

+114
-18
lines changed

apps/desktop/src-tauri/src/windows.rs

Lines changed: 13 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -75,10 +75,9 @@ fn is_system_dark_mode() -> bool {
7575
let hkcu = RegKey::predef(HKEY_CURRENT_USER);
7676
if let Ok(key) =
7777
hkcu.open_subkey("Software\\Microsoft\\Windows\\CurrentVersion\\Themes\\Personalize")
78+
&& let Ok(value) = key.get_value::<u32, _>("AppsUseLightTheme")
7879
{
79-
if let Ok(value) = key.get_value::<u32, _>("AppsUseLightTheme") {
80-
return value == 0;
81-
}
80+
return value == 0;
8281
}
8382
false
8483
}
@@ -858,17 +857,17 @@ impl ShowCapWindow {
858857
}
859858

860859
#[cfg(not(target_os = "macos"))]
861-
if let Self::InProgressRecording { .. } = self {
862-
if let Some(window) = self.id(app).get(app) {
863-
let width = 320.0;
864-
let height = 150.0;
865-
let recording_monitor = CursorMonitorInfo::get();
866-
let (pos_x, pos_y) = recording_monitor.bottom_center_position(width, height, 120.0);
867-
let _ = window.set_position(tauri::LogicalPosition::new(pos_x, pos_y));
868-
window.show().ok();
869-
window.set_focus().ok();
870-
return Ok(window);
871-
}
860+
if let Self::InProgressRecording { .. } = self
861+
&& let Some(window) = self.id(app).get(app)
862+
{
863+
let width = 320.0;
864+
let height = 150.0;
865+
let recording_monitor = CursorMonitorInfo::get();
866+
let (pos_x, pos_y) = recording_monitor.bottom_center_position(width, height, 120.0);
867+
let _ = window.set_position(tauri::LogicalPosition::new(pos_x, pos_y));
868+
window.show().ok();
869+
window.set_focus().ok();
870+
return Ok(window);
872871
}
873872

874873
if !matches!(self, Self::Camera { .. } | Self::InProgressRecording { .. })

crates/editor/PLAYBACK-FINDINGS.md

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,42 @@ The CPU RGBA→NV12 conversion was taking 15-25ms per frame for 3024x1964 resolu
355355

356356
---
357357

358+
### Session 2026-02-15 (Playback Validation + System Audio Sync)
359+
360+
**Goal**: Comprehensive playback benchmark validation, system audio start_time sync fix
361+
362+
**What was done**:
363+
1. Ran playback validation on fragmented and MP4 recordings
364+
2. Verified AVAssetReader graceful fallback on directory paths (no panics)
365+
3. Audited all decoder `unwrap()` calls for safety
366+
4. Added system audio to recording start_time sync chain (studio_recording.rs)
367+
368+
**Changes Made**:
369+
- `crates/recording/src/studio_recording.rs`: System audio start_time now syncs to mic (or display) when drift >30ms, matching the existing camera/display sync pattern. Improves playback alignment.
370+
371+
**Results (MP4 Mode)**:
372+
- ✅ Decoder: AVAssetReader (hardware), display init=162-174ms, camera init=21-32ms
373+
- ✅ Playback: 283-641 fps effective (target ≥60fps)
374+
- ✅ Latency: avg=1.6-3.5ms, p95=2.8-5.0ms (target p95 <50ms)
375+
- ✅ Camera sync: 0ms drift (target <100ms)
376+
- ✅ Mic sync: 93ms (target <100ms)
377+
- 🟡 System audio: 178-195ms (inherent macOS capture latency, sync fix improves alignment)
378+
379+
**Results (Fragmented Mode)**:
380+
- ✅ Decoder: FFmpeg (hardware) with VideoToolbox, display init=100ms, camera init=7ms
381+
- ✅ Playback: 156 fps effective (target ≥60fps)
382+
- ✅ Latency: avg=6.4ms, p95=9.5ms (target p95 <50ms)
383+
- ✅ Camera sync: 0ms drift (target <100ms)
384+
- ✅ Mic sync: 8.5ms (target <100ms)
385+
- ✅ System audio: 98ms (target <100ms)
386+
- ✅ AVAssetReader cleanly falls back to FFmpeg with descriptive error message
387+
388+
**Decoder audit**: All `unwrap()` in `avassetreader.rs` eliminated. Remaining `unwrap()` calls in ffmpeg.rs and avassetreader decoder loop are on guaranteed-non-empty BTreeMap caches (safe by construction).
389+
390+
**Stopping point**: All playback metrics healthy. System audio sync metadata fix applied.
391+
392+
---
393+
358394
## References
359395

360396
- `PLAYBACK-BENCHMARKS.md` - Raw performance test data (auto-updated by test runner)

crates/recording/FINDINGS.md

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -441,6 +441,49 @@ System Audio ────┘ ├─► MP4 (macos.rs) ─
441441

442442
---
443443

444+
### Session 2026-02-15 (Fix Attempts + System Audio Sync)
445+
446+
**Goal**: Fix known issues: MP4 encoder warmup dropped frames and system audio timing offset
447+
448+
**What was done**:
449+
1. Ran comprehensive benchmarks (MP4 cold, warm, thermal stress; fragmented)
450+
2. Attempted encoder warmup patience fix (increasing retry budget from 50ms to 200ms during first 3 frames)
451+
3. Reverted encoder warmup fix after it degraded performance (longer blocking caused pipeline backpressure)
452+
4. Implemented system audio start_time sync to match mic/display sync chain
453+
5. Verified all metrics stable after changes
454+
455+
**Changes Made**:
456+
- `crates/recording/src/studio_recording.rs`: Added system audio to the start_time sync chain. System audio now syncs to mic start time (preferred) or display start time when drift >30ms, matching the existing sync pattern for camera and display. Improves playback alignment of system audio.
457+
458+
**Encoder Warmup Investigation**:
459+
- Root cause: VideoToolbox hardware encoder first-frame latency (~160ms) causes `NotReadyForMore` for frames 2-5
460+
- Current retry budget: 100 × 500μs = 50ms. Frames during warmup are dropped after 50ms retry
461+
- Attempted fix: 400 × 500μs = 200ms patience for first 3 frames
462+
- Result: WORSE (71 frames instead of 149). Longer blocking prevented the encoder thread from draining the channel, causing capture-side drops from channel full
463+
- Conclusion: 50ms retry timeout is the correct safety valve. The ~3% dropped frames during warmup is the optimal tradeoff. Pre-warming the hardware encoder would require architectural changes (dummy frame encoding before recording starts)
464+
465+
**Results (MP4 - warm run, post system audio sync fix)**:
466+
- ✅ Frame rate: 29.0-29.2fps (target 30±2fps)
467+
- ✅ Jitter: 10.3-12.4ms (target <15ms)
468+
- ✅ A/V sync: 0ms across all streams (target <50ms)
469+
- ✅ Mic timing: 90-94ms (target <100ms)
470+
- 🟡 Dropped frames: 2.7-3.3% (encoder warmup, not actionable without architectural changes)
471+
- 🟡 System audio duration: 215-259ms shorter than video (inherent macOS capture latency, cannot be fixed with metadata sync)
472+
473+
**Results (Fragmented)**:
474+
- ✅ Frame rate: 29.5fps, jitter: 5.7ms, dropped: 1.3%
475+
- ✅ Mic timing: 13.5ms
476+
- 🟡 System audio duration: 111.5ms shorter
477+
478+
**Key findings**:
479+
- MP4 encoder warmup spike is NOT fixable by increasing retry patience (makes it worse)
480+
- System audio file duration is inherently shorter due to macOS ScreenCaptureKit capture latency
481+
- System audio start_time metadata sync improves playback alignment but not duration measurement
482+
483+
**Stopping point**: System audio sync metadata fix applied. Encoder warmup spike documented as architectural limitation.
484+
485+
---
486+
444487
## References
445488

446489
- `BENCHMARKS.md` - Raw performance test data (auto-updated by test runner)

crates/recording/src/studio_recording.rs

Lines changed: 22 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -762,10 +762,28 @@ async fn stop_recording(
762762
start_time: mic_start_time,
763763
device_id: s.mic_device_id.clone(),
764764
}),
765-
system_audio: s.pipeline.system_audio.map(|audio| AudioMeta {
766-
path: make_relative(&audio.path),
767-
start_time: Some(to_start_time(audio.first_timestamp)),
768-
device_id: None,
765+
system_audio: s.pipeline.system_audio.map(|audio| {
766+
let raw_sys_start = to_start_time(audio.first_timestamp);
767+
let sys_start_time = if let Some(mic_start) = mic_start_time {
768+
let sync_offset = raw_sys_start - mic_start;
769+
if sync_offset.abs() > 0.030 {
770+
mic_start
771+
} else {
772+
raw_sys_start
773+
}
774+
} else {
775+
let sync_offset = raw_sys_start - display_start_time;
776+
if sync_offset.abs() > 0.030 {
777+
display_start_time
778+
} else {
779+
raw_sys_start
780+
}
781+
};
782+
AudioMeta {
783+
path: make_relative(&audio.path),
784+
start_time: Some(sys_start_time),
785+
device_id: None,
786+
}
769787
}),
770788
cursor: s
771789
.pipeline

0 commit comments

Comments
 (0)