Merge pull request #1615 from CapSoftware/cursor/keyboard-and-captions-tracks-8d45

Keyboard and captions tracks

Author: richiemcilroy

Cargo.lock

@@ -22,7 +22,7 @@ use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextPar
 
 pub use cap_project::{CaptionSegment, CaptionSettings, CaptionWord};
 
-use crate::http_client;
+use crate::{general_settings::GeneralSettingsStore, http_client};
 
 #[derive(Debug, Serialize, Deserialize, Type, Clone)]
 pub struct CaptionData {
@@ -529,6 +529,7 @@ fn process_with_whisper(
     audio_path: &PathBuf,
     context: Arc<WhisperContext>,
     language: &str,
+    transcription_hints: &[String],
 ) -> Result<CaptionData, String> {
     log::info!("=== WHISPER TRANSCRIPTION START ===");
     log::info!("Processing audio file: {audio_path:?}");
@@ -544,6 +545,10 @@ fn process_with_whisper(
     params.set_language(Some(if language == "auto" { "auto" } else { language }));
     params.set_max_len(i32::MAX);
 
+    if let Some(initial_prompt) = build_initial_prompt(transcription_hints) {
+        params.set_initial_prompt(&initial_prompt);
+    }
+
     log::info!("Whisper params - translate: false, token_timestamps: true, max_len: MAX");
 
     let mut audio_file = File::open(audio_path)
@@ -783,10 +788,32 @@ fn process_with_whisper(
     })
 }
 
+fn build_initial_prompt(transcription_hints: &[String]) -> Option<String> {
+    let mut normalized = Vec::new();
+
+    for hint in transcription_hints {
+        let value = hint.replace('\0', "").trim().to_string();
+        if value.is_empty() || normalized.contains(&value) {
+            continue;
+        }
+        normalized.push(value);
+    }
+
+    if normalized.is_empty() {
+        None
+    } else {
+        Some(format!(
+            "Preferred spellings, names, and capitalization for this transcript: {}",
+            normalized.join("; ")
+        ))
+    }
+}
+
 #[tauri::command]
 #[specta::specta]
 #[instrument]
 pub async fn transcribe_audio(
+    app: AppHandle,
     video_path: String,
     model_path: String,
     language: String,
@@ -843,11 +870,18 @@ pub async fn transcribe_audio(
         }
     };
 
+    let transcription_hints = GeneralSettingsStore::get(&app)
+        .ok()
+        .flatten()
+        .map(|settings| settings.transcription_hints)
+        .unwrap_or_default();
+
     log::info!("Starting Whisper transcription in blocking task...");
-    let whisper_result =
-        tokio::task::spawn_blocking(move || process_with_whisper(&audio_path, context, &language))
-            .await
-            .map_err(|e| format!("Whisper task panicked: {e}"))?;
+    let whisper_result = tokio::task::spawn_blocking(move || {
+        process_with_whisper(&audio_path, context, &language, &transcription_hints)
+    })
+    .await
+    .map_err(|e| format!("Whisper task panicked: {e}"))?;
 
     match whisper_result {
         Ok(captions) => {

apps/desktop/src-tauri/src/captions.rs

@@ -293,6 +293,7 @@ pub async fn generate_export_preview(
         .iter()
         .map(|s| RenderSegment {
             cursor: s.cursor.clone(),
+            keyboard: s.keyboard.clone(),
             decoders: s.decoders.clone(),
         })
         .collect();

apps/desktop/src-tauri/src/export.rs

@@ -121,8 +121,10 @@ pub struct GeneralSettingsStore {
         skip_serializing_if = "no"
     )]
     pub enable_native_camera_preview: bool,
-    #[serde(default)]
+    #[serde(default = "default_true")]
     pub auto_zoom_on_clicks: bool,
+    #[serde(default = "default_true")]
+    pub capture_keyboard_events: bool,
     #[serde(default)]
     pub post_deletion_behaviour: PostDeletionBehaviour,
     #[serde(default = "default_excluded_windows")]
@@ -137,6 +139,8 @@ pub struct GeneralSettingsStore {
     pub crash_recovery_recording: bool,
     #[serde(default = "default_max_fps")]
     pub max_fps: u32,
+    #[serde(default = "default_transcription_hints")]
+    pub transcription_hints: Vec<String>,
     #[serde(default)]
     pub editor_preview_quality: EditorPreviewQuality,
     #[serde(default)]
@@ -167,6 +171,15 @@ fn default_max_fps() -> u32 {
     60
 }
 
+fn default_transcription_hints() -> Vec<String> {
+    vec![
+        "Cap".to_string(),
+        "TypeScript".to_string(),
+        "My Brand Name".to_string(),
+        "mywebsite.com".to_string(),
+    ]
+}
+
 fn default_server_url() -> String {
     std::option_env!("VITE_SERVER_URL")
         .unwrap_or("https://cap.so")
@@ -202,14 +215,16 @@ impl Default for GeneralSettingsStore {
             server_url: default_server_url(),
             recording_countdown: Some(3),
             enable_native_camera_preview: default_enable_native_camera_preview(),
-            auto_zoom_on_clicks: false,
+            auto_zoom_on_clicks: true,
+            capture_keyboard_events: true,
             post_deletion_behaviour: PostDeletionBehaviour::DoNothing,
             excluded_windows: default_excluded_windows(),
             delete_instant_recordings_after_upload: false,
             instant_mode_max_resolution: 1920,
             default_project_name_template: None,
             crash_recovery_recording: true,
             max_fps: 60,
+            transcription_hints: default_transcription_hints(),
             editor_preview_quality: EditorPreviewQuality::Half,
             main_window_position: None,
             camera_window_position: None,

apps/desktop/src-tauri/src/general_settings.rs

@@ -506,6 +506,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result<
                     mic: None,
                     system_audio: None,
                     cursor: None,
+                    keyboard: None,
                 }],
                 cursors: Cursors::default(),
                 status: Some(StudioRecordingStatus::InProgress),
@@ -599,6 +600,7 @@ pub async fn start_video_import(app: AppHandle, source_path: PathBuf) -> Result<
                                     mic: None,
                                     system_audio,
                                     cursor: None,
+                                    keyboard: None,
                                 }],
                                 cursors: Cursors::default(),
                                 status: Some(StudioRecordingStatus::Complete),

apps/desktop/src-tauri/src/import.rs

@@ -2123,6 +2123,51 @@ async fn generate_zoom_segments_from_clicks(
     Ok(zoom_segments)
 }
 
+#[tauri::command]
+#[specta::specta]
+#[instrument(skip(editor_instance))]
+async fn generate_keyboard_segments(
+    editor_instance: WindowEditorInstance,
+    grouping_threshold_ms: f64,
+    linger_duration_ms: f64,
+    show_modifiers: bool,
+    show_special_keys: bool,
+) -> Result<Vec<cap_project::KeyboardTrackSegment>, String> {
+    let meta = editor_instance.meta();
+
+    let RecordingMetaInner::Studio(studio_meta) = &meta.inner else {
+        return Ok(vec![]);
+    };
+
+    let segments = match studio_meta.as_ref() {
+        StudioRecordingMeta::MultipleSegments { inner, .. } => &inner.segments,
+        _ => return Ok(vec![]),
+    };
+
+    let mut all_events = cap_project::KeyboardEvents { presses: vec![] };
+
+    for segment in segments {
+        let events = segment.keyboard_events(meta);
+        all_events.presses.extend(events.presses);
+    }
+
+    all_events.presses.sort_by(|a, b| {
+        a.time_ms
+            .partial_cmp(&b.time_ms)
+            .unwrap_or(std::cmp::Ordering::Equal)
+    });
+
+    let grouped = cap_project::group_key_events(
+        &all_events,
+        grouping_threshold_ms,
+        linger_duration_ms,
+        show_modifiers,
+        show_special_keys,
+    );
+
+    Ok(grouped)
+}
+
 #[tauri::command]
 #[specta::specta]
 #[instrument]
@@ -3105,6 +3150,7 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) {
             set_project_config,
             update_project_config_in_memory,
             generate_zoom_segments_from_clicks,
+            generate_keyboard_segments,
             permissions::open_permission_settings,
             permissions::do_permissions_check,
             permissions::request_permission,
@@ -3673,13 +3719,14 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) {
                                             id,
                                             CapWindowId::TargetSelectOverlay { .. }
                                                 | CapWindowId::Main
-                                                | CapWindowId::Camera
                                         )
                                     {
                                         let _ = window.show();
                                     }
                                 }
 
+                                restore_camera_window(app);
+
                                 #[cfg(target_os = "windows")]
                                 if !has_open_editor_window(app) {
                                     reopen_main_window(app);
@@ -3694,12 +3741,12 @@ pub async fn run(recording_logging_handle: LoggingHandle, logs_dir: PathBuf) {
                                             id,
                                             CapWindowId::TargetSelectOverlay { .. }
                                                 | CapWindowId::Main
-                                                | CapWindowId::Camera
                                         )
                                     {
                                         let _ = window.show();
                                     }
                                 }
+                                restore_camera_window(app);
                                 return;
                             }
                             CapWindowId::TargetSelectOverlay { display_id } => {
@@ -3901,9 +3948,25 @@ fn restore_main_windows_if_no_editors(app: &AppHandle) {
         if let Some(main) = CapWindowId::Main.get(app) {
             let _ = main.show();
         }
-        if let Some(camera) = CapWindowId::Camera.get(app) {
-            let _ = camera.show();
-        }
+
+        restore_camera_window(app);
+    }
+}
+
+fn restore_camera_window(app: &AppHandle) {
+    let should_restore_camera = app
+        .state::<ArcLock<App>>()
+        .try_read()
+        .map(|state| state.selected_camera_id.is_some())
+        .unwrap_or(false);
+
+    if should_restore_camera {
+        let app = app.clone();
+        tokio::spawn(async move {
+            let operation_lock = app.state::<CameraWindowOperationLock>();
+            let _operation_guard = operation_lock.lock().await;
+            let _ = ShowCapWindow::Camera { centered: false }.show(&app).await;
+        });
     }
 }
 

apps/desktop/src-tauri/src/lib.rs

@@ -878,6 +878,12 @@ pub async fn start_recording(
                                     .map(|s| s.custom_cursor_capture)
                                     .unwrap_or_default(),
                             )
+                            .with_keyboard_capture(
+                                general_settings
+                                    .as_ref()
+                                    .map(|s| s.capture_keyboard_events)
+                                    .unwrap_or(true),
+                            )
                             .with_fragmented(
                                 general_settings
                                     .as_ref()
@@ -2370,6 +2376,8 @@ fn project_config_from_recording(
         scene_segments: Vec::new(),
         mask_segments: Vec::new(),
         text_segments: Vec::new(),
+        caption_segments: Vec::new(),
+        keyboard_segments: Vec::new(),
     });
 
     config
@@ -2391,7 +2399,7 @@ pub fn needs_fragment_remux(recording_dir: &Path, meta: &StudioRecordingMeta) ->
 }
 
 pub fn remux_fragmented_recording(recording_dir: &Path) -> Result<(), String> {
-    let incomplete_recording = RecoveryManager::find_incomplete_single(recording_dir);
+    let incomplete_recording = RecoveryManager::inspect_recording(recording_dir);
 
     if let Some(recording) = incomplete_recording {
         RecoveryManager::recover(&recording)