perf(editor): NV12 output and non-blocking render

Author: richiemcilroy

apps/desktop/src-tauri/src/editor_window.rs

@@ -3,6 +3,8 @@ use tauri::{AppHandle, Manager, Runtime, Window, ipc::CommandArg};
 use tokio::sync::{RwLock, watch};
 use tokio_util::sync::CancellationToken;
 
+use cap_rendering::GpuOutputFormat;
+
 use crate::{
     create_editor_instance_impl,
     frame_ws::{WSFrame, WSFrameFormat, create_watch_frame_ws},
@@ -29,16 +31,22 @@ async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult {
         path,
         Box::new(move |output| {
             let ws_frame = match output {
-                cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
-                    data: frame.data,
-                    width: frame.width,
-                    height: frame.height,
-                    stride: frame.y_stride,
-                    frame_number: frame.frame_number,
-                    target_time_ns: frame.target_time_ns,
-                    format: WSFrameFormat::Nv12,
-                    created_at: Instant::now(),
-                },
+                cap_editor::EditorFrameOutput::Nv12(frame) => {
+                    let ws_format = match frame.format {
+                        GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
+                        GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
+                    };
+                    WSFrame {
+                        data: frame.data,
+                        width: frame.width,
+                        height: frame.height,
+                        stride: frame.y_stride,
+                        frame_number: frame.frame_number,
+                        target_time_ns: frame.target_time_ns,
+                        format: ws_format,
+                        created_at: Instant::now(),
+                    }
+                }
                 cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
                     data: frame.data,
                     width: frame.width,
@@ -234,16 +242,22 @@ impl EditorInstances {
                     path,
                     Box::new(move |output| {
                         let ws_frame = match output {
-                            cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
-                                data: frame.data,
-                                width: frame.width,
-                                height: frame.height,
-                                stride: frame.y_stride,
-                                frame_number: frame.frame_number,
-                                target_time_ns: frame.target_time_ns,
-                                format: WSFrameFormat::Nv12,
-                                created_at: Instant::now(),
-                            },
+                            cap_editor::EditorFrameOutput::Nv12(frame) => {
+                                let ws_format = match frame.format {
+                                    GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
+                                    GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
+                                };
+                                WSFrame {
+                                    data: frame.data,
+                                    width: frame.width,
+                                    height: frame.height,
+                                    stride: frame.y_stride,
+                                    frame_number: frame.frame_number,
+                                    target_time_ns: frame.target_time_ns,
+                                    format: ws_format,
+                                    created_at: Instant::now(),
+                                }
+                            }
                             cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
                                 data: frame.data,
                                 width: frame.width,

crates/editor/src/editor.rs

@@ -148,7 +148,7 @@ impl Renderer {
                 }
             }
             match frame_renderer
-                .render_immediate(
+                .render_immediate_nv12(
                     current.segment_frames,
                     current.uniforms,
                     &current.cursor,
@@ -157,7 +157,7 @@ impl Renderer {
                 .await
             {
                 Ok(frame) => {
-                    (self.frame_cb)(EditorFrameOutput::Rgba(frame));
+                    (self.frame_cb)(EditorFrameOutput::Nv12(frame));
                 }
                 Err(e) => {
                     tracing::error!(error = %e, "Failed to render frame in editor");
@@ -170,25 +170,20 @@ impl Renderer {
 }
 
 impl RendererHandle {
-    async fn send(&self, msg: RendererMessage) {
-        let _ = self.tx.send(msg).await;
-    }
-
-    pub async fn render_frame(
+    pub fn render_frame(
         &self,
         segment_frames: DecodedSegmentFrames,
         uniforms: ProjectUniforms,
         cursor: Arc<CursorEvents>,
     ) {
         let (finished_tx, _finished_rx) = oneshot::channel();
 
-        self.send(RendererMessage::RenderFrame {
+        let _ = self.tx.try_send(RendererMessage::RenderFrame {
             segment_frames,
             uniforms,
             finished: finished_tx,
             cursor,
-        })
-        .await;
+        });
     }
 
     pub async fn stop(&self) {

crates/editor/src/editor_instance.rs

@@ -532,8 +532,7 @@ impl EditorInstance {
                                     &zoom_focus_interpolator,
                                 );
                                 self.renderer
-                                    .render_frame(segment_frames, uniforms, segment_medias.cursor.clone())
-                                    .await;
+                                    .render_frame(segment_frames, uniforms, segment_medias.cursor.clone());
                             } else {
                                 warn!("Preview renderer: no frames returned for frame {}", frame_number);
                             }