perf(rendering): NV12 stride alignment and shared pipelines

Author: richiemcilroy

crates/rendering/src/frame_pipeline.rs

@@ -17,6 +17,7 @@ pub struct RgbaToNv12Converter {
     pending: Option<PendingNv12Readback>,
     cached_width: u32,
     cached_height: u32,
+    cached_stride: u32,
     cached_bind_groups: Option<[wgpu::BindGroup; 2]>,
     cached_texture_view: Option<wgpu::TextureView>,
     cached_texture_ptr: usize,
@@ -108,20 +109,31 @@ impl RgbaToNv12Converter {
             pending: None,
             cached_width: 0,
             cached_height: 0,
+            cached_stride: 0,
             cached_bind_groups: None,
             cached_texture_view: None,
             cached_texture_ptr: 0,
         }
     }
 
+    fn aligned_stride(width: u32) -> u32 {
+        (width + 3) & !3
+    }
+
     fn nv12_size(width: u32, height: u32) -> u64 {
-        let y_size = (width as u64) * (height as u64);
-        let uv_size = (width as u64) * (height as u64 / 2);
+        let stride = Self::aligned_stride(width) as u64;
+        let aligned_height = ((height + 1) & !1) as u64;
+        let y_size = stride * aligned_height;
+        let uv_size = stride * (aligned_height / 2);
         y_size + uv_size
     }
 
     fn ensure_buffers(&mut self, device: &wgpu::Device, width: u32, height: u32) {
-        if self.cached_width == width && self.cached_height == height {
+        let stride = Self::aligned_stride(width);
+        if self.cached_width == width
+            && self.cached_height == height
+            && self.cached_stride == stride
+        {
             return;
         }
 
@@ -148,6 +160,7 @@ impl RgbaToNv12Converter {
         self.current_readback = 0;
         self.cached_width = width;
         self.cached_height = height;
+        self.cached_stride = stride;
         self.cached_bind_groups = None;
         self.cached_texture_view = None;
         self.cached_texture_ptr = 0;
@@ -165,7 +178,7 @@ impl RgbaToNv12Converter {
         frame_number: u32,
         frame_rate: u32,
     ) -> bool {
-        if width == 0 || height == 0 || !width.is_multiple_of(4) || !height.is_multiple_of(2) {
+        if width == 0 || height == 0 {
             return false;
         }
 
@@ -182,8 +195,8 @@ impl RgbaToNv12Converter {
         };
         self.current_readback = 1 - self.current_readback;
 
-        let y_stride = width;
-        let uv_stride = width;
+        let y_stride = Self::aligned_stride(width);
+        let uv_stride = Self::aligned_stride(width);
 
         let params = Nv12Params {
             width,

crates/rendering/src/layers/camera.rs

@@ -21,6 +21,7 @@ pub struct CameraLayer {
 }
 
 impl CameraLayer {
+    #[allow(dead_code)]
     pub fn new(device: &wgpu::Device) -> Self {
         Self::new_with_all_shared_pipelines(
             device,

crates/rendering/src/layers/display.rs

@@ -1,9 +1,11 @@
 use cap_project::XY;
 
+use std::sync::Arc;
+
 use crate::{
     DecodedSegmentFrames, PixelFormat,
     composite_frame::{CompositeVideoFramePipeline, CompositeVideoFrameUniforms},
-    yuv_converter::YuvToRgbaConverter,
+    yuv_converter::{YuvConverterPipelines, YuvToRgbaConverter},
 };
 
 struct PendingTextureCopy {
@@ -17,7 +19,7 @@ pub struct DisplayLayer {
     frame_texture_views: [wgpu::TextureView; 2],
     current_texture: usize,
     uniforms_buffer: wgpu::Buffer,
-    pipeline: CompositeVideoFramePipeline,
+    pipeline: std::sync::Arc<CompositeVideoFramePipeline>,
     bind_groups: [Option<wgpu::BindGroup>; 2],
     last_recording_time: Option<f32>,
     yuv_converter: YuvToRgbaConverter,
@@ -32,19 +34,32 @@ impl DisplayLayer {
     }
 
     pub fn new_with_options(device: &wgpu::Device, prefer_cpu_conversion: bool) -> Self {
+        Self::new_with_all_shared_pipelines(
+            device,
+            Arc::new(YuvConverterPipelines::new(device)),
+            Arc::new(CompositeVideoFramePipeline::new(device)),
+            prefer_cpu_conversion,
+        )
+    }
+
+    pub fn new_with_all_shared_pipelines(
+        device: &wgpu::Device,
+        yuv_pipelines: Arc<YuvConverterPipelines>,
+        composite_pipeline: Arc<CompositeVideoFramePipeline>,
+        prefer_cpu_conversion: bool,
+    ) -> Self {
         let frame_texture_0 = CompositeVideoFramePipeline::create_frame_texture(device, 1920, 1080);
         let frame_texture_1 = CompositeVideoFramePipeline::create_frame_texture(device, 1920, 1080);
         let frame_texture_view_0 = frame_texture_0.create_view(&Default::default());
         let frame_texture_view_1 = frame_texture_1.create_view(&Default::default());
 
         let uniforms_buffer = CompositeVideoFrameUniforms::default().to_buffer(device);
-        let pipeline = CompositeVideoFramePipeline::new(device);
         let bind_group_0 =
-            Some(pipeline.bind_group(device, &uniforms_buffer, &frame_texture_view_0));
+            Some(composite_pipeline.bind_group(device, &uniforms_buffer, &frame_texture_view_0));
         let bind_group_1 =
-            Some(pipeline.bind_group(device, &uniforms_buffer, &frame_texture_view_1));
+            Some(composite_pipeline.bind_group(device, &uniforms_buffer, &frame_texture_view_1));
 
-        let yuv_converter = YuvToRgbaConverter::new(device);
+        let yuv_converter = YuvToRgbaConverter::new_with_shared_pipelines(device, yuv_pipelines);
 
         if prefer_cpu_conversion {
             tracing::info!("DisplayLayer initialized with CPU YUV conversion preference");
@@ -55,7 +70,7 @@ impl DisplayLayer {
             frame_texture_views: [frame_texture_view_0, frame_texture_view_1],
             current_texture: 0,
             uniforms_buffer,
-            pipeline,
+            pipeline: composite_pipeline,
             bind_groups: [bind_group_0, bind_group_1],
             last_recording_time: None,
             yuv_converter,
@@ -64,6 +79,55 @@ impl DisplayLayer {
         }
     }
 
+    #[cfg(target_os = "windows")]
+    fn try_d3d11_staging_fallback(
+        &mut self,
+        device: &wgpu::Device,
+        queue: &wgpu::Queue,
+        screen_frame: &crate::DecodedFrame,
+        actual_width: u32,
+        actual_height: u32,
+        next_texture: usize,
+    ) -> bool {
+        let Some(nv12_texture) = screen_frame.d3d11_texture_backing() else {
+            return false;
+        };
+
+        let Ok(d3d11_device) = (unsafe { nv12_texture.GetDevice() }) else {
+            return false;
+        };
+
+        let Ok(d3d11_context) = (unsafe { d3d11_device.GetImmediateContext() }) else {
+            return false;
+        };
+
+        if self
+            .yuv_converter
+            .convert_nv12_with_fallback(
+                device,
+                queue,
+                &d3d11_device,
+                &d3d11_context,
+                nv12_texture,
+                screen_frame.d3d11_y_handle(),
+                screen_frame.d3d11_uv_handle(),
+                actual_width,
+                actual_height,
+            )
+            .is_ok()
+            && self.yuv_converter.output_texture().is_some()
+        {
+            self.pending_copy = Some(PendingTextureCopy {
+                width: actual_width,
+                height: actual_height,
+                dst_texture_index: next_texture,
+            });
+            true
+        } else {
+            false
+        }
+    }
+
     pub fn prepare(
         &mut self,
         device: &wgpu::Device,
@@ -282,7 +346,14 @@ impl DisplayLayer {
                                 false
                             }
                         } else {
-                            false
+                            self.try_d3d11_staging_fallback(
+                                device,
+                                queue,
+                                screen_frame,
+                                actual_width,
+                                actual_height,
+                                next_texture,
+                            )
                         }
                     }
 
@@ -541,7 +612,14 @@ impl DisplayLayer {
                                 Err(_) => false,
                             }
                         } else {
-                            false
+                            self.try_d3d11_staging_fallback(
+                                device,
+                                queue,
+                                screen_frame,
+                                actual_width,
+                                actual_height,
+                                next_texture,
+                            )
                         }
                     } else if let (Some(y_data), Some(uv_data)) =
                         (screen_frame.y_plane(), screen_frame.uv_plane())
@@ -575,7 +653,14 @@ impl DisplayLayer {
                             Err(_) => false,
                         }
                     } else {
-                        false
+                        self.try_d3d11_staging_fallback(
+                            device,
+                            queue,
+                            screen_frame,
+                            actual_width,
+                            actual_height,
+                            next_texture,
+                        )
                     }
 
                     #[cfg(not(target_os = "windows"))]

crates/rendering/src/lib.rs

@@ -1455,7 +1455,7 @@ impl ProjectUniforms {
         let height_scale = resolution_base.y as f32 / base_height as f32;
         let scale = width_scale.min(height_scale);
 
-        let scaled_width = ((base_width as f32 * scale) as u32 + 1) & !1;
+        let scaled_width = ((base_width as f32 * scale) as u32 + 3) & !3;
         let scaled_height = ((base_height as f32 * scale) as u32 + 1) & !1;
         (scaled_width, scaled_height)
     }
@@ -2320,6 +2320,24 @@ impl<'a> FrameRenderer<'a> {
         }
     }
 
+    pub async fn render_immediate_nv12(
+        &mut self,
+        segment_frames: DecodedSegmentFrames,
+        uniforms: ProjectUniforms,
+        cursor: &CursorEvents,
+        layers: &mut RendererLayers,
+    ) -> Result<frame_pipeline::Nv12RenderedFrame, RenderingError> {
+        if let Some(frame) = self
+            .render_nv12(segment_frames, uniforms, cursor, layers)
+            .await?
+        {
+            return Ok(frame);
+        }
+        self.flush_pipeline_nv12()
+            .await
+            .unwrap_or(Err(RenderingError::BufferMapWaitingFailed))
+    }
+
     pub async fn flush_pipeline_nv12(
         &mut self,
     ) -> Option<Result<frame_pipeline::Nv12RenderedFrame, RenderingError>> {
@@ -2443,13 +2461,30 @@ impl RendererLayers {
         queue: &wgpu::Queue,
         prefer_cpu_conversion: bool,
     ) -> Self {
+        let shared_yuv_pipelines = Arc::new(yuv_converter::YuvConverterPipelines::new(device));
+        let shared_composite_pipeline =
+            Arc::new(composite_frame::CompositeVideoFramePipeline::new(device));
+
         Self {
             background: BackgroundLayer::new(device),
             background_blur: BlurLayer::new(device),
-            display: DisplayLayer::new_with_options(device, prefer_cpu_conversion),
+            display: DisplayLayer::new_with_all_shared_pipelines(
+                device,
+                shared_yuv_pipelines.clone(),
+                shared_composite_pipeline.clone(),
+                prefer_cpu_conversion,
+            ),
             cursor: CursorLayer::new(device),
-            camera: CameraLayer::new(device),
-            camera_only: CameraLayer::new(device),
+            camera: CameraLayer::new_with_all_shared_pipelines(
+                device,
+                shared_yuv_pipelines.clone(),
+                shared_composite_pipeline.clone(),
+            ),
+            camera_only: CameraLayer::new_with_all_shared_pipelines(
+                device,
+                shared_yuv_pipelines,
+                shared_composite_pipeline,
+            ),
             mask: MaskLayer::new(device),
             text: TextLayer::new(device, queue),
             captions: CaptionsLayer::new(device, queue),