Merge pull request #1616 from CapSoftware/windows-playback-perf

Windows playback perf

Author: richiemcilroy

Cargo.lock

@@ -108,6 +108,7 @@ cidre = { git = "https://github.com/CapSoftware/cidre", rev = "bf84b67079a8" }
 # https://github.com/gfx-rs/wgpu/pull/7550
 # wgpu = { git = "https://github.com/gfx-rs/wgpu", rev = "cd41a6e32a6239b65d1cecbeccde6a43a100914a" }
 wgpu-hal = { path = "vendor/wgpu-hal" }
+tao = { path = "vendor/tao" }
 
 # https://github.com/CapSoftware/posthog-rs/commit/c7e9712be2f9a9122b1df685d5a067afa5415288
 posthog-rs = { git = "https://github.com/CapSoftware/posthog-rs", rev = "c7e9712be2f9a9122b1df685d5a067afa5415288" }

Cargo.toml

@@ -3,6 +3,8 @@ use tauri::{AppHandle, Manager, Runtime, Window, ipc::CommandArg};
 use tokio::sync::{RwLock, watch};
 use tokio_util::sync::CancellationToken;
 
+use cap_rendering::GpuOutputFormat;
+
 use crate::{
     create_editor_instance_impl,
     frame_ws::{WSFrame, WSFrameFormat, create_watch_frame_ws},
@@ -29,16 +31,22 @@ async fn do_prewarm(app: AppHandle, path: PathBuf) -> PendingResult {
         path,
         Box::new(move |output| {
             let ws_frame = match output {
-                cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
-                    data: frame.data,
-                    width: frame.width,
-                    height: frame.height,
-                    stride: frame.y_stride,
-                    frame_number: frame.frame_number,
-                    target_time_ns: frame.target_time_ns,
-                    format: WSFrameFormat::Nv12,
-                    created_at: Instant::now(),
-                },
+                cap_editor::EditorFrameOutput::Nv12(frame) => {
+                    let ws_format = match frame.format {
+                        GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
+                        GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
+                    };
+                    WSFrame {
+                        data: frame.data,
+                        width: frame.width,
+                        height: frame.height,
+                        stride: frame.y_stride,
+                        frame_number: frame.frame_number,
+                        target_time_ns: frame.target_time_ns,
+                        format: ws_format,
+                        created_at: Instant::now(),
+                    }
+                }
                 cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
                     data: frame.data,
                     width: frame.width,
@@ -234,16 +242,22 @@ impl EditorInstances {
                     path,
                     Box::new(move |output| {
                         let ws_frame = match output {
-                            cap_editor::EditorFrameOutput::Nv12(frame) => WSFrame {
-                                data: frame.data,
-                                width: frame.width,
-                                height: frame.height,
-                                stride: frame.y_stride,
-                                frame_number: frame.frame_number,
-                                target_time_ns: frame.target_time_ns,
-                                format: WSFrameFormat::Nv12,
-                                created_at: Instant::now(),
-                            },
+                            cap_editor::EditorFrameOutput::Nv12(frame) => {
+                                let ws_format = match frame.format {
+                                    GpuOutputFormat::Nv12 => WSFrameFormat::Nv12,
+                                    GpuOutputFormat::Rgba => WSFrameFormat::Rgba,
+                                };
+                                WSFrame {
+                                    data: frame.data,
+                                    width: frame.width,
+                                    height: frame.height,
+                                    stride: frame.y_stride,
+                                    frame_number: frame.frame_number,
+                                    target_time_ns: frame.target_time_ns,
+                                    format: ws_format,
+                                    created_at: Instant::now(),
+                                }
+                            }
                             cap_editor::EditorFrameOutput::Rgba(frame) => WSFrame {
                                 data: frame.data,
                                 width: frame.width,

apps/desktop/src-tauri/src/editor_window.rs

@@ -30,10 +30,10 @@ fn pack_nv12_frame_ref(
     data: &[u8],
     width: u32,
     height: u32,
+    y_stride: u32,
     frame_number: u32,
     target_time_ns: u64,
 ) -> Vec<u8> {
-    let y_stride = width;
     let metadata_size = 28;
     let mut output = Vec::with_capacity(data.len() + metadata_size);
     output.extend_from_slice(data);
@@ -90,6 +90,7 @@ fn pack_ws_frame_ref(frame: &WSFrame) -> Vec<u8> {
             &frame.data,
             frame.width,
             frame.height,
+            frame.stride,
             frame.frame_number,
             frame.target_time_ns,
         ),

apps/desktop/src-tauri/src/frame_ws.rs

@@ -80,7 +80,7 @@ export const getPreviewResolution = (
 	quality: EditorPreviewQuality,
 ): XY<number> => {
 	const scale = previewQualityScale[quality];
-	const width = (Math.max(2, Math.round(OUTPUT_SIZE.x * scale)) + 1) & ~1;
+	const width = (Math.max(4, Math.round(OUTPUT_SIZE.x * scale)) + 3) & ~3;
 	const height = (Math.max(2, Math.round(OUTPUT_SIZE.y * scale)) + 1) & ~1;
 
 	return { x: width, y: height };

apps/desktop/src/routes/editor/context.ts

@@ -38,7 +38,7 @@ function convertNv12ToRgba(
 	const ySize = yStride * height;
 	const yPlane = nv12Data;
 	const uvPlane = nv12Data.subarray(ySize);
-	const uvStride = width;
+	const uvStride = yStride;
 
 	for (let row = 0; row < height; row++) {
 		const yRowOffset = row * yStride;
@@ -285,7 +285,7 @@ function createScreenshotEditorContext() {
 				if (!width || !height) return;
 
 				const ySize = yStride * height;
-				const uvSize = width * (height / 2);
+				const uvSize = yStride * (height / 2);
 				const totalSize = ySize + uvSize;
 
 				const nv12Data = new Uint8ClampedArray(buffer, 0, totalSize);

apps/desktop/src/routes/screenshot-editor/context.tsx

@@ -231,7 +231,7 @@ function parseFrameMetadata(bytes: Uint8Array): FrameMetadata | null {
 			if (!width || !height) return null;
 
 			const ySize = yStride * height;
-			const uvSize = width * (height / 2);
+			const uvSize = yStride * (height / 2);
 			const totalSize = ySize + uvSize;
 
 			if (bytes.byteLength - 28 < totalSize) {
@@ -303,7 +303,7 @@ function convertNv12ToRgba(
 	const ySize = yStride * height;
 	const yPlane = nv12Data;
 	const uvPlane = nv12Data.subarray(ySize);
-	const uvStride = width;
+	const uvStride = yStride;
 
 	for (let row = 0; row < height; row++) {
 		const yRowOffset = row * yStride;

apps/desktop/src/utils/frame-worker.ts

@@ -62,7 +62,7 @@ function convertNv12ToRgbaMainThread(
 	const ySize = yStride * height;
 	const yPlane = nv12Data;
 	const uvPlane = nv12Data.subarray(ySize);
-	const uvStride = width;
+	const uvStride = yStride;
 
 	for (let row = 0; row < height; row++) {
 		const yRowOffset = row * yStride;
@@ -308,7 +308,7 @@ export function createImageDataWS(
 
 		if (width > 0 && height > 0) {
 			const ySize = yStride * height;
-			const uvSize = width * (height / 2);
+			const uvSize = yStride * (height / 2);
 			const totalSize = ySize + uvSize;
 
 			const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
@@ -351,7 +351,7 @@ export function createImageDataWS(
 
 		if (width > 0 && height > 0) {
 			const ySize = yStride * height;
-			const uvSize = width * (height / 2);
+			const uvSize = yStride * (height / 2);
 			const totalSize = ySize + uvSize;
 
 			const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
@@ -669,7 +669,7 @@ export function createImageDataWS(
 
 				if (width > 0 && height > 0) {
 					const ySize = yStride * height;
-					const uvSize = width * (height / 2);
+					const uvSize = yStride * (height / 2);
 					const totalSize = ySize + uvSize;
 
 					const frameData = new Uint8ClampedArray(buffer, 0, totalSize);
@@ -735,7 +735,7 @@ export function createImageDataWS(
 
 				if (width > 0 && height > 0) {
 					const ySize = yStride * height;
-					const uvSize = width * (height / 2);
+					const uvSize = yStride * (height / 2);
 					const totalSize = ySize + uvSize;
 
 					const nv12Data = new Uint8ClampedArray(buffer, 0, totalSize);

apps/desktop/src/utils/socket.ts

@@ -403,6 +403,7 @@ videoImportProgress: "video-import-progress"
 
 /** user-defined types **/
 
+export type AllGpusInfo = { gpus: GpuInfoDiag[]; primaryGpuIndex: number | null; isMultiGpuSystem: boolean; hasDiscreteGpu: boolean }
 export type Annotation = { id: string; type: AnnotationType; x: number; y: number; width: number; height: number; strokeColor: string; strokeWidth: number; fillColor: string; opacity: number; rotation: number; text: string | null; maskType?: MaskType | null; maskLevel?: number | null }
 export type AnnotationType = "arrow" | "circle" | "rectangle" | "text" | "mask"
 export type AppTheme = "system" | "light" | "dark"
@@ -476,6 +477,7 @@ quality: number | null;
  */
 fast: boolean | null }
 export type GlideDirection = "none" | "left" | "right" | "up" | "down"
+export type GpuInfoDiag = { vendor: string; description: string; dedicatedVideoMemoryMb: number; adapterIndex: number; isSoftwareAdapter: boolean; isBasicRenderDriver: boolean; supportsHardwareEncoding: boolean }
 export type HapticPattern = "alignment" | "levelChange" | "generic"
 export type HapticPerformanceTime = "default" | "now" | "drawCompleted"
 export type Hotkey = { code: string; meta: boolean; ctrl: boolean; alt: boolean; shift: boolean }
@@ -489,7 +491,6 @@ export type JsonValue<T> = [T]
 export type LogicalBounds = { position: LogicalPosition; size: LogicalSize }
 export type LogicalPosition = { x: number; y: number }
 export type LogicalSize = { width: number; height: number }
-export type MacOSVersionInfo = { major: number; minor: number; patch: number; displayName: string; buildNumber: string; isAppleSilicon: boolean }
 export type MainWindowRecordingStartBehaviour = "close" | "minimise"
 export type MaskKeyframes = { position?: MaskVectorKeyframe[]; size?: MaskVectorKeyframe[]; intensity?: MaskScalarKeyframe[] }
 export type MaskKind = "sensitive" | "highlight"
@@ -533,6 +534,7 @@ export type RecordingStatus = "pending" | "recording"
 export type RecordingStopped = null
 export type RecordingTargetMode = "display" | "window" | "area" | "camera"
 export type RenderFrameEvent = { frame_number: number; fps: number; resolution_base: XY<number> }
+export type RenderingStatus = { isUsingSoftwareRendering: boolean; isUsingBasicRenderDriver: boolean; hardwareEncodingAvailable: boolean; warningMessage: string | null }
 export type RequestOpenRecordingPicker = { target_mode: RecordingTargetMode | null }
 export type RequestOpenSettings = { page: string }
 export type RequestScreenCapturePrewarm = { force?: boolean }
@@ -555,7 +557,7 @@ export type StartRecordingInputs = { capture_target: ScreenCaptureTarget; captur
 export type StereoMode = "stereo" | "monoL" | "monoR"
 export type StudioRecordingMeta = { segment: SingleSegment } | { inner: MultipleSegments }
 export type StudioRecordingStatus = { status: "InProgress" } | { status: "NeedsRemux" } | { status: "Failed"; error: string } | { status: "Complete" }
-export type SystemDiagnostics = { macosVersion: MacOSVersionInfo | null; availableEncoders: string[]; screenCaptureSupported: boolean; metalSupported: boolean; gpuName: string | null }
+export type SystemDiagnostics = { windowsVersion: WindowsVersionInfo | null; gpuInfo: GpuInfoDiag | null; allGpus: AllGpusInfo | null; renderingStatus: RenderingStatus; availableEncoders: string[]; graphicsCaptureSupported: boolean; d3D11VideoProcessorAvailable: boolean }
 export type TargetUnderCursor = { display_id: DisplayId | null; window: WindowUnderCursor | null }
 export type TextSegment = { start: number; end: number; enabled?: boolean; content?: string; center?: XY<number>; size?: XY<number>; fontFamily?: string; fontSize?: number; fontWeight?: number; italic?: boolean; color?: string; fadeDuration?: number }
 export type TimelineConfiguration = { segments: TimelineSegment[]; zoomSegments: ZoomSegment[]; sceneSegments?: SceneSegment[]; maskSegments?: MaskSegment[]; textSegments?: TextSegment[] }
@@ -574,6 +576,7 @@ export type WindowExclusion = { bundleIdentifier?: string | null; ownerName?: st
 export type WindowId = string
 export type WindowPosition = { x: number; y: number; displayId?: DisplayId | null }
 export type WindowUnderCursor = { id: WindowId; app_name: string; bounds: LogicalBounds }
+export type WindowsVersionInfo = { major: number; minor: number; build: number; displayName: string; meetsRequirements: boolean; isWindows11: boolean }
 export type XY<T> = { x: T; y: T }
 export type ZoomMode = "auto" | { manual: { x: number; y: number } }
 export type ZoomSegment = { start: number; end: number; amount: number; mode: ZoomMode; glideDirection?: GlideDirection; glideSpeed?: number; instantAnimation?: boolean; edgeSnapRatio?: number }

apps/desktop/src/utils/tauri.ts

@@ -380,7 +380,7 @@ export function renderNv12FrameWebGPU(
 	const ySize = yStride * height;
 	const uvWidth = width / 2;
 	const uvHeight = height / 2;
-	const uvStride = width;
+	const uvStride = yStride;
 	const uvSize = uvStride * uvHeight;
 
 	if (data.byteLength < ySize + uvSize) {