Desktop: Directly upload frame buffer (#2930)

* Upload frame buffer directly to gpu texture

* Disable cef GPU acceleration to prevent crashes

* Cleanup code

* Address review comments

---------

Co-authored-by: Timon Schelling <me@timon.zip>

Author: TrueDoctor

desktop/src/app.rs

@@ -1,7 +1,7 @@
 use crate::CustomEvent;
-use crate::FrameBuffer;
 use crate::WindowSize;
 use crate::render::GraphicsState;
+use crate::render::WgpuContext;
 use std::sync::Arc;
 use std::sync::mpsc::Sender;
 use std::time::Duration;
@@ -12,7 +12,6 @@ use winit::event::StartCause;
 use winit::event::WindowEvent;
 use winit::event_loop::ActiveEventLoop;
 use winit::event_loop::ControlFlow;
-use winit::event_loop::EventLoopProxy;
 use winit::window::Window;
 use winit::window::WindowId;
 
@@ -22,33 +21,34 @@ pub(crate) struct WinitApp {
 	pub(crate) cef_context: cef::Context<cef::Initialized>,
 	pub(crate) window: Option<Arc<Window>>,
 	cef_schedule: Option<Instant>,
-	ui_frame_buffer: Option<FrameBuffer>,
+	_ui_frame_buffer: Option<wgpu::Texture>,
 	window_size_sender: Sender<WindowSize>,
-	_viewport_frame_buffer: Option<FrameBuffer>,
+	_viewport_frame_buffer: Option<wgpu::Texture>,
 	graphics_state: Option<GraphicsState>,
-	event_loop_proxy: EventLoopProxy<CustomEvent>,
+	wgpu_context: WgpuContext,
 }
 
 impl WinitApp {
-	pub(crate) fn new(cef_context: cef::Context<cef::Initialized>, window_size_sender: Sender<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>) -> Self {
+	pub(crate) fn new(cef_context: cef::Context<cef::Initialized>, window_size_sender: Sender<WindowSize>, wgpu_context: WgpuContext) -> Self {
 		Self {
 			cef_context,
 			window: None,
 			cef_schedule: Some(Instant::now()),
 			_viewport_frame_buffer: None,
-			ui_frame_buffer: None,
+			_ui_frame_buffer: None,
 			graphics_state: None,
 			window_size_sender,
-			event_loop_proxy,
+			wgpu_context,
 		}
 	}
 }
 
 impl ApplicationHandler<CustomEvent> for WinitApp {
 	fn about_to_wait(&mut self, event_loop: &ActiveEventLoop) {
 		// Set a timeout in case we miss any cef schedule requests
-		let timeout = Instant::now() + Duration::from_millis(100);
+		let timeout = Instant::now() + Duration::from_millis(10);
 		let wait_until = timeout.min(self.cef_schedule.unwrap_or(timeout));
+		self.cef_context.work();
 		event_loop.set_control_flow(ControlFlow::WaitUntil(wait_until));
 	}
 
@@ -71,7 +71,7 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 				)
 				.unwrap(),
 		);
-		let graphics_state = futures::executor::block_on(GraphicsState::new(window.clone()));
+		let graphics_state = GraphicsState::new(window.clone(), self.wgpu_context.clone());
 
 		self.window = Some(window);
 		self.graphics_state = Some(graphics_state);
@@ -81,24 +81,21 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 
 	fn user_event(&mut self, _: &ActiveEventLoop, event: CustomEvent) {
 		match event {
-			CustomEvent::UiUpdate(frame_buffer) => {
+			CustomEvent::UiUpdate(texture) => {
 				if let Some(graphics_state) = self.graphics_state.as_mut() {
-					graphics_state.update_texture(&frame_buffer);
+					graphics_state.bind_texture(&texture);
+					graphics_state.resize(texture.width(), texture.height());
 				}
-				self.ui_frame_buffer = Some(frame_buffer);
 				if let Some(window) = &self.window {
 					window.request_redraw();
 				}
 			}
 			CustomEvent::ScheduleBrowserWork(instant) => {
-				if let Some(graphics_state) = self.graphics_state.as_mut()
-					&& let Some(frame_buffer) = &self.ui_frame_buffer
-					&& graphics_state.ui_texture_outdated(frame_buffer)
-				{
+				if instant <= Instant::now() {
 					self.cef_context.work();
-					let _ = self.event_loop_proxy.send_event(CustomEvent::ScheduleBrowserWork(Instant::now() + Duration::from_millis(1)));
+				} else {
+					self.cef_schedule = Some(instant);
 				}
-				self.cef_schedule = Some(instant);
 			}
 		}
 	}
@@ -113,9 +110,6 @@ impl ApplicationHandler<CustomEvent> for WinitApp {
 			}
 			WindowEvent::Resized(PhysicalSize { width, height }) => {
 				let _ = self.window_size_sender.send(WindowSize::new(width as usize, height as usize));
-				if let Some(ref mut graphics_state) = self.graphics_state {
-					graphics_state.resize(width, height);
-				}
 				self.cef_context.notify_of_resize();
 			}
 

desktop/src/cef.rs

@@ -1,4 +1,4 @@
-use crate::{CustomEvent, FrameBuffer};
+use crate::{CustomEvent, WgpuContext, render::FrameBufferRef};
 use std::{
 	sync::{Arc, Mutex, mpsc::Receiver},
 	time::Instant,
@@ -15,7 +15,7 @@ use winit::event_loop::EventLoopProxy;
 
 pub(crate) trait CefEventHandler: Clone {
 	fn window_size(&self) -> WindowSize;
-	fn draw(&self, frame_buffer: FrameBuffer);
+	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>);
 	/// Scheudule the main event loop to run the cef event loop after the timeout
 	///  [`_cef_browser_process_handler_t::on_schedule_message_pump_work`] for more documentation.
 	fn schedule_cef_message_loop_work(&self, scheduled_time: Instant);
@@ -37,6 +37,7 @@ impl WindowSize {
 pub(crate) struct CefHandler {
 	window_size_receiver: Arc<Mutex<WindowSizeReceiver>>,
 	event_loop_proxy: EventLoopProxy<CustomEvent>,
+	wgpu_context: WgpuContext,
 }
 struct WindowSizeReceiver {
 	receiver: Receiver<WindowSize>,
@@ -51,10 +52,11 @@ impl WindowSizeReceiver {
 	}
 }
 impl CefHandler {
-	pub(crate) fn new(window_size_receiver: Receiver<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>) -> Self {
+	pub(crate) fn new(window_size_receiver: Receiver<WindowSize>, event_loop_proxy: EventLoopProxy<CustomEvent>, wgpu_context: WgpuContext) -> Self {
 		Self {
 			window_size_receiver: Arc::new(Mutex::new(WindowSizeReceiver::new(window_size_receiver))),
 			event_loop_proxy,
+			wgpu_context,
 		}
 	}
 }
@@ -71,8 +73,44 @@ impl CefEventHandler for CefHandler {
 		}
 		*window_size
 	}
-	fn draw(&self, frame_buffer: FrameBuffer) {
-		let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(frame_buffer));
+	fn draw<'a>(&self, frame_buffer: FrameBufferRef<'a>) {
+		let width = frame_buffer.width() as u32;
+		let height = frame_buffer.height() as u32;
+		let texture = self.wgpu_context.device.create_texture(&wgpu::TextureDescriptor {
+			label: Some("CEF Texture"),
+			size: wgpu::Extent3d {
+				width,
+				height,
+				depth_or_array_layers: 1,
+			},
+			mip_level_count: 1,
+			sample_count: 1,
+			dimension: wgpu::TextureDimension::D2,
+			format: wgpu::TextureFormat::Bgra8UnormSrgb,
+			usage: wgpu::TextureUsages::TEXTURE_BINDING | wgpu::TextureUsages::COPY_DST,
+			view_formats: &[],
+		});
+		self.wgpu_context.queue.write_texture(
+			wgpu::TexelCopyTextureInfo {
+				texture: &texture,
+				mip_level: 0,
+				origin: wgpu::Origin3d::ZERO,
+				aspect: wgpu::TextureAspect::All,
+			},
+			frame_buffer.buffer(),
+			wgpu::TexelCopyBufferLayout {
+				offset: 0,
+				bytes_per_row: Some(4 * width),
+				rows_per_image: Some(height),
+			},
+			wgpu::Extent3d {
+				width,
+				height,
+				depth_or_array_layers: 1,
+			},
+		);
+
+		let _ = self.event_loop_proxy.send_event(CustomEvent::UiUpdate(texture));
 	}
 
 	fn schedule_cef_message_loop_work(&self, scheduled_time: std::time::Instant) {

desktop/src/cef/internal/app.rs

@@ -1,6 +1,6 @@
 use cef::rc::{Rc, RcImpl};
 use cef::sys::{_cef_app_t, cef_base_ref_counted_t};
-use cef::{BrowserProcessHandler, ImplApp, SchemeRegistrar, WrapApp};
+use cef::{BrowserProcessHandler, CefString, ImplApp, ImplCommandLine, SchemeRegistrar, WrapApp};
 
 use crate::cef::CefEventHandler;
 use crate::cef::scheme_handler::GraphiteSchemeHandlerFactory;
@@ -29,6 +29,14 @@ impl<H: CefEventHandler + Clone> ImplApp for AppImpl<H> {
 		GraphiteSchemeHandlerFactory::register_schemes(registrar);
 	}
 
+	fn on_before_command_line_processing(&self, _process_type: Option<&cef::CefString>, command_line: Option<&mut cef::CommandLine>) {
+		if let Some(cmd) = command_line {
+			// Disable GPU acceleration, because it is not supported for Offscreen Rendering and can cause crashes.
+			cmd.append_switch(Some(&CefString::from("disable-gpu")));
+			cmd.append_switch(Some(&CefString::from("disable-gpu-compositing")));
+		}
+	}
+
 	fn get_raw(&self) -> *mut _cef_app_t {
 		self.object.cast()
 	}

desktop/src/cef/internal/browser_process_handler.rs

@@ -25,13 +25,13 @@ impl<H: CefEventHandler + Clone> ImplBrowserProcessHandler for BrowserProcessHan
 		cef::register_scheme_handler_factory(Some(&CefString::from(GRAPHITE_SCHEME)), None, Some(&mut SchemeHandlerFactory::new(GraphiteSchemeHandlerFactory::new())));
 	}
 
-	fn get_raw(&self) -> *mut _cef_browser_process_handler_t {
-		self.object.cast()
-	}
-
 	fn on_schedule_message_pump_work(&self, delay_ms: i64) {
 		self.event_handler.schedule_cef_message_loop_work(Instant::now() + Duration::from_millis(delay_ms as u64));
 	}
+
+	fn get_raw(&self) -> *mut _cef_browser_process_handler_t {
+		self.object.cast()
+	}
 }
 
 impl<H: CefEventHandler + Clone> Clone for BrowserProcessHandlerImpl<H> {

desktop/src/cef/internal/render_handler.rs

@@ -2,8 +2,8 @@ use cef::rc::{Rc, RcImpl};
 use cef::sys::{_cef_render_handler_t, cef_base_ref_counted_t};
 use cef::{Browser, ImplRenderHandler, PaintElementType, Rect, WrapRenderHandler};
 
-use crate::FrameBuffer;
 use crate::cef::CefEventHandler;
+use crate::render::FrameBufferRef;
 
 pub(crate) struct RenderHandlerImpl<H: CefEventHandler> {
 	object: *mut RcImpl<_cef_render_handler_t, Self>,
@@ -42,7 +42,7 @@ impl<H: CefEventHandler> ImplRenderHandler for RenderHandlerImpl<H> {
 	) {
 		let buffer_size = (width * height * 4) as usize;
 		let buffer_slice = unsafe { std::slice::from_raw_parts(buffer, buffer_size) };
-		let frame_buffer = FrameBuffer::new(buffer_slice.to_vec(), width as usize, height as usize).expect("Failed to create frame buffer");
+		let frame_buffer = FrameBufferRef::new(buffer_slice, width as usize, height as usize).expect("Failed to create frame buffer");
 
 		self.event_handler.draw(frame_buffer)
 	}

desktop/src/main.rs

@@ -9,7 +9,7 @@ mod cef;
 use cef::{Setup, WindowSize};
 
 mod render;
-use render::FrameBuffer;
+use render::WgpuContext;
 
 mod app;
 use app::WinitApp;
@@ -18,7 +18,7 @@ mod dirs;
 
 #[derive(Debug)]
 pub(crate) enum CustomEvent {
-	UiUpdate(FrameBuffer),
+	UiUpdate(wgpu::Texture),
 	ScheduleBrowserWork(Instant),
 }
 
@@ -38,7 +38,8 @@ fn main() {
 
 	let (window_size_sender, window_size_receiver) = std::sync::mpsc::channel();
 
-	let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy())) {
+	let wgpu_context = futures::executor::block_on(WgpuContext::new());
+	let cef_context = match cef_context.init(cef::CefHandler::new(window_size_receiver, event_loop.create_proxy(), wgpu_context.clone())) {
 		Ok(c) => c,
 		Err(cef::InitError::InitializationFailed) => {
 			tracing::error!("Cef initialization failed");
@@ -48,7 +49,7 @@ fn main() {
 
 	tracing::info!("Cef initialized successfully");
 
-	let mut winit_app = WinitApp::new(cef_context, window_size_sender, event_loop.create_proxy());
+	let mut winit_app = WinitApp::new(cef_context, window_size_sender, wgpu_context);
 
 	event_loop.run_app(&mut winit_app).unwrap();
 }