feat(moq-video): native V4L2 capture, NVENC via dlopen, drop VAAPI + nokhwa(Linux) (#1704)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: kixelated

Cargo.lock

@@ -101,3 +101,11 @@ opt-level = "z"
 lto = "fat"
 codegen-units = 1
 strip = true
+
+# moq-video's `nvenc` feature needs a `dynamic-loading` mode on
+# nvidia-video-codec-sdk so libnvidia-encode is dlopen'd at runtime instead of
+# linked: that lets a portable moq-cli build (and load) on machines without the
+# NVIDIA driver, and lets a GPU-less CI/release builder compile it. Upstream
+# doesn't have that feature yet, so consume a fork branch until it's merged.
+[patch.crates-io]
+nvidia-video-codec-sdk = { git = "https://github.com/kixelated/nvidia-video-codec-sdk.git", branch = "dynamic-loading" }

Cargo.toml

@@ -47,6 +47,7 @@ allow = [
     "BSD-3-Clause",
     "CC0-1.0",
     "CDLA-Permissive-2.0", # webpki-roots cert bundle
+    "IJG", # mozjpeg-sys (MJPEG decode via nokhwa in moq-video)
     "ISC",
     "MIT",
     "MPL-2.0",
@@ -66,3 +67,7 @@ allow-wildcard-paths = true
 unknown-registry = "deny"
 unknown-git = "deny"
 allow-registry = ["https://github.com/rust-lang/crates.io-index"]
+# moq-video patches nvidia-video-codec-sdk to a fork that adds dlopen
+# (`dynamic-loading`); see the root Cargo.toml [patch.crates-io]. Drop this once
+# the feature is upstreamed and the patch is removed.
+allow-git = ["https://github.com/kixelated/nvidia-video-codec-sdk"]

deny.toml

@@ -38,9 +38,19 @@ ci FILES="":
     fi
     just check --workspace
     cargo check --workspace --no-default-features
-    cargo check --workspace --all-features
+    # NVENC's SDK crate has no macOS bindings, so moq-video can't be part of the
+    # blanket --all-features runs on a Mac. Exclude it and cover it on Linux only,
+    # where the `nvenc` feature dlopens libnvidia-encode (so it builds, links, and
+    # falls back to software without a GPU/driver).
+    cargo check --workspace --all-features --exclude moq-video
     cargo deny check --show-stats
-    just rs test --all-features
+    cargo test --workspace --all-targets --all-features --exclude moq-video
+    if [[ "$(uname)" == "Linux" ]]; then
+    	cargo check -p moq-video --all-targets --all-features
+    	cargo test -p moq-video --all-targets --all-features
+    else
+    	cargo test -p moq-video --all-targets
+    fi
     just build
 
 # Auto-fix clippy/format/shear/sort, then sweep stale target/.

rs/justfile

@@ -16,27 +16,21 @@ categories = ["multimedia", "multimedia::video", "multimedia::encoding"]
 doctest = false
 
 [features]
-# NVIDIA NVENC hardware encoder (Linux only). Off by default; release builds
-# for Linux enable it. cudarc loads CUDA dynamically (no build-time CUDA), and
-# the NVENC API lives in the driver, so this is not a build dependency.
-nvenc = ["dep:nvidia-video-codec-sdk", "cudarc/dynamic-loading", "cudarc/cuda-12020"]
-# Intel/AMD VAAPI hardware encoder (Linux only). Off by default; release builds
-# for Linux enable it. Links libva (stable soname); pairs with a V4L2 dmabuf
-# capture for a fully zero-copy path.
-vaapi = ["dep:cros-codecs"]
+# NVIDIA NVENC hardware encoder (Linux). Off by default; enable for a build that
+# should use NVENC where present. Everything is dlopen'd at runtime: cudarc loads
+# CUDA, and nvidia-video-codec-sdk's `dynamic-loading` (a fork feature, see the
+# root [patch.crates-io]) loads libnvidia-encode. So an nvenc-enabled binary still
+# links on a GPU-less builder and still starts on machines without the NVIDIA
+# driver, falling back to software (see backend::open). `cuda-12020` pins the CUDA
+# API version so the build needs no CUDA toolkit.
+nvenc = ["dep:nvidia-video-codec-sdk", "nvidia-video-codec-sdk/dynamic-loading", "cudarc/fallback-dynamic-loading", "cudarc/cuda-12020"]
 
 [dependencies]
 anyhow = "1"
 bytes = "1"
-# VAAPI H.264 encoder + dmabuf import. Gated behind `vaapi` + a linux cfg; pinned
-# (pre-1.0 churns). Pulls libva (cros-libva) when the feature is on.
-cros-codecs = { version = "=0.0.6", optional = true, default-features = false, features = ["vaapi"] }
-cudarc = { version = "0.16", optional = true, default-features = false, features = ["driver"] }
+cudarc = { version = "0.19", optional = true, default-features = false, features = ["driver"] }
 moq-mux = { workspace = true }
 moq-net = { workspace = true }
-# Pure-Rust webcam capture (avfoundation / v4l2 / msmf), decoding MJPEG/YUYV to
-# RGBA. Platform input backends are enabled per-target below.
-nokhwa = { version = "0.10.11", default-features = false, features = ["decoding"] }
 # NVENC backend, gated behind the `nvenc` feature + a linux cfg. Listed here
 # (not under a target table) so the feature can reference it on any host; the
 # code that uses it is `cfg(target_os = "linux")`.
@@ -50,12 +44,14 @@ tracing = "0.1"
 yuv = "0.8.14"
 
 [target.'cfg(target_os="linux")'.dependencies]
-nokhwa = { version = "0.10.11", features = ["input-v4l"] }
+# Native V4L2 webcam capture (replaces nokhwa). `v4l` streams MMAP buffers;
+# MJPEG cameras are decoded with the pure-Rust `zune-jpeg` (no libjpeg/IJG).
+v4l = "0.14"
+zune-jpeg = "0.5"
 
 [target.'cfg(target_os="macos")'.dependencies]
 block2 = "0.6.2"
 dispatch2 = "0.3.1"
-nokhwa = { version = "0.10.11", features = ["input-avfoundation"] }
 objc2 = "0.6.4"
 objc2-av-foundation = "0.3.2"
 objc2-core-foundation = "0.3.2"
@@ -66,4 +62,6 @@ objc2-screen-capture-kit = "0.3.2"
 objc2-video-toolbox = "0.3.2"
 
 [target.'cfg(target_os="windows")'.dependencies]
-nokhwa = { version = "0.10.11", features = ["input-msmf"] }
+# Windows still uses nokhwa (Media Foundation) until a native MF capture lands;
+# `decoding` handles its MJPEG/YUYV -> RGBA conversion.
+nokhwa = { version = "0.10.11", default-features = false, features = ["input-msmf", "decoding"] }

rs/moq-video/Cargo.toml

@@ -351,12 +351,63 @@ Where the implementation differs from the plan above:
   `write` matches NVENC's pitch (safe only for 64-aligned widths; we warn otherwise);
   (3) forced-IDR via `picture_type` with picture-type-decision enabled.
 
+### VAAPI removed (cros-codecs is not buildable on modern libva)
+
+The VAAPI backend (added behind the `vaapi` feature in #1691) and the V4L2 dmabuf
+capture written to feed it have been **removed**. `cros-codecs 0.0.6` (the latest)
+caret-pins `cros-libva 0.0.12`, which does not compile against libva >= 2.23: the
+newer headers add `seg_id_block_size` / `va_reserved8` to
+`VAEncPictureParameterBufferVP9`, and `cros-libva 0.0.12`'s struct literal omits
+them (`cros-libva 0.0.13` fixes it, but `cros-codecs 0.0.6` won't accept it). Since
+modern distros and the nix toolchain ship libva 2.23+, the feature could not build
+or ship, and CI's `--all-features` could not be made green while it existed.
+`cros-codecs` is the only pure-Rust VAAPI H.264 encoder, and reintroducing ffmpeg
+for VAAPI defeats the purpose of this effort, so VAAPI is parked until `cros-codecs`
+releases against `cros-libva >= 0.0.13` (or a better VAAPI path appears). Intel/AMD
+Linux falls back to the openh264 software encoder meanwhile; NVIDIA uses NVENC.
+
+When it returns: re-add `backend/vaapi.rs`, `capture/v4l2.rs`, the `vaapi` feature
+(+ `cros-codecs`/`v4l`/`libc` deps), `Frame::DmaBuf`, the `requires_dmabuf` capture
+coupling, and `libva` in the nix devShell. See this PR's history for the prior
+implementation; the V4L2 capture (REQBUFS/EXPBUF/QBUF/DQBUF) was compile-verified
+but never runtime-tested.
+
+### NVENC ships via dlopen (no driver dependency at build or load)
+
+For the "single binary reaches the GPU at runtime" goal, NVENC must not hard-link
+the driver. The stock `nvidia-video-codec-sdk` emits
+`cargo:rustc-link-lib=nvidia-encode` / `nvcuvid`, which would make an
+`--features nvenc` binary (a) impossible to link on a GPU-less builder and (b)
+fail to even load on a machine without the NVIDIA driver (`DT_NEEDED
+libnvidia-encode.so.1`), before `backend::open`'s software fallback could run.
+
+So `nvenc` dlopens everything at runtime, like `cudarc` does for CUDA:
+
+- `cudarc/fallback-dynamic-loading` dlopens `libcuda`; `cudarc/cuda-12020` pins the
+  CUDA API version so the build needs no CUDA toolkit.
+- `nvidia-video-codec-sdk/dynamic-loading` dlopens `libnvidia-encode`. This is a
+  small fork feature (see the root `[patch.crates-io]`): the SDK already routes
+  every call through a function table built from two entry points
+  (`NvEncodeAPICreateInstance` / `GetMaxSupportedVersion`); `dynamic-loading`
+  resolves those two via `dlopen` instead of linking them, and `build.rs` skips the
+  link directives.
+
+Result, verified on a GPU-less Linux box: `--features nvenc` builds, links, and the
+test suite runs and passes (NVENC unavailable -> falls back to openh264), and the
+binary has no `libnvidia-encode` / `libcuda` `DT_NEEDED`. So one portable `moq-cli`
+can carry NVENC and use it only where the driver is present. Upstream the
+`dynamic-loading` feature and drop the patch once merged.
+
 ### Follow-ups
 
-- VAAPI backend (phase 6) in a separate PR, behind a `vaapi` feature.
-- NVENC hardware validation on Linux+GPU / CI.
-- Live camera run (capture needs camera permission; only synthetic-frame encode is
-  tested here).
+- CI builds + tests NVENC normally on Linux (`cargo {check,test} -p moq-video
+  --all-features`); the dlopen feature means no GPU/driver and no special flags are
+  needed. moq-video stays excluded from the *workspace* `--all-features` runs only
+  because its SDK crate has no macOS bindings (see `rs/justfile`'s `ci` recipe).
+- Still needed on real hardware: NVENC encode validation on a Linux+GPU box (pitch
+  alignment, forced-IDR); only synthetic-frame software encode is tested here.
+- Live camera run (capture needs camera/screen TCC permission, which a headless or
+  agent-spawned process can't obtain; run `moq-cli ... capture` from a user terminal).
 - `doc/bin/cli.md`: the `capture` device-string semantics changed (index-or-path).
 - Consider reusing NVENC input/output buffers across frames (currently allocated
   per frame to sidestep the self-referential Session borrow).

rs/moq-video/DESIGN-native-codecs.md

@@ -2,9 +2,9 @@
 //! per-source:
 //! - macOS camera -> AVFoundation, screen -> ScreenCaptureKit, both yielding
 //!   zero-copy `CVPixelBuffer` surfaces straight to VideoToolbox.
-//! - other platforms -> [`nokhwa`](https://crates.io/crates/nokhwa) camera
-//!   (CPU RGBA -> I420) until a native zero-copy path (V4L2 dmabuf / PipeWire)
-//!   lands.
+//! - Linux camera -> native V4L2 (YUYV / MJPEG -> CPU I420).
+//! - Windows camera -> [`nokhwa`](https://crates.io/crates/nokhwa) (Media
+//!   Foundation), CPU RGBA -> I420, until a native MF capture lands.
 //!
 //! [`encode::publish_capture`](crate::encode::publish_capture) consumes [`Config`].
 
@@ -18,7 +18,12 @@ mod queue;
 #[cfg(target_os = "macos")]
 mod screencapture;
 
-#[cfg(not(target_os = "macos"))]
+// Native V4L2 camera capture on Linux.
+#[cfg(target_os = "linux")]
+mod v4l2;
+
+// nokhwa CPU webcam: Windows only, until a native Media Foundation capture lands.
+#[cfg(target_os = "windows")]
 mod webcam;
 
 /// What to capture.
@@ -72,10 +77,20 @@ pub(crate) fn open(config: &Config) -> Result<Box<dyn FrameSource>, Error> {
 			{
 				Ok(Box::new(avfoundation::Camera::open(config)?))
 			}
-			#[cfg(not(target_os = "macos"))]
+			#[cfg(target_os = "linux")]
+			{
+				Ok(Box::new(v4l2::Camera::open(config)?))
+			}
+			#[cfg(target_os = "windows")]
 			{
 				Ok(Box::new(webcam::Camera::open(config)?))
 			}
+			#[cfg(not(any(target_os = "macos", target_os = "linux", target_os = "windows")))]
+			{
+				Err(Error::Codec(anyhow::anyhow!(
+					"camera capture is not supported on this platform"
+				)))
+			}
 		}
 		Source::Display => {
 			#[cfg(target_os = "macos")]