feat(moq-video,libmoq): native H.264 decode (drop ffmpeg dependency) (#1796)

Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

Author: kixelated

Cargo.lock

@@ -7,6 +7,13 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Native video decode C API: `moq_consume_video_raw` (+ `_close`, `_frame`,
+  `_frame_free`) subscribes to an H.264 track and hands back decoded I420 frames,
+  the video counterpart to `moq_consume_audio_raw`. Decoding happens inside
+  libmoq (VideoToolbox / openh264), so consumers no longer need ffmpeg.
+
 ## [0.3.7](https://github.com/moq-dev/moq/compare/libmoq-v0.3.6...libmoq-v0.3.7) - 2026-06-19
 
 ### Fixed

rs/libmoq/CHANGELOG.md

@@ -24,6 +24,7 @@ moq-audio = { workspace = true }
 moq-mux = { workspace = true }
 moq-native = { workspace = true, default-features = true }
 moq-net = { workspace = true, features = ["serde"] }
+moq-video = { workspace = true }
 thiserror = "2"
 tokio = { workspace = true, features = ["macros"] }
 tracing = "0.1"

rs/libmoq/Cargo.toml

@@ -492,6 +492,25 @@ impl Consume {
 		Ok(())
 	}
 
+	/// Look up a video rendition by catalog index, returning the
+	/// (broadcast, config, name) tuple needed to subscribe — mirrors
+	/// the index-based selection in `video_ordered`.
+	pub fn video_rendition(
+		&self,
+		catalog: Id,
+		index: usize,
+	) -> Result<(moq_net::BroadcastConsumer, hang::catalog::VideoConfig, String), Error> {
+		let consume = self.catalog.get(catalog).ok_or(Error::CatalogNotFound)?;
+		let (name, config) = consume
+			.catalog
+			.video
+			.renditions
+			.iter()
+			.nth(index)
+			.ok_or(Error::NoIndex)?;
+		Ok((consume.broadcast.clone(), config.clone(), name.clone()))
+	}
+
 	/// Look up an audio rendition by catalog index, returning the
 	/// (broadcast, config, name) tuple needed to subscribe — mirrors
 	/// the index-based selection in `audio_ordered`.

rs/libmoq/src/consume.rs

@@ -142,6 +142,10 @@ pub enum Error {
 	/// Error from the moq-audio codec layer.
 	#[error("audio error: {0}")]
 	Audio(Arc<moq_audio::AudioError>),
+
+	/// Error from the moq-video codec layer.
+	#[error("video error: {0}")]
+	Video(Arc<moq_video::Error>),
 }
 
 impl From<moq_audio::AudioError> for Error {
@@ -150,6 +154,12 @@ impl From<moq_audio::AudioError> for Error {
 	}
 }
 
+impl From<moq_video::Error> for Error {
+	fn from(err: moq_video::Error) -> Self {
+		Error::Video(Arc::new(err))
+	}
+}
+
 impl From<tracing::metadata::ParseLevelError> for Error {
 	fn from(err: tracing::metadata::ParseLevelError) -> Self {
 		Error::Level(Arc::new(err))
@@ -195,6 +205,7 @@ impl ffi::ReturnCode for Error {
 			Error::Native(_) => -33,
 			Error::Unauthorized => -34,
 			Error::Forbidden => -35,
+			Error::Video(_) => -36,
 		}
 	}
 }

rs/libmoq/src/error.rs

@@ -26,11 +26,13 @@ mod origin;
 mod publish;
 mod session;
 mod state;
+mod video;
 
 pub use api::*;
 pub use audio::*;
 pub use error::*;
 pub use id::*;
+pub use video::*;
 
 pub(crate) use consume::*;
 pub(crate) use origin::*;

rs/libmoq/src/lib.rs

@@ -1,13 +1,14 @@
 use std::sync::{LazyLock, Mutex, MutexGuard};
 
-use crate::{Consume, Origin, Publish, Session, audio::Audio};
+use crate::{Consume, Origin, Publish, Session, audio::Audio, video::Video};
 
 pub struct State {
 	pub session: Session,
 	pub origin: Origin,
 	pub publish: Publish,
 	pub consume: Consume,
 	pub audio: Audio,
+	pub video: Video,
 }
 
 impl State {
@@ -18,6 +19,7 @@ impl State {
 			publish: Publish::default(),
 			consume: Consume::default(),
 			audio: Audio::default(),
+			video: Video::default(),
 		}
 	}
 

rs/libmoq/src/state.rs

@@ -926,6 +926,106 @@ fn video_publish_consume() {
 	assert_eq!(moq_origin_close(origin), 0);
 }
 
+/// End-to-end native decode: publish real H.264 (encoded by moq-video) and
+/// consume it through `moq_consume_video_raw`, asserting decoded I420 frames.
+#[test]
+fn video_raw_decode() {
+	// Encode a few gray frames to Annex-B (avc3, SPS/PPS inline on the keyframe).
+	let mut config = moq_video::encode::Config::new(320, 240, 30);
+	config.kind = moq_video::encode::Kind::Software;
+	let mut encoder = moq_video::encode::Encoder::new(&config).expect("openh264 encoder");
+	let gray = vec![0x80u8; 320 * 240 * 4];
+	let mut frames: Vec<bytes::Bytes> = Vec::new();
+	for i in 0..5 {
+		frames.extend(encoder.encode_rgba(&gray, 320, 240, i == 0).unwrap());
+	}
+	frames.extend(encoder.finish().unwrap());
+	assert!(!frames.is_empty(), "encoder produced no frames");
+
+	let origin = id(moq_origin_create());
+	let broadcast = id(moq_publish_create());
+
+	// The init's SPS/PPS only seed catalog metadata; avc3 frames carry their own
+	// inline parameter sets, so the decoder reads the true 320x240 from the wire.
+	let init = h264_init();
+	let format = b"avc3";
+	let media = id(unsafe {
+		moq_publish_media_ordered(
+			broadcast,
+			format.as_ptr() as *const c_char,
+			format.len(),
+			init.as_ptr(),
+			init.len(),
+		)
+	});
+
+	let path = b"video-raw-test";
+	let _publish = id(unsafe { moq_origin_publish(origin, path.as_ptr() as *const c_char, path.len(), broadcast) });
+
+	let consume = request_broadcast(origin, path);
+	let catalog_cb = Callback::new();
+	let catalog_task = id(unsafe { moq_consume_catalog(consume, Some(channel_callback), catalog_cb.ptr) });
+	let catalog_id = id(catalog_cb.recv());
+
+	// Subscribe + decode before publishing frames so the keyframe group is delivered.
+	let output = moq_video_decoder_output { latency_max_ms: 10_000 };
+	let frame_cb = Callback::new();
+	let consumer = id(unsafe { moq_consume_video_raw(catalog_id, 0, &output, Some(channel_callback), frame_cb.ptr) });
+
+	for (i, frame) in frames.iter().enumerate() {
+		assert_eq!(
+			unsafe { moq_publish_media_frame(media, frame.as_ptr(), frame.len(), (i as u64) * 33_000) },
+			0
+		);
+	}
+
+	// First decoded frame: packed I420 at the encoder resolution.
+	let frame_id = id(frame_cb.recv());
+	let mut frame = moq_video_frame {
+		timestamp_us: 0,
+		width: 0,
+		height: 0,
+		data: std::ptr::null(),
+		data_size: 0,
+	};
+	assert_eq!(unsafe { moq_consume_video_raw_frame(frame_id, &mut frame) }, 0);
+	assert_eq!(frame.width, 320);
+	assert_eq!(frame.height, 240);
+	assert_eq!(frame.data_size, 320 * 240 * 3 / 2, "tightly-packed I420");
+	assert!(!frame.data.is_null());
+
+	assert_eq!(moq_consume_video_raw_frame_free(frame_id), 0);
+	assert_eq!(moq_consume_video_raw_close(consumer), 0);
+
+	// Drain any other decoded frames already queued, then expect the terminal 0.
+	loop {
+		let code = frame_cb.recv();
+		if code > 0 {
+			assert_eq!(moq_consume_video_raw_frame_free(id(code)), 0);
+		} else {
+			assert_eq!(code, 0, "raw video close delivers terminal 0");
+			break;
+		}
+	}
+	assert_eq!(moq_consume_catalog_free(catalog_id), 0);
+	assert_eq!(moq_consume_catalog_close(catalog_task), 0);
+	// The publisher may emit more than one catalog snapshot (e.g. as the track's
+	// stats settle), so drain any extra snapshots before the terminal.
+	loop {
+		let code = catalog_cb.recv();
+		if code > 0 {
+			assert_eq!(moq_consume_catalog_free(id(code)), 0);
+		} else {
+			assert_eq!(code, 0, "catalog close delivers terminal 0");
+			break;
+		}
+	}
+	assert_eq!(moq_consume_close(consume), 0);
+	assert_eq!(moq_publish_media_close(media), 0);
+	assert_eq!(moq_publish_close(broadcast), 0);
+	assert_eq!(moq_origin_close(origin), 0);
+}
+
 #[test]
 fn multiple_frames_ordering() {
 	let origin = id(moq_origin_create());