Skip to content

Commit 0579e4f

Browse files
chenosauruschenosaurus
committed
add user_data trailer type
1 parent e63fb1e commit 0579e4f

20 files changed

Lines changed: 279 additions & 43 deletions

File tree

‎examples/local_video/src/publisher.rs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1574,7 +1574,7 @@ async fn run_capture_loop(
15741574
debug_assert_eq!(burned_timestamp_us, Some(capture_wall_time_us));
15751575
}
15761576
frame.frame_metadata = if user_ts.is_some() || fid.is_some() {
1577-
Some(FrameMetadata { user_timestamp: user_ts, frame_id: fid })
1577+
Some(FrameMetadata { user_timestamp: user_ts, frame_id: fid, user_data: None })
15781578
} else {
15791579
None
15801580
};
@@ -1819,7 +1819,7 @@ async fn run_argus_capture_loop(
18191819
None
18201820
};
18211821
let frame_metadata = if user_ts.is_some() || fid.is_some() {
1822-
Some(FrameMetadata { user_timestamp: user_ts, frame_id: fid })
1822+
Some(FrameMetadata { user_timestamp: user_ts, frame_id: fid, user_data: None })
18231823
} else {
18241824
None
18251825
};

‎libwebrtc/src/native/packet_trailer.rs‎

Lines changed: 13 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -159,13 +159,15 @@ impl PacketTrailerHandler {
159159
}
160160

161161
/// Lookup the frame metadata for a given RTP timestamp (receiver side).
162-
/// Returns `Some((user_timestamp, frame_id))` if found, `None` otherwise.
163-
/// The entry is removed from the map after a successful lookup.
164-
pub fn lookup_frame_metadata(&self, rtp_timestamp: u32) -> Option<(u64, u32)> {
162+
/// Returns `Some((user_timestamp, frame_id, user_data))` if found,
163+
/// `None` otherwise. The entry is removed from the map after a
164+
/// successful lookup.
165+
pub fn lookup_frame_metadata(&self, rtp_timestamp: u32) -> Option<(u64, u32, Vec<u8>)> {
165166
let ts = self.sys_handle.lookup_timestamp(rtp_timestamp);
166167
if ts != u64::MAX {
167168
let frame_id = self.sys_handle.last_lookup_frame_id();
168-
Some((ts, frame_id))
169+
let user_data = self.sys_handle.last_lookup_user_data();
170+
Some((ts, frame_id, user_data))
169171
} else {
170172
None
171173
}
@@ -188,8 +190,14 @@ impl PacketTrailerHandler {
188190
capture_timestamp_us: i64,
189191
user_timestamp: u64,
190192
frame_id: u32,
193+
user_data: &[u8],
191194
) {
192-
self.sys_handle.store_frame_metadata(capture_timestamp_us, user_timestamp, frame_id);
195+
self.sys_handle.store_frame_metadata(
196+
capture_timestamp_us,
197+
user_timestamp,
198+
frame_id,
199+
user_data,
200+
);
193201
}
194202

195203
pub(crate) fn sys_handle(&self) -> SharedPtr<sys_pt::PacketTrailerHandler> {

‎libwebrtc/src/native/video_source.rs‎

Lines changed: 19 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@ impl NativeVideoSource {
8989
has_packet_trailer: false,
9090
user_timestamp: 0,
9191
frame_id: 0,
92+
user_data: Vec::new(),
9293
},
9394
);
9495
}
@@ -115,9 +116,14 @@ impl NativeVideoSource {
115116
};
116117
builder.pin_mut().set_timestamp_us(capture_ts);
117118

118-
let (has_trailer, user_ts, fid) = match frame.frame_metadata {
119-
Some(meta) => (true, meta.user_timestamp.unwrap_or(0), meta.frame_id.unwrap_or(0)),
120-
None => (false, 0, 0),
119+
let (has_trailer, user_ts, fid, user_data) = match &frame.frame_metadata {
120+
Some(meta) => (
121+
true,
122+
meta.user_timestamp.unwrap_or(0),
123+
meta.frame_id.unwrap_or(0),
124+
meta.user_data.clone().unwrap_or_default(),
125+
),
126+
None => (false, 0, 0, Vec::new()),
121127
};
122128

123129
self.inner.lock().captured_frames += 1;
@@ -128,6 +134,7 @@ impl NativeVideoSource {
128134
has_packet_trailer: has_trailer,
129135
user_timestamp: user_ts,
130136
frame_id: fid,
137+
user_data,
131138
},
132139
);
133140
}
@@ -167,9 +174,14 @@ impl NativeVideoSource {
167174
timestamp_us: i64,
168175
frame_metadata: Option<FrameMetadata>,
169176
) -> bool {
170-
let (has_trailer, user_ts, fid) = match frame_metadata {
171-
Some(meta) => (true, meta.user_timestamp.unwrap_or(0), meta.frame_id.unwrap_or(0)),
172-
None => (false, 0, 0),
177+
let (has_trailer, user_ts, fid, user_data) = match frame_metadata {
178+
Some(meta) => (
179+
true,
180+
meta.user_timestamp.unwrap_or(0),
181+
meta.frame_id.unwrap_or(0),
182+
meta.user_data.unwrap_or_default(),
183+
),
184+
None => (false, 0, 0, Vec::new()),
173185
};
174186

175187
self.inner.lock().captured_frames += 1;
@@ -183,6 +195,7 @@ impl NativeVideoSource {
183195
has_packet_trailer: has_trailer,
184196
user_timestamp: user_ts,
185197
frame_id: fid,
198+
user_data,
186199
},
187200
)
188201
}

‎libwebrtc/src/native/video_stream.rs‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -112,14 +112,15 @@ impl VideoTrackObserver {
112112
) -> Option<FrameMetadata> {
113113
handler
114114
.and_then(|handler| {
115-
handler.lookup_frame_metadata(rtp_timestamp).map(|(ts, fid)| {
115+
handler.lookup_frame_metadata(rtp_timestamp).map(|(ts, fid, user_data)| {
116116
handler.emit_subscribe_timing(SubscribeTimingStage::DecoderOutput, ts, fid);
117-
(ts, fid)
117+
(ts, fid, user_data)
118118
})
119119
})
120-
.map(|(ts, fid)| FrameMetadata {
120+
.map(|(ts, fid, user_data)| FrameMetadata {
121121
user_timestamp: Some(ts),
122122
frame_id: if fid != 0 { Some(fid) } else { None },
123+
user_data: if user_data.is_empty() { None } else { Some(user_data) },
123124
})
124125
}
125126
}

‎libwebrtc/src/video_frame.rs‎

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,13 +55,20 @@ pub enum VideoBufferType {
5555
/// Metadata carried alongside a video frame via the packet trailer mechanism.
5656
///
5757
/// Each field corresponds to an independently negotiable packet trailer feature
58-
/// (`PTF_USER_TIMESTAMP`, `PTF_FRAME_ID`), so individual fields are `Option`.
59-
#[derive(Debug, Clone, Copy)]
58+
/// (`PTF_USER_TIMESTAMP`, `PTF_FRAME_ID`, `PTF_USER_DATA`), so individual fields
59+
/// are `Option`.
60+
#[derive(Debug, Clone)]
6061
pub struct FrameMetadata {
6162
/// Wall-clock capture time in microseconds, when `PTF_USER_TIMESTAMP` is enabled.
6263
pub user_timestamp: Option<u64>,
6364
/// Monotonically increasing frame identifier, when `PTF_FRAME_ID` is enabled.
6465
pub frame_id: Option<u32>,
66+
/// Arbitrary application-supplied bytes, when `PTF_USER_DATA` is enabled.
67+
///
68+
/// Bounded by the packet trailer size budget (~232 bytes when the other
69+
/// features are also active); oversize payloads are dropped on the send
70+
/// side rather than truncated.
71+
pub user_data: Option<Vec<u8>>,
6572
}
6673

6774
#[derive(Debug)]

‎livekit-ffi/protocol/track.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -164,4 +164,5 @@ enum AudioTrackFeature {
164164
enum FrameMetadataFeature {
165165
FMF_USER_TIMESTAMP = 0;
166166
FMF_FRAME_ID = 1;
167+
FMF_USER_DATA = 2;
167168
}

‎livekit-ffi/protocol/video_frame.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -157,6 +157,7 @@ message OwnedVideoBuffer {
157157
message FrameMetadata {
158158
optional uint64 user_timestamp = 1;
159159
optional uint32 frame_id = 2;
160+
optional bytes user_data = 3;
160161
}
161162

162163
//

‎livekit-ffi/src/conversion/room.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ fn frame_metadata_features_from_proto(features: Vec<i32>) -> FrameMetadataFeatur
4444
proto::FrameMetadataFeature::FmfFrameId => {
4545
frame_metadata_features.frame_id = true;
4646
}
47+
proto::FrameMetadataFeature::FmfUserData => {
48+
frame_metadata_features.user_data = true;
49+
}
4750
}
4851
}
4952

‎livekit-ffi/src/conversion/track.rs‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ impl From<livekit_protocol::PacketTrailerFeature> for proto::FrameMetadataFeatur
173173
livekit_protocol::PacketTrailerFeature::PtfFrameId => {
174174
proto::FrameMetadataFeature::FmfFrameId
175175
}
176+
livekit_protocol::PacketTrailerFeature::PtfUserData => {
177+
proto::FrameMetadataFeature::FmfUserData
178+
}
176179
}
177180
}
178181
}

‎livekit-ffi/src/server/video_source.rs‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -29,11 +29,16 @@ impl FfiHandle for FfiVideoSource {}
2929

3030
fn frame_metadata_from_proto(metadata: Option<proto::FrameMetadata>) -> Option<FrameMetadata> {
3131
let metadata = metadata?;
32-
let frame_metadata =
33-
FrameMetadata { user_timestamp: metadata.user_timestamp, frame_id: metadata.frame_id };
32+
let frame_metadata = FrameMetadata {
33+
user_timestamp: metadata.user_timestamp,
34+
frame_id: metadata.frame_id,
35+
user_data: metadata.user_data,
36+
};
3437

35-
(frame_metadata.user_timestamp.is_some() || frame_metadata.frame_id.is_some())
36-
.then_some(frame_metadata)
38+
(frame_metadata.user_timestamp.is_some()
39+
|| frame_metadata.frame_id.is_some()
40+
|| frame_metadata.user_data.is_some())
41+
.then_some(frame_metadata)
3742
}
3843

3944
impl FfiVideoSource {
@@ -106,10 +111,12 @@ mod tests {
106111
let metadata = frame_metadata_from_proto(Some(proto::FrameMetadata {
107112
user_timestamp: Some(123),
108113
frame_id: Some(456),
114+
user_data: Some(vec![7, 8, 9]),
109115
}))
110116
.unwrap();
111117

112118
assert_eq!(metadata.user_timestamp, Some(123));
113119
assert_eq!(metadata.frame_id, Some(456));
120+
assert_eq!(metadata.user_data, Some(vec![7, 8, 9]));
114121
}
115122
}

0 commit comments

Comments
 (0)