Skip to content

Commit 70d8c88

Browse files
lostmsulostmsu
authored
AV1 support at transport level (no codecs) (#1553)
1 parent d33261e commit 70d8c88

7 files changed

Lines changed: 755 additions & 4 deletions

File tree

‎src/SIPSorcery/app/Media/Sources/VideoTestPatternSource.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,13 +40,15 @@ public class VideoTestPatternSource : IVideoSource, IDisposable
4040
private const int TIMER_DISPOSE_WAIT_MILLISECONDS = 1000;
4141
private const int VP8_SUGGESTED_FORMAT_ID = 96;
4242
private const int H264_SUGGESTED_FORMAT_ID = 100;
43+
private const int AV1_SUGGESTED_FORMAT_ID = 101;
4344

4445
public static readonly ILogger logger = LogFactory.CreateLogger<VideoTestPatternSource>();
4546

4647
public static readonly List<VideoFormat> SupportedFormats = new List<VideoFormat>
4748
{
4849
new VideoFormat(VideoCodecsEnum.VP8, VP8_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE),
49-
new VideoFormat(VideoCodecsEnum.H264, H264_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE, "packetization-mode=1")
50+
new VideoFormat(VideoCodecsEnum.H264, H264_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE, "packetization-mode=1"),
51+
new VideoFormat(VideoCodecsEnum.AV1, AV1_SUGGESTED_FORMAT_ID, VIDEO_SAMPLING_RATE)
5052
};
5153

5254
private int _frameSpacing;

‎src/SIPSorcery/net/RTP/Packetisation/AV1Depacketiser.cs‎

Lines changed: 237 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,237 @@
1+
//-----------------------------------------------------------------------------
2+
// Filename: AV1Depacketiser.cs
3+
//
4+
// Description: Reassembles RTP payloads using the AV1 RTP payload format.
5+
//
6+
// Based on the Alliance for Open Media RTP Payload Format for AV1:
7+
// https://aomediacodec.github.io/av1-rtp-spec/
8+
//
9+
// Author(s):
10+
// OpenAI
11+
//
12+
// History:
13+
// 28 Mar 2026 OpenAI Created, Vancouver.
14+
//
15+
// License:
16+
// BSD 3-Clause "New" or "Revised" License, see included LICENSE.md file.
17+
//-----------------------------------------------------------------------------
18+
19+
using System;
20+
using System.Collections.Generic;
21+
using System.IO;
22+
23+
namespace SIPSorcery.Net;
24+
25+
public class AV1Depacketiser
26+
{
27+
private const byte Z_MASK = 0x80;
28+
private const byte Y_MASK = 0x40;
29+
private const byte N_MASK = 0x08;
30+
31+
private uint _previousTimestamp;
32+
private readonly List<KeyValuePair<int, byte[]>> _temporaryRtpPayloads = new List<KeyValuePair<int, byte[]>>();
33+
private readonly MemoryStream _fragmentedObu = new MemoryStream();
34+
35+
public virtual MemoryStream ProcessRTPPayload(byte[] rtpPayload, ushort seqNum, uint timestamp, int markerBit, out bool isKeyFrame)
36+
{
37+
if (_previousTimestamp != timestamp && _previousTimestamp > 0)
38+
{
39+
_temporaryRtpPayloads.Clear();
40+
_previousTimestamp = 0;
41+
_fragmentedObu.SetLength(0);
42+
}
43+
44+
_temporaryRtpPayloads.Add(new KeyValuePair<int, byte[]>(seqNum, rtpPayload));
45+
46+
if (markerBit == 1)
47+
{
48+
if (_temporaryRtpPayloads.Count > 1)
49+
{
50+
_temporaryRtpPayloads.Sort((a, b) =>
51+
(Math.Abs(b.Key - a.Key) > (0xFFFF - 2000)) ? -a.Key.CompareTo(b.Key) : a.Key.CompareTo(b.Key));
52+
}
53+
54+
byte[] frame = ProcessAV1PayloadFrame(_temporaryRtpPayloads, out isKeyFrame);
55+
_temporaryRtpPayloads.Clear();
56+
_previousTimestamp = 0;
57+
_fragmentedObu.SetLength(0);
58+
59+
if (frame == null)
60+
{
61+
return null;
62+
}
63+
64+
var frameStream = new MemoryStream(frame.Length);
65+
frameStream.Write(frame, 0, frame.Length);
66+
frameStream.Position = 0;
67+
return frameStream;
68+
}
69+
70+
isKeyFrame = false;
71+
_previousTimestamp = timestamp;
72+
return null;
73+
}
74+
75+
protected virtual byte[] ProcessAV1PayloadFrame(List<KeyValuePair<int, byte[]>> rtpPayloads, out bool isKeyFrame)
76+
{
77+
var obuElements = new List<byte[]>();
78+
isKeyFrame = false;
79+
80+
foreach (var rtpPayload in rtpPayloads)
81+
{
82+
var payload = rtpPayload.Value;
83+
if (payload == null || payload.Length == 0)
84+
{
85+
continue;
86+
}
87+
88+
bool z = (payload[0] & Z_MASK) != 0;
89+
bool y = (payload[0] & Y_MASK) != 0;
90+
int w = (payload[0] >> 4) & 0x03;
91+
bool n = (payload[0] & N_MASK) != 0;
92+
93+
if (n)
94+
{
95+
isKeyFrame = true;
96+
}
97+
98+
var packetElements = ParseObuElements(payload, w);
99+
AddPacketElements(packetElements, z, y, obuElements);
100+
}
101+
102+
if (_fragmentedObu.Length > 0)
103+
{
104+
_fragmentedObu.SetLength(0);
105+
}
106+
107+
if (obuElements.Count == 0)
108+
{
109+
return null;
110+
}
111+
112+
int totalLength = 0;
113+
for (int i = 0; i < obuElements.Count; i++)
114+
{
115+
totalLength += obuElements[i].Length;
116+
}
117+
118+
var frame = new byte[totalLength];
119+
int offset = 0;
120+
for (int i = 0; i < obuElements.Count; i++)
121+
{
122+
Buffer.BlockCopy(obuElements[i], 0, frame, offset, obuElements[i].Length);
123+
offset += obuElements[i].Length;
124+
}
125+
126+
return frame;
127+
}
128+
129+
private List<byte[]> ParseObuElements(byte[] payload, int w)
130+
{
131+
var obuElements = new List<byte[]>();
132+
int offset = 1;
133+
134+
if (w == 0)
135+
{
136+
while (offset < payload.Length)
137+
{
138+
if (!AV1Packetiser.TryReadLeb128(payload, ref offset, out int obuElementLength, out _))
139+
{
140+
break;
141+
}
142+
143+
if (offset + obuElementLength > payload.Length)
144+
{
145+
break;
146+
}
147+
148+
var obuElement = new byte[obuElementLength];
149+
Buffer.BlockCopy(payload, offset, obuElement, 0, obuElementLength);
150+
offset += obuElementLength;
151+
obuElements.Add(obuElement);
152+
}
153+
}
154+
else
155+
{
156+
for (int elementIndex = 0; elementIndex < w && offset < payload.Length; elementIndex++)
157+
{
158+
int obuElementLength;
159+
if (elementIndex == w - 1)
160+
{
161+
obuElementLength = payload.Length - offset;
162+
}
163+
else if (!AV1Packetiser.TryReadLeb128(payload, ref offset, out obuElementLength, out _))
164+
{
165+
break;
166+
}
167+
168+
if (offset + obuElementLength > payload.Length)
169+
{
170+
break;
171+
}
172+
173+
var obuElement = new byte[obuElementLength];
174+
Buffer.BlockCopy(payload, offset, obuElement, 0, obuElementLength);
175+
offset += obuElementLength;
176+
obuElements.Add(obuElement);
177+
}
178+
}
179+
180+
return obuElements;
181+
}
182+
183+
private void AddPacketElements(List<byte[]> packetElements, bool z, bool y, List<byte[]> completedObus)
184+
{
185+
if (packetElements.Count == 0)
186+
{
187+
return;
188+
}
189+
190+
int startIndex = 0;
191+
int endExclusive = packetElements.Count;
192+
193+
if (z)
194+
{
195+
_fragmentedObu.Write(packetElements[0], 0, packetElements[0].Length);
196+
197+
if (!(y && packetElements.Count == 1))
198+
{
199+
AddCompletedObu(_fragmentedObu.ToArray(), completedObus);
200+
_fragmentedObu.SetLength(0);
201+
}
202+
203+
startIndex = 1;
204+
}
205+
206+
if (y && packetElements.Count > startIndex)
207+
{
208+
endExclusive = packetElements.Count - 1;
209+
}
210+
211+
for (int i = startIndex; i < endExclusive; i++)
212+
{
213+
AddCompletedObu(packetElements[i], completedObus);
214+
}
215+
216+
if (y && packetElements.Count > startIndex)
217+
{
218+
byte[] lastElement = packetElements[packetElements.Count - 1];
219+
_fragmentedObu.Write(lastElement, 0, lastElement.Length);
220+
}
221+
}
222+
223+
private static void AddCompletedObu(byte[] obu, List<byte[]> completedObus)
224+
{
225+
if (obu == null || obu.Length == 0)
226+
{
227+
return;
228+
}
229+
230+
var obuType = AV1Packetiser.GetObuType(obu);
231+
if (obuType != AV1Packetiser.AV1ObuType.TemporalDelimiter &&
232+
obuType != AV1Packetiser.AV1ObuType.TileList)
233+
{
234+
completedObus.Add(obu);
235+
}
236+
}
237+
}

0 commit comments

Comments
 (0)