Skip to content

Implement turns #1434

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
paulomorgado wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

Implement turns #1434

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
14 changes: 10 additions & 4 deletions examples/OpenAIExamples/AliceAndBob/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@
using SIPSorcery.OpenAI.Realtime.Models;
using SIPSorceryMedia.Abstractions;
using SIPSorcery.Media;
using System.Buffers;
using CommunityToolkit.HighPerformance.Buffers;

namespace demo;

Expand Down Expand Up @@ -161,11 +163,15 @@ private static void InitialiseWindowsAudioEndPoint(IWebRTCEndPoint webrtcEndPoin

webrtcEndPoint.OnAudioFrameReceived += (EncodedAudioFrame encodedAudioFrame) =>
{
var decodedSample = audioEncoder.DecodeAudio(encodedAudioFrame.EncodedAudio, AudioCommonlyUsedFormats.OpusWebRTC);
using var buffer = new ArrayPoolBufferWriter<short>(8192);
audioEncoder.DecodeAudio(encodedAudioFrame.EncodedAudio.Span, AudioCommonlyUsedFormats.OpusWebRTC, buffer);
var decodedSample = buffer.WrittenSpan;

var samples = decodedSample
.Select(s => new Complex(s / 32768f, 0f))
.ToArray();
var samples = new Complex[decodedSample.Length];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = new Complex(decodedSample[i] / 32768f, 0f);
}

var frame = _audioScopeForm?.Invoke(() => _audioScopeForm.ProcessAudioSample(samples, audioScopeNumber));
};
Expand Down
26 changes: 19 additions & 7 deletions examples/SIPExamples/CustomAudioCodec/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,18 +14,23 @@
//-----------------------------------------------------------------------------

using System;
using System.Buffers;
using System.Buffers.Binary;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Threading;
using System.Threading.Tasks;
using CommunityToolkit.HighPerformance;
using CommunityToolkit.HighPerformance.Buffers;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Extensions.Logging;
using SIPSorcery.Media;
using SIPSorcery.SIP;
using SIPSorcery.SIP.App;
using SIPSorceryMedia.Windows;
using SIPSorceryMedia.Abstractions;
using SIPSorceryMedia.Windows;

namespace demo
{
Expand All @@ -50,15 +55,21 @@ public G729Codec()
_g729Decoder = new G729Decoder();
}

public short[] DecodeAudio(byte[] encodedSample, AudioFormat format)
public void DecodeAudio(ReadOnlySpan<byte> encodedSample, AudioFormat format, IBufferWriter<short> destination)
{
var pcm = _g729Decoder.Process(encodedSample);
return pcm.Where((x, i) => i % 2 == 0).Select((y, i) => (short)(pcm[i * 2 + 1] << 8 | pcm[i * 2])).ToArray();
using var buffer = new ArrayPoolBufferWriter<byte>(8192);
_g729Decoder.Process(encodedSample, buffer);

var pcm = buffer.WrittenSpan;
for (int i = 0; i < pcm.Length / 2; i++)
{
destination.Write(BinaryPrimitives.ReadInt16LittleEndian(pcm.Slice(i * 2, 2)));
}
}

public byte[] EncodeAudio(short[] pcm, AudioFormat format)
public void EncodeAudio(ReadOnlySpan<short> pcm, AudioFormat format, IBufferWriter<byte> destination)
{
return _g729Encoder.Process(pcm.SelectMany(x => new byte[] { (byte)(x), (byte)(x >> 8) }).ToArray());
_g729Encoder.Process(MemoryMarshal.AsBytes(pcm), destination);
}
}

Expand Down Expand Up @@ -105,7 +116,8 @@ static async Task Main()
Console.WriteLine("Hanging up established call.");
userAgent.Hangup();
}
};
}
;

exitCts.Cancel();
};
Expand Down
4 changes: 2 additions & 2 deletions examples/SIPExamples/RecordCall/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -102,13 +102,13 @@ private static void OnRtpPacketReceived(IPEndPoint remoteEndPoint, SDPMediaTypes
{
if (rtpPacket.Header.PayloadType == (int)SDPWellKnownMediaFormatsEnum.PCMA)
{
short pcm = NAudio.Codecs.ALawDecoder.ALawToLinearSample(sample[index]);
short pcm = NAudio.Codecs.ALawDecoder.ALawToLinearSample(sample.Span[index]);
byte[] pcmSample = new byte[] { (byte)(pcm & 0xFF), (byte)(pcm >> 8) };
_waveFile.Write(pcmSample, 0, 2);
}
else
{
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample[index]);
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample.Span[index]);
byte[] pcmSample = new byte[] { (byte)(pcm & 0xFF), (byte)(pcm >> 8) };
_waveFile.Write(pcmSample, 0, 2);
}
Expand Down
4 changes: 2 additions & 2 deletions examples/SIPExamples/RecordIncomingCall/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,13 +81,13 @@ private static void OnRtpPacketReceived(IPEndPoint remoteEndPoint, SDPMediaTypes
{
if (rtpPacket.Header.PayloadType == (int)SDPWellKnownMediaFormatsEnum.PCMA)
{
short pcm = NAudio.Codecs.ALawDecoder.ALawToLinearSample(sample[index]);
short pcm = NAudio.Codecs.ALawDecoder.ALawToLinearSample(sample.Span[index]);
byte[] pcmSample = new byte[] { (byte)(pcm & 0xFF), (byte)(pcm >> 8) };
_waveFile.Write(pcmSample, 0, 2);
}
else
{
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample[index]);
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample.Span[index]);
byte[] pcmSample = new byte[] { (byte)(pcm & 0xFF), (byte)(pcm >> 8) };
_waveFile.Write(pcmSample, 0, 2);
}
Expand Down
2 changes: 1 addition & 1 deletion examples/SIPExamples/SIPCallResampleAudio/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ private static void RtpSession_OnRtpPacketReceived(IPEndPoint remoteEndPoint, SD

for (int index = 0; index < sample.Length; index++)
{
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample[index]);
short pcm = NAudio.Codecs.MuLawDecoder.MuLawToLinearSample(sample.Span[index]);
float s16 = pcm / 32768f;

for (int i = 0; i < _ratio; i++)
Expand Down
5 changes: 3 additions & 2 deletions examples/SIPExamples/VideoPhoneCmdLine/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
//-----------------------------------------------------------------------------

using System;
using System.Buffers;
using System.Collections.Generic;
using System.Drawing;
using System.Linq;
Expand Down Expand Up @@ -110,13 +111,13 @@ public DecoderVideoSink(IVideoEncoder videoDecoder)
public void GotVideoRtp(IPEndPoint remoteEndPoint, uint ssrc, uint seqnum, uint timestamp, int payloadID, bool marker, byte[] payload) =>
throw new ApplicationException("This Video End Point requires full video frames rather than individual RTP packets.");

public void GotVideoFrame(IPEndPoint remoteEndPoint, uint timestamp, byte[] frame, VideoFormat format)
public void GotVideoFrame(IPEndPoint remoteEndPoint, uint timestamp, ReadOnlyMemory<byte> frame, VideoFormat format)
{
if (OnVideoSinkDecodedSample != null)
{
try
{
foreach (var decoded in _videoDecoder.DecodeVideo(frame, VideoPixelFormatsEnum.Bgr, format.Codec))
foreach (var decoded in _videoDecoder.DecodeVideo(frame.ToArray(), VideoPixelFormatsEnum.Bgr, format.Codec))
{
OnVideoSinkDecodedSample(decoded.Sample, decoded.Width, decoded.Height, (int)(decoded.Width * 3), VideoPixelFormatsEnum.Bgr);
}
Expand Down
17 changes: 9 additions & 8 deletions examples/TurnServerExample/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//-----------------------------------------------------------------------------
//-----------------------------------------------------------------------------
// Filename: Program.cs
//
// Description: An example TURN server (RFC 5766) console application that
Expand Down Expand Up @@ -104,13 +104,13 @@ static async Task RunClientDemo(int serverPort)
var challenge = await ReceiveStun(stream);
var errorAttr = challenge.Attributes.FirstOrDefault(
a => a.AttributeType == STUNAttributeTypesEnum.ErrorCode);
int errorCode = errorAttr.Value[2] * 100 + errorAttr.Value[3];
int errorCode = errorAttr.Value.Span[2] * 100 + errorAttr.Value.Span[3];
Console.WriteLine($"[Client] Got {errorCode} challenge with REALM and NONCE.");

// Extract nonce from the challenge
var nonce = Encoding.UTF8.GetString(
challenge.Attributes.First(
a => a.AttributeType == STUNAttributeTypesEnum.Nonce).Value);
a => a.AttributeType == STUNAttributeTypesEnum.Nonce).Value.Span);

Console.WriteLine("[Client] Sending authenticated Allocate...");
var authReq = new STUNMessage(STUNMessageTypesEnum.Allocate);
Expand Down Expand Up @@ -191,12 +191,12 @@ await peer.SendAsync(response, response.Length,
while (read < remaining)
read += await stream.ReadAsync(full, 4 + read, remaining - read);

var dataInd = STUNMessage.ParseSTUNMessage(full, full.Length);
var dataInd = STUNMessage.ParseSTUNMessage(full);
var dataAttr = dataInd.Attributes.FirstOrDefault(
a => a.AttributeType == STUNAttributeTypesEnum.Data);
if (dataAttr != null)
{
Console.WriteLine($"[Client] Received via relay: \"{Encoding.UTF8.GetString(dataAttr.Value)}\"\n");
Console.WriteLine($"[Client] Received via relay: \"{Encoding.UTF8.GetString(dataAttr.Value.Span)}\"\n");
}
}
}
Expand Down Expand Up @@ -229,8 +229,9 @@ static byte[] ComputeHmacKey(string username, string realm, string password)

static async Task SendStun(NetworkStream stream, STUNMessage msg, byte[] hmacKey = null)
{
var bytes = msg.ToByteBuffer(hmacKey, false);
await stream.WriteAsync(bytes, 0, bytes.Length);
var bytes = new byte[msg.GetByteBufferSize(hmacKey, false)];
msg.WriteToBuffer(bytes, hmacKey, false);
await stream.WriteAsync(bytes);
await stream.FlushAsync();
}

Expand All @@ -250,7 +251,7 @@ static async Task<STUNMessage> ReceiveStun(NetworkStream stream)
while (read < remaining)
read += await stream.ReadAsync(full, 4 + read, remaining - read);

return STUNMessage.ParseSTUNMessage(full, full.Length);
return STUNMessage.ParseSTUNMessage(full);
}

#endregion
Expand Down
1 change: 1 addition & 0 deletions examples/WebRTCExamples/FfmpegToWebRTC/FfmpegToWebRTC.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<OutputType>Exe</OutputType>
<TargetFramework>net8.0</TargetFramework>
<PlatformTarget>x64</PlatformTarget>
<LangVersion>9.0</LangVersion>
</PropertyGroup>

<PropertyGroup>
Expand Down
2 changes: 1 addition & 1 deletion examples/WebRTCExamples/FfmpegToWebRTC/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -270,7 +270,7 @@ private static RTCPeerConnection Createpc(WebSocketContext context, SDPAudioVide
if (media == SDPMediaTypesEnum.video && pc.VideoDestinationEndPoint != null)
{
//logger.LogDebug($"Forwarding {media} RTP packet to webrtc peer timestamp {rtpPkt.Header.Timestamp}.");
pc.SendRtpRaw(media, rtpPkt.Payload, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
pc.SendRtpRaw(media, rtpPkt.Payload.Span, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
}
};
}
Expand Down
4 changes: 3 additions & 1 deletion examples/WebRTCExamples/JanusWebRTCStream/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,11 @@
//-----------------------------------------------------------------------------

using System;
using System.Buffers;
using System.Drawing;
using System.Drawing.Imaging;
using System.Linq;
using System.Net;
using System.Threading;
using System.Threading.Tasks;
using System.Windows.Forms;
Expand Down Expand Up @@ -86,7 +88,7 @@ static async Task Main()

MediaStreamTrack videoTrack = new MediaStreamTrack(videoSink.GetVideoSourceFormats(), MediaStreamStatusEnum.SendRecv);
pc.addTrack(videoTrack);
pc.OnVideoFrameReceived += videoSink.GotVideoFrame;
pc.OnVideoFrameReceived += (remoteEndPoint, timestamp, frame, format) => videoSink.GotVideoFrame(remoteEndPoint, timestamp, frame.ToArray(), format);
videoSource.OnVideoSourceEncodedSample += pc.SendVideo;

pc.OnVideoFormatsNegotiated += (formats) =>
Expand Down
4 changes: 2 additions & 2 deletions examples/WebRTCExamples/RtspToWebRTCAudioAndVideo/WebSocketSignalingServer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,7 +114,7 @@ private RTCPeerConnection Createpc(WebSocketContext context)
{
try
{
pc.SendRtpRaw(media, rtpPkt.Payload, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
pc.SendRtpRaw(media, rtpPkt.Payload.Span, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
}
catch (Exception ex)
{
Expand All @@ -125,7 +125,7 @@ private RTCPeerConnection Createpc(WebSocketContext context)
{
try
{
pc.SendRtpRaw(media, rtpPkt.Payload, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
pc.SendRtpRaw(media, rtpPkt.Payload.Span, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
}
catch (Exception ex)
{
Expand Down
5 changes: 3 additions & 2 deletions examples/WebRTCExamples/SIPToWebRtcBridgeVideo/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
//-----------------------------------------------------------------------------

using System;
using System.Buffers;
using System.Collections.Generic;
using System.Linq;
using System.Net;
Expand Down Expand Up @@ -226,15 +227,15 @@ private static void ForwardAudioToPeerConnection(IPEndPoint remote, SDPMediaType
}
}

private static void ForwardVideoFrameToSIP(IPEndPoint remoteEP, uint timestamp, byte[] frame, VideoFormat format)
private static void ForwardVideoFrameToSIP(IPEndPoint remoteEP, uint timestamp, ReadOnlyMemory<byte> frame, VideoFormat format)
{
if (_rtpSession != null && !_rtpSession.IsClosed)
{
_rtpSession.SendVideo(3000, frame);
}
}

private static void ForwardVideoFrameToPeerConnection(IPEndPoint remoteEP, uint timestamp, byte[] frame, VideoFormat format)
private static void ForwardVideoFrameToPeerConnection(IPEndPoint remoteEP, uint timestamp, ReadOnlyMemory<byte> frame, VideoFormat format)
{
if (_peerConnection != null && _peerConnection.connectionState == RTCPeerConnectionState.connected)
{
Expand Down
2 changes: 1 addition & 1 deletion examples/WebRTCExamples/WebRTCEchoServer/WebRTCEchoServer.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -90,7 +90,7 @@ public async Task<RTCSessionDescriptionInit> GotOffer(RTCSessionDescriptionInit

pc.OnRtpPacketReceived += (IPEndPoint rep, SDPMediaTypesEnum media, RTPPacket rtpPkt) =>
{
pc.SendRtpRaw(media, rtpPkt.Payload, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
pc.SendRtpRaw(media, rtpPkt.Payload.Span, rtpPkt.Header.Timestamp, rtpPkt.Header.MarkerBit, rtpPkt.Header.PayloadType);
//_logger.LogDebug($"RTP {media} pkt received, SSRC {rtpPkt.Header.SyncSource}, SeqNum {rtpPkt.Header.SequenceNumber}.");
};
pc.OnRtpEvent += async (ep, ev, hdr) =>
Expand Down
2 changes: 1 addition & 1 deletion examples/WebRTCExamples/WebRTCGetStartedDataChannel/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,7 +197,7 @@ private static string DoJavscriptSHA256(byte[] buffer)

using (var sha256 = SHA256.Create())
{
return sha256.ComputeHash(hashOfHashes).HexStr();
return TypeExtensions.HexStr(sha256.ComputeHash(hashOfHashes));
}
}

Expand Down
16 changes: 11 additions & 5 deletions examples/WebRTCExamples/WebRTCOpenGL/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@
using AudioScope;
using System.Numerics;
using SIPSorceryMedia.Abstractions;
using System.Buffers;
using CommunityToolkit.HighPerformance.Buffers;

namespace demo
{
Expand Down Expand Up @@ -158,11 +160,15 @@ private static Task<RTCPeerConnection> CreatePeerConnection()

if (media == SDPMediaTypesEnum.audio)
{
var decodedSample = audioEncoder.DecodeAudio(rtpPkt.Payload, pc.AudioStream.NegotiatedFormat.ToAudioFormat());

var samples = decodedSample
.Select(s => new Complex(s / 32768f, 0f))
.ToArray();
using var buffer = new ArrayPoolBufferWriter<short>(8192);
audioEncoder.DecodeAudio(rtpPkt.Payload.Span, pc.AudioStream.NegotiatedFormat.ToAudioFormat(), buffer);
var decodedSample = buffer.WrittenSpan;

var samples = new Complex[decodedSample.Length];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = new Complex(decodedSample[i] / 32768f, 0f);
}

var frame = _audioScopeForm.Invoke(() => _audioScopeForm.ProcessAudioSample(samples));

Expand Down
27 changes: 16 additions & 11 deletions examples/WebRTCExamples/WebRTCOpenGLSource/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
using AudioScope;
using System.Numerics;
using SIPSorceryMedia.Abstractions;
using CommunityToolkit.HighPerformance.Buffers;

namespace demo;

Expand Down Expand Up @@ -157,23 +158,27 @@ private static Task<RTCPeerConnection> CreatePeerConnection()
}
};

audioSource.OnAudioSourceEncodedSample += (uint durationRtpUnits, byte[] sample) =>
audioSource.OnAudioSourceEncodedSample += (durationRtpUnits, sample) =>
{
//logger.LogDebug($"RTP {media} pkt received, SSRC {rtpPkt.Header.SyncSource}, payload {rtpPkt.Header.PayloadType}, SeqNum {rtpPkt.Header.SequenceNumber}.");

var decodedSample = audioEncoder.DecodeAudio(sample, pc.AudioStream.NegotiatedFormat.ToAudioFormat());
using var buffer = new ArrayPoolBufferWriter<short>(8192);
audioEncoder.DecodeAudio(sample.Span, pc.AudioStream.NegotiatedFormat.ToAudioFormat(), buffer);
var decodedSample = buffer.WrittenSpan;

var samples = decodedSample
.Select(s => new Complex(s / 32768f, 0f))
.ToArray();
var samples = new Complex[decodedSample.Length];
for (int i = 0; i < samples.Length; i++)
{
samples[i] = new Complex(decodedSample[i] / 32768f, 0f);
}

var frame = _audioScopeForm.Invoke(() => _audioScopeForm.ProcessAudioSample(samples));
var frame = _audioScopeForm.Invoke(() => _audioScopeForm.ProcessAudioSample(samples));

videoEncoderEndPoint.ExternalVideoSourceRawSample(AUDIO_PACKET_DURATION,
FormAudioScope.AUDIO_SCOPE_WIDTH,
FormAudioScope.AUDIO_SCOPE_HEIGHT,
frame,
VideoPixelFormatsEnum.Rgb);
videoEncoderEndPoint.ExternalVideoSourceRawSample(AUDIO_PACKET_DURATION,
FormAudioScope.AUDIO_SCOPE_WIDTH,
FormAudioScope.AUDIO_SCOPE_HEIGHT,
frame,
VideoPixelFormatsEnum.Rgb);
};

pc.onconnectionstatechange += async (state) =>
Expand Down
2 changes: 1 addition & 1 deletion examples/WebRTCExamples/WebRTCReceiveAudio/Program.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,7 @@ private static async Task<RTCPeerConnection> SendSDPOffer(WebSocketContext conte

if (media == SDPMediaTypesEnum.audio)
{
windowsAudioEP.GotAudioRtp(rep, rtpPkt.Header.SyncSource, rtpPkt.Header.SequenceNumber, rtpPkt.Header.Timestamp, rtpPkt.Header.PayloadType, rtpPkt.Header.MarkerBit == 1, rtpPkt.Payload);
windowsAudioEP.GotAudioRtp(rep, rtpPkt.Header.SyncSource, rtpPkt.Header.SequenceNumber, rtpPkt.Header.Timestamp, rtpPkt.Header.PayloadType, rtpPkt.Header.MarkerBit == 1, rtpPkt.Payload.ToArray());
}
};

Expand Down
Loading
Loading