Skip to content

Commit 8ef8d30

Browse files
hwangsihuhwangsihu
committed
Enable Opus in-band FEC recovery and voice encoder hints
1 parent 174641b commit 8ef8d30

7 files changed

Lines changed: 171 additions & 20 deletions

File tree

‎Library/TeamTalkLib/codec/OpusDecoder.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,13 +62,15 @@ void OpusDecode::Reset()
6262
}
6363
}
6464

65-
int OpusDecode::Decode(const char* input_buffer, int input_bufsize,
66-
short* output_buffer, int output_samples)
65+
int OpusDecode::Decode(const char* input_buffer, int input_bufsize,
66+
short* output_buffer, int output_samples, bool fec)
6767
{
6868
assert(m_decoder);
6969
assert(output_buffer);
70-
return opus_decode(m_decoder,
71-
reinterpret_cast<const unsigned char*>((input_buffer != nullptr)?input_buffer:nullptr),
72-
(input_buffer != nullptr)?input_bufsize:0, output_buffer,
73-
output_samples, (input_buffer != nullptr)?0:1);
70+
71+
auto const data = reinterpret_cast<const unsigned char*>(input_buffer);
72+
int const len = (input_buffer != nullptr) ? input_bufsize : 0;
73+
int const decode_fec = (fec && input_buffer != nullptr) ? 1 : 0;
74+
return opus_decode(m_decoder, (input_buffer != nullptr) ? data : nullptr,
75+
len, output_buffer, output_samples, decode_fec);
7476
}

‎Library/TeamTalkLib/codec/OpusDecoder.h‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,8 @@ class OpusDecode : private NonCopyable
3737
void Close();
3838
void Reset();
3939

40-
int Decode(const char* input_buffer, int input_bufsize,
41-
short* output_buffer, int output_samples);
40+
int Decode(const char* input_buffer, int input_bufsize,
41+
short* output_buffer, int output_samples, bool fec = false);
4242

4343
private:
4444
OpusDecoder* m_decoder;

‎Library/TeamTalkLib/codec/OpusEncoder.cpp‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,40 @@ bool OpusEncode::SetFEC(bool enable)
119119
return err == 0;
120120
}
121121

122+
bool OpusEncode::SetPacketLossPerc(int perc)
123+
{
124+
assert(m_encoder);
125+
if(m_encoder == nullptr)
126+
return false;
127+
128+
int const err = opus_encoder_ctl(m_encoder, OPUS_SET_PACKET_LOSS_PERC(perc));
129+
assert(err == 0);
130+
return err == 0;
131+
}
132+
133+
bool OpusEncode::SetSignalVoice(bool voice)
134+
{
135+
assert(m_encoder);
136+
if(m_encoder == nullptr)
137+
return false;
138+
139+
int const value = voice ? OPUS_SIGNAL_VOICE : OPUS_AUTO;
140+
int const err = opus_encoder_ctl(m_encoder, OPUS_SET_SIGNAL(value));
141+
assert(err == 0);
142+
return err == 0;
143+
}
144+
145+
bool OpusEncode::SetLSBDepth(int bits)
146+
{
147+
assert(m_encoder);
148+
if(m_encoder == nullptr)
149+
return false;
150+
151+
int const err = opus_encoder_ctl(m_encoder, OPUS_SET_LSB_DEPTH(bits));
152+
assert(err == 0);
153+
return err == 0;
154+
}
155+
122156
bool OpusEncode::SetBitrate(int bitrate)
123157
{
124158
assert(m_encoder);

‎Library/TeamTalkLib/codec/OpusEncoder.h‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,8 @@
3030
int OpusGetCbSize(int samplerate, int msec);
3131
int OpusGetCbMSec(int samplerate, int cb_samples);
3232

33+
constexpr int OPUS_DEFAULT_PACKET_LOSS_PERC = 10;
34+
3335
class OpusEncode : private NonCopyable
3436
{
3537
public:
@@ -41,10 +43,13 @@ class OpusEncode : private NonCopyable
4143

4244
bool SetComplexity(int complex);
4345
bool SetFEC(bool enable);
46+
bool SetPacketLossPerc(int perc);
4447
bool SetBitrate(int bitrate);
4548
bool SetVBR(bool enable);
4649
bool SetVBRConstraint(bool enable);
4750
bool SetDTX(bool enable);
51+
bool SetSignalVoice(bool voice);
52+
bool SetLSBDepth(int bits);
4853

4954
int Encode(const short* input_buffer, int input_samples,
5055
char* output_buffer, int output_bufsize);

‎Library/TeamTalkLib/teamtalk/client/AudioThread.cpp‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -130,10 +130,13 @@ bool AudioThread::StartEncoder(const audioencodercallback_t& callback,
130130
codec.opus.application) ||
131131
!m_opus->SetComplexity(codec.opus.complexity) ||
132132
!m_opus->SetFEC(codec.opus.fec) ||
133+
!m_opus->SetPacketLossPerc(codec.opus.fec ? OPUS_DEFAULT_PACKET_LOSS_PERC : 0) ||
133134
!m_opus->SetDTX(codec.opus.dtx) ||
134135
!m_opus->SetBitrate(codec.opus.bitrate) ||
135136
!m_opus->SetVBR(codec.opus.vbr) ||
136-
!m_opus->SetVBRConstraint(codec.opus.vbr_constraint))
137+
!m_opus->SetVBRConstraint(codec.opus.vbr_constraint) ||
138+
!m_opus->SetSignalVoice(codec.opus.application == OPUS_APPLICATION_VOIP) ||
139+
!m_opus->SetLSBDepth(16))
137140
{
138141
StopEncoder();
139142
return false;

‎Library/TeamTalkLib/teamtalk/client/StreamPlayers.cpp‎

Lines changed: 35 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -583,17 +583,41 @@ bool OpusPlayer::DecodeFrame(const encframe& enc_frame,
583583
}
584584
return true;
585585
}
586-
//packet lost
587-
MYTRACE(ACE_TEXT("User #%d is missing packet %d\n"), m_userid, m_play_pkt_no);
588-
int fpp = GetAudioCodecFramesPerPacket(m_codec);
589-
int decoffset = 0;
590-
for (int i=0;i<fpp;i++)
591-
{
592-
m_decoder.Decode(NULL, 0, &output_buffer[decoffset*channels], framesize);
593-
decoffset += framesize;
594-
}
595-
//increment 'm_played_packet_time' with GetAudioCodecCbMillis()?
596-
return false;
586+
//packet lost
587+
MYTRACE(ACE_TEXT("User #%d is missing packet %d\n"), m_userid, m_play_pkt_no);
588+
589+
int const fpp = GetAudioCodecFramesPerPacket(m_codec);
590+
591+
// Opus in-band FEC: the LBRR copy of the last lost frame is carried in
592+
// the first sub-frame of the next packet. With fpp > 1 only the last
593+
// sub-frame is recoverable; earlier ones fall back to PLC.
594+
const encframe* next = nullptr;
595+
auto nextii = m_buffer.find(static_cast<uint16_t>(m_play_pkt_no + 1));
596+
if (nextii != m_buffer.end() && !nextii->second.enc_frames.empty() &&
597+
!nextii->second.enc_frame_sizes.empty() &&
598+
nextii->second.stream_id == m_stream_id)
599+
next = &nextii->second;
600+
601+
int decoffset = 0;
602+
for (int i = 0; i < fpp; i++)
603+
{
604+
bool const fec = (next != nullptr) && (i == fpp - 1);
605+
if (fec)
606+
ret = m_decoder.Decode(next->enc_frames.data(), next->enc_frame_sizes[0],
607+
&output_buffer[decoffset*channels], framesize, true);
608+
else
609+
ret = m_decoder.Decode(NULL, 0, &output_buffer[decoffset*channels],
610+
framesize, false);
611+
MYTRACE_COND(ret != framesize,
612+
ACE_TEXT("OPUS %s failed for #%d. Ret = %d\n"),
613+
fec ? ACE_TEXT("FEC decode") : ACE_TEXT("PLC"), m_userid, ret);
614+
decoffset += framesize;
615+
}
616+
MYTRACE_COND(next != nullptr,
617+
ACE_TEXT("User #%d recovered packet %d via Opus FEC\n"),
618+
m_userid, m_play_pkt_no);
619+
//increment 'm_played_packet_time' with GetAudioCodecCbMillis()?
620+
return false;
597621

598622
}
599623
#endif

‎Library/TeamTalkLib/test/CatchDefault.cpp‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
#if defined(ENABLE_OPUS)
4848
#include "avstream/OpusFileStreamer.h"
4949
#include "codec/OpusDecoder.h"
50+
#include "codec/OpusEncoder.h"
5051
#endif
5152

5253
#if defined(ENABLE_FFMPEG)
@@ -3641,6 +3642,88 @@ TEST_CASE("OPUSFileSeek")
36413642
REQUIRE(opusdecfile.GetDurationMSec() == mfi.uDurationMSec);
36423643
}
36433644

3645+
TEST_CASE("OpusFECRecovery")
3646+
{
3647+
const int SAMPLERATE = 48000;
3648+
const int CHANNELS = 1;
3649+
const int FRAMESIZE = SAMPLERATE * 20 / 1000;
3650+
const int N_FRAMES = 12;
3651+
const int LOST_FRAME = 4;
3652+
3653+
OpusEncode enc;
3654+
REQUIRE(enc.Open(SAMPLERATE, CHANNELS, OPUS_APPLICATION_VOIP));
3655+
REQUIRE(enc.SetBitrate(16000));
3656+
REQUIRE(enc.SetComplexity(10));
3657+
REQUIRE(enc.SetFEC(true));
3658+
REQUIRE(enc.SetPacketLossPerc(30));
3659+
REQUIRE(enc.SetSignalVoice(true));
3660+
REQUIRE(enc.SetLSBDepth(16));
3661+
REQUIRE(enc.SetDTX(false));
3662+
REQUIRE(enc.SetVBR(false));
3663+
3664+
std::vector<short> input(FRAMESIZE * N_FRAMES);
3665+
for (int f = 0; f < N_FRAMES; ++f)
3666+
{
3667+
int period = 60 + f * 8;
3668+
int half = period / 2;
3669+
for (int s = 0; s < FRAMESIZE; ++s)
3670+
{
3671+
int phase = s % period;
3672+
int tri = (phase < half) ? phase : (period - phase);
3673+
input[f * FRAMESIZE + s] = static_cast<short>((tri - half / 2) * 400);
3674+
}
3675+
}
3676+
3677+
std::vector<std::vector<char>> packets(N_FRAMES);
3678+
for (int i = 0; i < N_FRAMES; ++i)
3679+
{
3680+
std::vector<char> buf(4000);
3681+
int ret = enc.Encode(&input[i * FRAMESIZE], FRAMESIZE, buf.data(), int(buf.size()));
3682+
REQUIRE(ret > 0);
3683+
buf.resize(ret);
3684+
packets[i] = std::move(buf);
3685+
}
3686+
3687+
OpusDecode decPLC, decFEC;
3688+
REQUIRE(decPLC.Open(SAMPLERATE, CHANNELS));
3689+
REQUIRE(decFEC.Open(SAMPLERATE, CHANNELS));
3690+
3691+
std::vector<short> outPLC(FRAMESIZE), outFEC(FRAMESIZE), warm(FRAMESIZE);
3692+
3693+
for (int i = 0; i < LOST_FRAME; ++i)
3694+
{
3695+
REQUIRE(decPLC.Decode(packets[i].data(), int(packets[i].size()), warm.data(), FRAMESIZE) == FRAMESIZE);
3696+
REQUIRE(decFEC.Decode(packets[i].data(), int(packets[i].size()), warm.data(), FRAMESIZE) == FRAMESIZE);
3697+
}
3698+
3699+
int retPLC = decPLC.Decode(nullptr, 0, outPLC.data(), FRAMESIZE, false);
3700+
int retFEC = decFEC.Decode(packets[LOST_FRAME + 1].data(),
3701+
int(packets[LOST_FRAME + 1].size()),
3702+
outFEC.data(), FRAMESIZE, true);
3703+
REQUIRE(retPLC == FRAMESIZE);
3704+
REQUIRE(retFEC == FRAMESIZE);
3705+
3706+
long long fecEnergy = 0;
3707+
for (short s : outFEC) fecEnergy += (s < 0 ? -int(s) : int(s));
3708+
REQUIRE(fecEnergy > 0);
3709+
3710+
bool differs = false;
3711+
for (int i = 0; i < FRAMESIZE; ++i)
3712+
if (outPLC[i] != outFEC[i]) { differs = true; break; }
3713+
REQUIRE(differs);
3714+
3715+
const short* truth = &input[LOST_FRAME * FRAMESIZE];
3716+
long long ssePLC = 0, sseFEC = 0;
3717+
for (int i = 0; i < FRAMESIZE; ++i)
3718+
{
3719+
long long dP = outPLC[i] - truth[i];
3720+
long long dF = outFEC[i] - truth[i];
3721+
ssePLC += dP * dP;
3722+
sseFEC += dF * dF;
3723+
}
3724+
REQUIRE(sseFEC < ssePLC);
3725+
}
3726+
36443727
TEST_CASE("OPUSStreamer")
36453728
{
36463729
const auto IN_SAMPLERATE = 12000;

0 commit comments

Comments
 (0)