Skip to content

Commit b6ea338

Browse files
FiloppiFiloppi
committed
cubeb - re-enable and polish IAudioClient3 to achieve lower latencies
1 parent 27d2a10 commit b6ea338

1 file changed

Lines changed: 181 additions & 59 deletions

File tree

‎src/cubeb_wasapi.cpp‎

Lines changed: 181 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,31 @@
3333
#include "cubeb_tracing.h"
3434
#include "cubeb_utils.h"
3535

36+
// Some people have reported glitches with IAudioClient3 capture streams:
37+
// http://blog.nirbheek.in/2018/03/low-latency-audio-on-windows-with.html
38+
// https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
39+
#define ALLOW_AUDIO_CLIENT_3_FOR_INPUT 0
40+
// IAudioClient3::GetSharedModeEnginePeriod() seem to return min latencies
41+
// bigger than IAudioClient::GetDevicePeriod(), which is confusing (10ms vs
42+
// 3ms), though the default latency is usually the same and we should use the
43+
// IAudioClient3 function anyway, as it's more correct
44+
#define USE_AUDIO_CLIENT_3_MIN_PERIOD 1
45+
// If this is true, we allow IAudioClient3 the creation of sessions with a
46+
// latency above the default one (usually 10ms).
47+
// Whether we should default this to true or false depend on many things:
48+
// -Does creating a shared IAudioClient3 session (not locked to a format)
49+
// actually forces all the IAudioClient(1) sessions to have the same latency?
50+
// I could find no proof of that.
51+
// -Does creating a shared IAudioClient3 session with a latency >= the default
52+
// one actually improve the latency (as in how late the audio is) at all?
53+
// -Maybe we could expose this as cubeb stream pref
54+
// (e.g. take priority over other apps)?
55+
#define ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT 1
56+
// If this is true and the user specified a target latency >= the IAudioClient3
57+
// one, then we reject it and fall back to IAudioClient(1). There wouldn't be
58+
// much point in having a low latency if that's not what the user wants.
59+
#define REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX 0
60+
3661
// Windows 10 exposes the IAudioClient3 interface to create low-latency streams.
3762
// Copy the interface definition from audioclient.h here to make the code
3863
// simpler and so that we can still access IAudioClient3 via COM if cubeb was
@@ -1867,6 +1892,44 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
18671892
return CUBEB_ERROR;
18681893
}
18691894

1895+
#if USE_AUDIO_CLIENT_3_MIN_PERIOD
1896+
// This is unreliable as we can't know the actual mixer format cubeb will
1897+
// ask for later on (nor we can branch on ALLOW_AUDIO_CLIENT_3_FOR_INPUT),
1898+
// and the min latency can change based on that.
1899+
com_ptr<IAudioClient3> client3;
1900+
hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
1901+
client3.receive_vpp());
1902+
if (SUCCEEDED(hr)) {
1903+
WAVEFORMATEX * mix_format = nullptr;
1904+
hr = client3->GetMixFormat(&mix_format);
1905+
1906+
if (SUCCEEDED(hr)) {
1907+
uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
1908+
max_period = 0;
1909+
hr = client3->GetSharedModeEnginePeriod(mix_format, &default_period,
1910+
&fundamental_period, &min_period,
1911+
&max_period);
1912+
1913+
auto sample_rate = mix_format->nSamplesPerSec;
1914+
CoTaskMemFree(mix_format);
1915+
if (SUCCEEDED(hr)) {
1916+
// Print values in the same format as IAudioDevice::GetDevicePeriod()
1917+
REFERENCE_TIME min_period_rt(frames_to_hns(sample_rate, min_period));
1918+
REFERENCE_TIME default_period_rt(
1919+
frames_to_hns(sample_rate, default_period));
1920+
LOG("default device period: %I64d, minimum device period: %I64d",
1921+
default_period_rt, min_period_rt);
1922+
1923+
*latency_frames = min_period;
1924+
1925+
LOG("Minimum latency in frames: %u", *latency_frames);
1926+
1927+
return CUBEB_OK;
1928+
}
1929+
}
1930+
}
1931+
#endif
1932+
18701933
com_ptr<IAudioClient> client;
18711934
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
18721935
client.receive_vpp());
@@ -1886,18 +1949,8 @@ wasapi_get_min_latency(cubeb * ctx, cubeb_stream_params params,
18861949
LOG("default device period: %I64d, minimum device period: %I64d",
18871950
default_period, minimum_period);
18881951

1889-
/* If we're on Windows 10, we can use IAudioClient3 to get minimal latency.
1890-
Otherwise, according to the docs, the best latency we can achieve is by
1891-
synchronizing the stream and the engine.
1892-
http://msdn.microsoft.com/en-us/library/windows/desktop/dd370871%28v=vs.85%29.aspx
1893-
*/
1894-
1895-
// #ifdef _WIN32_WINNT_WIN10
1896-
#if 0
1897-
*latency_frames = hns_to_frames(params.rate, minimum_period);
1898-
#else
1952+
// The minimum_period is only relevant in exclusive streams.
18991953
*latency_frames = hns_to_frames(params.rate, default_period);
1900-
#endif
19011954

19021955
LOG("Minimum latency in frames: %u", *latency_frames);
19031956

@@ -1987,7 +2040,10 @@ handle_channel_layout(cubeb_stream * stm, EDataFlow direction,
19872040
if (hr == S_FALSE) {
19882041
/* Channel layout not supported, but WASAPI gives us a suggestion. Use it,
19892042
and handle the eventual upmix/downmix ourselves. Ignore the subformat of
1990-
the suggestion, since it seems to always be IEEE_FLOAT. */
2043+
the suggestion, since it seems to always be IEEE_FLOAT.
2044+
This fallback doesn't update the bit depth, so if a device
2045+
only supported bit depths cubeb doesn't support, so IAudioClient3
2046+
streams might fail */
19912047
LOG("Using WASAPI suggested format: channels: %d", closest->nChannels);
19922048
XASSERT(closest->wFormatTag == WAVE_FORMAT_EXTENSIBLE);
19932049
WAVEFORMATEXTENSIBLE * closest_pcm =
@@ -2031,12 +2087,12 @@ initialize_iaudioclient2(com_ptr<IAudioClient> & audio_client)
20312087
return CUBEB_OK;
20322088
}
20332089

2034-
#if 0
20352090
bool
20362091
initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
20372092
cubeb_stream * stm,
20382093
const com_heap_ptr<WAVEFORMATEX> & mix_format,
2039-
DWORD flags, EDataFlow direction)
2094+
DWORD flags, EDataFlow direction,
2095+
REFERENCE_TIME latency_hns)
20402096
{
20412097
com_ptr<IAudioClient3> audio_client3;
20422098
audio_client->QueryInterface<IAudioClient3>(audio_client3.receive());
@@ -2052,24 +2108,22 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
20522108
return false;
20532109
}
20542110

2055-
// Some people have reported glitches with capture streams:
2056-
// http://blog.nirbheek.in/2018/03/low-latency-audio-on-windows-with.html
2057-
if (direction == eCapture) {
2058-
LOG("Audio stream is capture, not using IAudioClient3");
2059-
return false;
2060-
}
2061-
20622111
// Possibly initialize a shared-mode stream using IAudioClient3. Initializing
20632112
// a stream this way lets you request lower latencies, but also locks the
20642113
// global WASAPI engine at that latency.
20652114
// - If we request a shared-mode stream, streams created with IAudioClient
2066-
// will
2067-
// have their latency adjusted to match. When the shared-mode stream is
2068-
// closed, they'll go back to normal.
2069-
// - If there's already a shared-mode stream running, then we cannot request
2070-
// the engine change to a different latency - we have to match it.
2071-
// - It's antisocial to lock the WASAPI engine at its default latency. If we
2072-
// would do this, then stop and use IAudioClient instead.
2115+
// might have their latency adjusted to match. When the shared-mode stream
2116+
// is closed, they'll go back to normal.
2117+
// - If there's already a shared-mode stream running, if it created with the
2118+
// AUDCLNT_STREAMOPTIONS_MATCH_FORMAT option, the audio engine would be
2119+
// locked to that format, so we have to match it (a custom one would fail).
2120+
// - We don't lock the WASAPI engine to a format, as it's antisocial towards
2121+
// other apps, especially if we locked to a latency >= than its default.
2122+
// - If the user requested latency is >= the default one, we might still
2123+
// accept it (without locking the format) depending on
2124+
// ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT, as we might want to prioritize
2125+
// to lower our latency over other apps
2126+
// (there might still be latency advantages compared to IAudioDevice(1)).
20732127

20742128
HRESULT hr;
20752129
uint32_t default_period = 0, fundamental_period = 0, min_period = 0,
@@ -2081,28 +2135,59 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
20812135
LOG("Could not get shared mode engine period: error: %lx", hr);
20822136
return false;
20832137
}
2084-
uint32_t requested_latency = stm->latency;
2138+
uint32_t requested_latency =
2139+
hns_to_frames(mix_format->nSamplesPerSec, latency_hns);
2140+
#if !ALLOW_AUDIO_CLIENT_3_LATENCY_OVER_DEFAULT
20852141
if (requested_latency >= default_period) {
2086-
LOG("Requested latency %i greater than default latency %i, not using "
2087-
"IAudioClient3",
2142+
LOG("Requested latency %i equal or greater than default latency %i,"
2143+
" not using IAudioClient3",
20882144
requested_latency, default_period);
20892145
return false;
20902146
}
2147+
#elif REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX
2148+
if (requested_latency > max_period) {
2149+
// Fallback to IAudioClient(1) as it's more accepting of large latencies
2150+
LOG("Requested latency %i greater than max latency %i,"
2151+
" not using IAudioClient3",
2152+
requested_latency, max_period);
2153+
return false;
2154+
}
2155+
#endif
20912156
LOG("Got shared mode engine period: default=%i fundamental=%i min=%i max=%i",
20922157
default_period, fundamental_period, min_period, max_period);
20932158
// Snap requested latency to a valid value
20942159
uint32_t old_requested_latency = requested_latency;
2160+
// The period is required to be a multiple of the fundamental period
2161+
// (and >= min and <= max, which should still be true)
2162+
requested_latency -= requested_latency % fundamental_period;
20952163
if (requested_latency < min_period) {
20962164
requested_latency = min_period;
20972165
}
2098-
requested_latency -= (requested_latency - min_period) % fundamental_period;
2166+
// Likely unnecessary, but won't hurt
2167+
if (requested_latency > max_period) {
2168+
requested_latency = max_period;
2169+
}
20992170
if (requested_latency != old_requested_latency) {
21002171
LOG("Requested latency %i was adjusted to %i", old_requested_latency,
21012172
requested_latency);
21022173
}
21032174

2104-
hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
2175+
DWORD new_flags = flags;
2176+
// Always add these flags to IAudioClient3, they might help
2177+
// if the stream doesn't have the same format as the audio engine.
2178+
new_flags |= AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM;
2179+
new_flags |= AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY;
2180+
2181+
hr = audio_client3->InitializeSharedAudioStream(new_flags, requested_latency,
21052182
mix_format.get(), NULL);
2183+
// This error should be returned first even if
2184+
// the period was locked (AUDCLNT_E_ENGINE_PERIODICITY_LOCKED)
2185+
if (hr == AUDCLNT_E_INVALID_STREAM_FLAG) {
2186+
LOG("Got AUDCLNT_E_INVALID_STREAM_FLAG, removing some flags");
2187+
hr = audio_client3->InitializeSharedAudioStream(flags, requested_latency,
2188+
mix_format.get(), NULL);
2189+
}
2190+
21062191
if (SUCCEEDED(hr)) {
21072192
return true;
21082193
} else if (hr == AUDCLNT_E_ENGINE_PERIODICITY_LOCKED) {
@@ -2114,22 +2199,37 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
21142199
}
21152200

21162201
uint32_t current_period = 0;
2117-
WAVEFORMATEX * current_format = nullptr;
2202+
WAVEFORMATEX * current_format_ptr = nullptr;
21182203
// We have to pass a valid WAVEFORMATEX** and not nullptr, otherwise
21192204
// GetCurrentSharedModeEnginePeriod will return E_POINTER
2120-
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format,
2205+
hr = audio_client3->GetCurrentSharedModeEnginePeriod(&current_format_ptr,
21212206
&current_period);
2122-
CoTaskMemFree(current_format);
21232207
if (FAILED(hr)) {
21242208
LOG("Could not get current shared mode engine period: error: %lx", hr);
21252209
return false;
21262210
}
2211+
com_heap_ptr<WAVEFORMATEX> current_format(current_format_ptr);
2212+
if (current_format->nSamplesPerSec != mix_format->nSamplesPerSec) {
2213+
// Unless some other external app locked the shared mode engine period
2214+
// within our audio initialization, this is unlikely to happen, though we
2215+
// can't respect the user selected latency, so we fallback on IAudioClient
2216+
LOG("IAudioClient3::GetCurrentSharedModeEnginePeriod() returned a "
2217+
"different mixer format (nSamplesPerSec) from "
2218+
"IAudioClient::GetMixFormat(); not using IAudioClient3");
2219+
return false;
2220+
}
21272221

2128-
if (current_period >= default_period) {
2129-
LOG("Current shared mode engine period %i too high, not using IAudioClient",
2130-
current_period);
2222+
#if REJECT_AUDIO_CLIENT_3_LATENCY_OVER_MAX
2223+
// Reject IAudioClient3 if we can't respect the user target latency.
2224+
// We don't need to check against default_latency anymore,
2225+
// as the current_period is already the best one we could get.
2226+
if (old_requested_latency > current_period) {
2227+
LOG("Requested latency %i greater than currently locked shared mode "
2228+
"latency %i, not using IAudioClient3",
2229+
old_requested_latency, current_period);
21312230
return false;
21322231
}
2232+
#endif
21332233

21342234
hr = audio_client3->InitializeSharedAudioStream(flags, current_period,
21352235
mix_format.get(), NULL);
@@ -2142,7 +2242,6 @@ initialize_iaudioclient3(com_ptr<IAudioClient> & audio_client,
21422242
LOG("Could not initialize shared stream with IAudioClient3: error: %lx", hr);
21432243
return false;
21442244
}
2145-
#endif
21462245

21472246
#define DIRECTION_NAME (direction == eCapture ? "capture" : "render")
21482247

@@ -2166,6 +2265,12 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
21662265
return CUBEB_ERROR;
21672266
}
21682267

2268+
#if ALLOW_AUDIO_CLIENT_3_FOR_INPUT
2269+
constexpr bool allow_audio_client_3 = true;
2270+
#else
2271+
const bool allow_audio_client_3 = direction == eRender;
2272+
#endif
2273+
21692274
stm->stream_reset_lock.assert_current_thread_owns();
21702275
// If user doesn't specify a particular device, we can choose another one when
21712276
// the given devid is unavailable.
@@ -2202,17 +2307,14 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
22022307

22032308
/* Get a client. We will get all other interfaces we need from
22042309
* this pointer. */
2205-
#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
2206-
hr = device->Activate(__uuidof(IAudioClient3),
2207-
CLSCTX_INPROC_SERVER,
2208-
NULL, audio_client.receive_vpp());
2209-
if (hr == E_NOINTERFACE) {
2210-
#endif
2211-
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
2212-
audio_client.receive_vpp());
2213-
#if 0
2310+
if (allow_audio_client_3) {
2311+
hr = device->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER, NULL,
2312+
audio_client.receive_vpp());
2313+
}
2314+
if (!allow_audio_client_3 || hr == E_NOINTERFACE) {
2315+
hr = device->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER, NULL,
2316+
audio_client.receive_vpp());
22142317
}
2215-
#endif
22162318

22172319
if (FAILED(hr)) {
22182320
LOG("Could not activate the device to get an audio"
@@ -2341,16 +2443,15 @@ setup_wasapi_stream_one_side(cubeb_stream * stm,
23412443
}
23422444
}
23432445

2344-
#if 0 // See https://bugzilla.mozilla.org/show_bug.cgi?id=1590902
2345-
if (initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction)) {
2446+
if (allow_audio_client_3 &&
2447+
initialize_iaudioclient3(audio_client, stm, mix_format, flags, direction,
2448+
latency_hns)) {
23462449
LOG("Initialized with IAudioClient3");
23472450
} else {
2348-
#endif
2349-
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns, 0,
2350-
mix_format.get(), NULL);
2351-
#if 0
2451+
hr = audio_client->Initialize(AUDCLNT_SHAREMODE_SHARED, flags, latency_hns,
2452+
0, mix_format.get(), NULL);
23522453
}
2353-
#endif
2454+
23542455
if (FAILED(hr)) {
23552456
LOG("Unable to initialize audio client for %s: %lx.", DIRECTION_NAME, hr);
23562457
return CUBEB_ERROR;
@@ -3310,6 +3411,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
33103411
CUBEB_DEVICE_FMT_S16NE);
33113412
ret.default_format = CUBEB_DEVICE_FMT_F32NE;
33123413
prop_variant fmtvar;
3414+
WAVEFORMATEX* wfx = NULL;
33133415
hr = propstore->GetValue(PKEY_AudioEngine_DeviceFormat, &fmtvar);
33143416
if (SUCCEEDED(hr) && fmtvar.vt == VT_BLOB) {
33153417
if (fmtvar.blob.cbSize == sizeof(PCMWAVEFORMAT)) {
@@ -3319,8 +3421,7 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
33193421
ret.max_rate = ret.min_rate = ret.default_rate = pcm->wf.nSamplesPerSec;
33203422
ret.max_channels = pcm->wf.nChannels;
33213423
} else if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX)) {
3322-
WAVEFORMATEX * wfx =
3323-
reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
3424+
wfx = reinterpret_cast<WAVEFORMATEX *>(fmtvar.blob.pBlobData);
33243425

33253426
if (fmtvar.blob.cbSize >= sizeof(WAVEFORMATEX) + wfx->cbSize ||
33263427
wfx->wFormatTag == WAVE_FORMAT_PCM) {
@@ -3330,6 +3431,27 @@ wasapi_create_device(cubeb * ctx, cubeb_device_info & ret,
33303431
}
33313432
}
33323433

3434+
#if USE_AUDIO_CLIENT_3_MIN_PERIOD
3435+
// Here we assume an IAudioClient3 stream will successfully
3436+
// be initialized later (it might fail)
3437+
#if ALLOW_AUDIO_CLIENT_3_FOR_INPUT
3438+
constexpr bool allow_audio_client_3 = true;
3439+
#else
3440+
const bool allow_audio_client_3 = flow == eRender;
3441+
#endif
3442+
com_ptr<IAudioClient3> client3;
3443+
uint32_t def, fun, min, max;
3444+
if (allow_audio_client_3 && wfx &&
3445+
SUCCEEDED(dev->Activate(__uuidof(IAudioClient3), CLSCTX_INPROC_SERVER,
3446+
NULL, client3.receive_vpp())) &&
3447+
SUCCEEDED(
3448+
client3->GetSharedModeEnginePeriod(wfx, &def, &fun, &min, &max))) {
3449+
ret.latency_lo = min;
3450+
// This latency might actually be used as "default" and not "max" later on,
3451+
// so we return the default (we never really want to use the max anyway)
3452+
ret.latency_hi = def;
3453+
} else
3454+
#endif
33333455
if (SUCCEEDED(dev->Activate(__uuidof(IAudioClient), CLSCTX_INPROC_SERVER,
33343456
NULL, client.receive_vpp())) &&
33353457
SUCCEEDED(client->GetDevicePeriod(&def_period, &min_period))) {

0 commit comments

Comments
 (0)