Skip to content

Commit 1c03b55

Browse files
authored
feat(audio): runtime audio processing options and engine-wide state read-back (#1107)
## What Runtime control of audio processing (AEC / NS / AGC / HPF) for local audio tracks, plus an engine-wide diagnostic read-back, built on the WebRTC-SDK audio processing options API (webrtc-sdk/webrtc#247 + webrtc-sdk/webrtc#254). ## API **Set** — `AudioProcessingOptions` with per-component enabled flags and modes (`automatic` / `platform` / `software`), applied either at capture time via `AudioCaptureOptions` or at runtime: ```dart final result = await localAudioTrack.setAudioProcessingOptions(options); ``` Caller bugs (invalid combination, remote track) throw `AudioProcessingException`; legitimate outcomes return a typed `AudioProcessingApplyResult` (`applied` / `stored` / rejections). **Read** — the audio processing module is owned by the native peer connection factory and shared engine-wide, so the snapshot lives on `AudioManager`: ```dart final state = await AudioManager.instance.getAudioProcessingState(); ``` Per component: `requested` (nullable — null means nothing was ever applied), `isSoftwareResolved` / `isSoftwareActive`, `isPlatformAvailable` / `isPlatformResolved` / `isPlatformActive`, and `effective` as the merged verdict. Same requested → resolved → active → effective vocabulary as the native SDKs. ## Commits Bottom-up, each builds standalone: 1. `chore(deps)`: WebRTC-SDK pin bump 2. Dart `AudioProcessingOptions` for `LocalAudioTrack` 3. Routing through the LiveKit native plugin (iOS + Android handlers) 4. Typed apply results 5. Engine-wide v2 state read-back on `AudioManager` ## Dependencies / not yet done - **Lib pin is still `144.7559.08`** — will bump to `144.7559.09` (which carries the state v2 API from webrtc-sdk/webrtc#254) once published. The state read-back native code requires `.09` to compile. - **Android requires `FlutterWebRTCPlugin.getPeerConnectionFactory()`** — flutter-webrtc/flutter-webrtc#2077. - Device smoke test on iOS + Android pending the `.09` artifacts.
1 parent d031c3f commit 1c03b55

14 files changed

Lines changed: 704 additions & 15 deletions

File tree

.changes/runtime-audio-options

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
minor type="added" "Runtime audio processing options for local audio tracks and engine-wide audio processing state read-back on AudioManager"

android/build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ android {
5959
dependencies {
6060
testImplementation("org.jetbrains.kotlin:kotlin-test")
6161
testImplementation("org.mockito:mockito-core:5.0.0")
62-
implementation 'io.github.webrtc-sdk:android:144.7559.01'
62+
implementation 'io.github.webrtc-sdk:android:144.7559.09'
6363
implementation 'io.livekit:noise:2.0.0'
6464
}
6565

android/src/main/kotlin/io/livekit/plugin/LiveKitPlugin.kt

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,13 @@ import com.cloudwebrtc.webrtc.FlutterWebRTCPlugin
2929
import com.cloudwebrtc.webrtc.audio.LocalAudioTrack
3030
import io.flutter.plugin.common.BinaryMessenger
3131
import org.webrtc.AudioTrack
32+
import org.webrtc.audio.AudioProcessingComponentOptions
33+
import org.webrtc.audio.AudioProcessingComponentState
34+
import org.webrtc.audio.AudioProcessingImplementation
35+
import org.webrtc.audio.AudioProcessingMode
36+
import org.webrtc.audio.AudioProcessingOptions
37+
import org.webrtc.audio.AudioProcessingOptionsResult
38+
import org.webrtc.audio.AudioProcessingState
3239

3340
/** LiveKitPlugin */
3441
class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
@@ -210,6 +217,113 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
210217
result.success(true)
211218
}
212219

220+
private fun handleSetAudioProcessingOptions(call: MethodCall, result: Result) {
221+
val trackId = call.argument<String>("trackId")
222+
if (trackId == null) {
223+
result.error("INVALID_ARGUMENT", "trackId is required", null)
224+
return
225+
}
226+
227+
val mediaTrack = (flutterWebRTCPlugin.getLocalTrack(trackId) as? LocalAudioTrack)?.track
228+
if (mediaTrack !is AudioTrack) {
229+
result.error("INVALID_ARGUMENT", "track is not a local audio track", null)
230+
return
231+
}
232+
233+
val options = AudioProcessingOptions(
234+
AudioProcessingComponentOptions(
235+
call.argument<Boolean>("echoCancellation") ?: true,
236+
audioProcessingMode(call.argument<String>("echoCancellationMode")),
237+
),
238+
AudioProcessingComponentOptions(
239+
call.argument<Boolean>("noiseSuppression") ?: true,
240+
audioProcessingMode(call.argument<String>("noiseSuppressionMode")),
241+
),
242+
AudioProcessingComponentOptions(
243+
call.argument<Boolean>("autoGainControl") ?: true,
244+
audioProcessingMode(call.argument<String>("autoGainControlMode")),
245+
),
246+
AudioProcessingComponentOptions(
247+
call.argument<Boolean>("highPassFilter") ?: false,
248+
audioProcessingMode(call.argument<String>("highPassFilterMode")),
249+
),
250+
)
251+
252+
val processingResult = mediaTrack.setAudioProcessingOptions(options)
253+
result.success(
254+
mapOf(
255+
"result" to processingResult.isSuccess,
256+
"code" to audioProcessingResultCodeString(processingResult.code),
257+
"message" to processingResult.message,
258+
),
259+
)
260+
}
261+
262+
private fun audioProcessingMode(value: String?): AudioProcessingMode = when (value) {
263+
"platform" -> AudioProcessingMode.PLATFORM
264+
"software" -> AudioProcessingMode.SOFTWARE
265+
else -> AudioProcessingMode.AUTOMATIC
266+
}
267+
268+
private fun audioProcessingResultCodeString(code: AudioProcessingOptionsResult.Code): String = when (code) {
269+
AudioProcessingOptionsResult.Code.APPLIED -> "applied"
270+
AudioProcessingOptionsResult.Code.STORED -> "stored"
271+
AudioProcessingOptionsResult.Code.REJECTED_REMOTE_TRACK -> "rejectedRemoteTrack"
272+
AudioProcessingOptionsResult.Code.REJECTED_INVALID_COMBINATION -> "rejectedInvalidCombination"
273+
AudioProcessingOptionsResult.Code.REJECTED_PLATFORM_UNAVAILABLE -> "rejectedPlatformUnavailable"
274+
AudioProcessingOptionsResult.Code.APPLY_FAILED -> "applyFailed"
275+
}
276+
277+
private fun handleGetAudioProcessingState(result: Result) {
278+
val factory = flutterWebRTCPlugin.getPeerConnectionFactory()
279+
if (factory == null) {
280+
result.success(null)
281+
return
282+
}
283+
result.success(audioProcessingStateToMap(factory.audioProcessingState))
284+
}
285+
286+
private fun audioProcessingModeString(mode: AudioProcessingMode): String = when (mode) {
287+
AudioProcessingMode.PLATFORM -> "platform"
288+
AudioProcessingMode.SOFTWARE -> "software"
289+
AudioProcessingMode.AUTOMATIC -> "auto"
290+
}
291+
292+
private fun audioProcessingImplementationString(implementation: AudioProcessingImplementation): String =
293+
when (implementation) {
294+
AudioProcessingImplementation.UNKNOWN -> "unknown"
295+
AudioProcessingImplementation.DISABLED -> "disabled"
296+
AudioProcessingImplementation.SOFTWARE -> "software"
297+
AudioProcessingImplementation.PLATFORM -> "platform"
298+
AudioProcessingImplementation.SOFTWARE_AND_PLATFORM -> "softwareAndPlatform"
299+
}
300+
301+
private fun requestedToMap(requested: AudioProcessingComponentOptions?): Map<String, Any?>? =
302+
requested?.let {
303+
mapOf(
304+
"enabled" to it.isEnabled,
305+
"mode" to audioProcessingModeString(it.mode),
306+
)
307+
}
308+
309+
private fun componentToMap(state: AudioProcessingComponentState): Map<String, Any?> = mapOf(
310+
"requested" to requestedToMap(state.requested),
311+
"isSoftwareResolved" to state.isSoftwareResolved,
312+
"isSoftwareActive" to state.isSoftwareActive,
313+
"isPlatformAvailable" to state.isPlatformAvailable,
314+
"isPlatformResolved" to state.isPlatformResolved,
315+
"isPlatformActive" to state.isPlatformActive,
316+
"effective" to audioProcessingImplementationString(state.effective),
317+
)
318+
319+
private fun audioProcessingStateToMap(state: AudioProcessingState): Map<String, Any?> = mapOf(
320+
"hasAudioProcessingModule" to state.hasAudioProcessingModule,
321+
"echoCancellation" to componentToMap(state.echoCancellation),
322+
"noiseSuppression" to componentToMap(state.noiseSuppression),
323+
"autoGainControl" to componentToMap(state.autoGainControl),
324+
"highPassFilter" to componentToMap(state.highPassFilter),
325+
)
326+
213327
override fun onMethodCall(@NonNull call: MethodCall, @NonNull result: Result) {
214328
when (call.method) {
215329
"startVisualizer" -> {
@@ -228,6 +342,14 @@ class LiveKitPlugin : FlutterPlugin, MethodCallHandler {
228342
handleStopAudioRenderer(call, result)
229343
}
230344

345+
"setAudioProcessingOptions" -> {
346+
handleSetAudioProcessingOptions(call, result)
347+
}
348+
349+
"getAudioProcessingState" -> {
350+
handleGetAudioProcessingState(result)
351+
}
352+
231353
else -> {
232354
result.notImplemented()
233355
}

ios/livekit_client.podspec

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,6 @@ Pod::Spec.new do |s|
1616
s.static_framework = true
1717

1818
s.dependency 'Flutter'
19-
s.dependency 'WebRTC-SDK', '144.7559.01'
19+
s.dependency 'WebRTC-SDK', '144.7559.09'
2020
s.dependency 'flutter_webrtc'
2121
end

lib/livekit_client.dart

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,14 @@ export 'src/agent/room_agent.dart';
4141
export 'src/participant/local.dart';
4242
export 'src/participant/participant.dart';
4343
export 'src/participant/remote.dart' hide ParticipantCreationResult;
44+
export 'src/audio/audio_manager.dart';
4445
export 'src/audio/audio_frame_capture.dart' show AudioFormat, AudioFrame, AudioFrameCallback, AudioRendererOptions;
4546
export 'src/preconnect/pre_connect_audio_buffer.dart';
4647
export 'src/publication/local.dart';
4748
export 'src/publication/remote.dart';
4849
export 'src/publication/track_publication.dart';
4950
export 'src/support/platform.dart';
51+
export 'src/audio/audio_processing_state.dart';
5052
export 'src/track/audio_visualizer.dart';
5153
export 'src/track/local/audio.dart';
5254
export 'src/track/local/local.dart';

lib/src/audio/audio_manager.dart

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
// Copyright 2024 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import '../support/native.dart';
16+
import 'audio_processing_state.dart';
17+
18+
/// Controls LiveKit's process-wide platform audio behavior.
19+
///
20+
/// The platform audio engine and its audio processing module are global to the
21+
/// app process, so engine-scoped audio state lives here rather than on a `Room`
22+
/// or an individual track.
23+
class AudioManager {
24+
AudioManager._();
25+
26+
static final AudioManager instance = AudioManager._();
27+
28+
/// Diagnostic snapshot of the resolved audio processing state.
29+
///
30+
/// The audio processing module is owned by the native peer connection factory
31+
/// and shared engine-wide, so this reflects what is actually applied across
32+
/// the engine rather than any single track — use it to verify what a
33+
/// `LocalAudioTrack.setAudioProcessingOptions` request resolved to. Returns
34+
/// `null` when the native side cannot provide it.
35+
Future<AudioProcessingState?> getAudioProcessingState() async {
36+
final response = await Native.getAudioProcessingState();
37+
if (response == null) return null;
38+
return AudioProcessingState.fromMap(response);
39+
}
40+
}
Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
// Copyright 2026 LiveKit, Inc.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
import '../track/options.dart';
16+
17+
/// The implementation in effect for an audio processing component.
18+
enum AudioProcessingImplementation {
19+
unknown('unknown'),
20+
disabled('disabled'),
21+
software('software'),
22+
platform('platform'),
23+
softwareAndPlatform('softwareAndPlatform');
24+
25+
const AudioProcessingImplementation(this.value);
26+
27+
final String value;
28+
29+
static AudioProcessingImplementation fromValue(String? value) => AudioProcessingImplementation.values.firstWhere(
30+
(e) => e.value == value,
31+
orElse: () => AudioProcessingImplementation.unknown,
32+
);
33+
}
34+
35+
AudioProcessingMode _modeFromValue(String? value) {
36+
for (final mode in AudioProcessingMode.values) {
37+
if (mode.constraintValue == value) return mode;
38+
}
39+
return AudioProcessingMode.automatic;
40+
}
41+
42+
/// The caller's request for one audio processing component: enabled flag plus
43+
/// implementation mode.
44+
class AudioProcessingComponentRequest {
45+
const AudioProcessingComponentRequest({
46+
required this.enabled,
47+
required this.mode,
48+
});
49+
50+
factory AudioProcessingComponentRequest.fromMap(Map<dynamic, dynamic> map) => AudioProcessingComponentRequest(
51+
enabled: (map['enabled'] as bool?) ?? false,
52+
mode: _modeFromValue(map['mode'] as String?),
53+
);
54+
55+
final bool enabled;
56+
final AudioProcessingMode mode;
57+
}
58+
59+
/// Diagnostic state of one audio processing component (echo cancellation,
60+
/// noise suppression, auto gain control or high-pass filter), observed at
61+
/// three stages of one pipeline: requested (caller intent) -> resolved (the
62+
/// engine's per-path decision) -> active (live truth), with [effective] as
63+
/// the merged verdict.
64+
class AudioProcessingComponentState {
65+
const AudioProcessingComponentState({
66+
this.requested,
67+
required this.isSoftwareResolved,
68+
required this.isSoftwareActive,
69+
required this.isPlatformAvailable,
70+
required this.isPlatformResolved,
71+
required this.isPlatformActive,
72+
required this.effective,
73+
});
74+
75+
factory AudioProcessingComponentState.fromMap(Map<dynamic, dynamic> map) => AudioProcessingComponentState(
76+
requested: map['requested'] is Map
77+
? AudioProcessingComponentRequest.fromMap(Map<dynamic, dynamic>.from(map['requested'] as Map))
78+
: null,
79+
isSoftwareResolved: (map['isSoftwareResolved'] as bool?) ?? false,
80+
isSoftwareActive: (map['isSoftwareActive'] as bool?) ?? false,
81+
isPlatformAvailable: (map['isPlatformAvailable'] as bool?) ?? false,
82+
isPlatformResolved: (map['isPlatformResolved'] as bool?) ?? false,
83+
isPlatformActive: (map['isPlatformActive'] as bool?) ?? false,
84+
effective: AudioProcessingImplementation.fromValue(map['effective'] as String?),
85+
);
86+
87+
/// What the caller most recently requested for this component. Null when no
88+
/// audio processing options have ever been applied — "nobody asked".
89+
final AudioProcessingComponentRequest? requested;
90+
91+
/// Whether the resolver decided the WebRTC software (APM) implementation
92+
/// should run, after weighing the requested mode against platform
93+
/// availability, coupling, and policy.
94+
final bool isSoftwareResolved;
95+
96+
/// Whether APM's live configuration currently has this component enabled.
97+
final bool isSoftwareActive;
98+
99+
/// Whether this device/OS offers a built-in implementation at all.
100+
final bool isPlatformAvailable;
101+
102+
/// Whether the engine asked the OS to run the platform implementation. The
103+
/// OS owns the outcome: it can decline, defer, or couple components.
104+
final bool isPlatformResolved;
105+
106+
/// Whether the device reports the platform implementation actually running.
107+
final bool isPlatformActive;
108+
109+
/// The verdict: which implementation is in effect right now.
110+
final AudioProcessingImplementation effective;
111+
}
112+
113+
/// Diagnostic snapshot of the resolved audio processing state for the shared
114+
/// audio processing module.
115+
///
116+
/// The module is owned by the native peer connection factory and shared
117+
/// engine-wide, so this reflects what is actually applied (per-component
118+
/// [AudioProcessingComponentState.effective]) versus what was requested — for
119+
/// the whole engine, not a single track.
120+
class AudioProcessingState {
121+
const AudioProcessingState({
122+
required this.hasAudioProcessingModule,
123+
required this.echoCancellation,
124+
required this.noiseSuppression,
125+
required this.autoGainControl,
126+
required this.highPassFilter,
127+
});
128+
129+
factory AudioProcessingState.fromMap(Map<dynamic, dynamic> map) => AudioProcessingState(
130+
hasAudioProcessingModule: (map['hasAudioProcessingModule'] as bool?) ?? false,
131+
echoCancellation:
132+
AudioProcessingComponentState.fromMap(Map<dynamic, dynamic>.from(map['echoCancellation'] as Map)),
133+
noiseSuppression:
134+
AudioProcessingComponentState.fromMap(Map<dynamic, dynamic>.from(map['noiseSuppression'] as Map)),
135+
autoGainControl:
136+
AudioProcessingComponentState.fromMap(Map<dynamic, dynamic>.from(map['autoGainControl'] as Map)),
137+
highPassFilter: AudioProcessingComponentState.fromMap(Map<dynamic, dynamic>.from(map['highPassFilter'] as Map)),
138+
);
139+
140+
final bool hasAudioProcessingModule;
141+
final AudioProcessingComponentState echoCancellation;
142+
final AudioProcessingComponentState noiseSuppression;
143+
final AudioProcessingComponentState autoGainControl;
144+
final AudioProcessingComponentState highPassFilter;
145+
}

0 commit comments

Comments
 (0)