Skip to content

Commit 2950203

Browse files
MiloszFilimowskiMiloszFilimowski
authored
FCE-3503: Local audio extraction (#60)
* add local audio extraction support * fix ci * add PR improvements
1 parent e9eb286 commit 2950203

4 files changed

Lines changed: 476 additions & 158 deletions

File tree

‎.github/workflows/ios_ci.yml‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,15 @@ jobs:
1414
runs-on: macos-latest
1515
steps:
1616
- uses: actions/checkout@v4
17+
18+
# react-native 0.71.4's bundled Yoga (log.cpp) does not compile under
19+
# the runner's default Xcode 26.5; pin to 26.3 (also installed on the
20+
# macos-26 image) which builds the example cleanly.
21+
- name: Select Xcode 26.3
22+
run: |
23+
sudo xcode-select -s /Applications/Xcode_26.3.app
24+
xcodebuild -version
25+
1726
- uses: actions/setup-node@v4
1827
with:
1928
node-version-file: '.nvmrc'

‎android/src/main/java/com/oney/WebRTCModule/AudioExtractionController.java‎

Lines changed: 242 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,242 @@
1+
package com.oney.WebRTCModule;
2+
3+
import android.util.Log;
4+
5+
import com.facebook.react.bridge.Promise;
6+
import com.facebook.react.bridge.ReactApplicationContext;
7+
import com.facebook.react.bridge.ReadableMap;
8+
import com.facebook.react.turbomodule.core.CallInvokerHolderImpl;
9+
10+
import org.webrtc.AudioTrack;
11+
import org.webrtc.AudioTrackSink;
12+
import org.webrtc.MediaStreamTrack;
13+
import org.webrtc.audio.JavaAudioDeviceModule;
14+
15+
import java.nio.ByteBuffer;
16+
import java.util.HashMap;
17+
import java.util.Map;
18+
import java.util.Set;
19+
import java.util.concurrent.ConcurrentHashMap;
20+
21+
/**
22+
* Audio extraction: pulls int16 PCM from local mic and remote tracks and feeds
23+
* it to the native converter ({@link FJAudioSinkInstaller}) for delivery to JS.
24+
*/
25+
final class AudioExtractionController {
26+
private static final String TAG = "AudioExtractionController";
27+
28+
// pcId the JS side uses to mean "the local mic track", not a remote pc track.
29+
static final int LOCAL_TRACK_PC_ID = -1;
30+
31+
// miniaudio's MA_MAX_FILTER_ORDER (high-quality resampler lpfOrder).
32+
private static final int MA_MAX_FILTER_ORDER = 8;
33+
34+
interface TrackResolver {
35+
MediaStreamTrack getTrack(int pcId, String trackId);
36+
}
37+
38+
private final ReactApplicationContext reactContext;
39+
private final TrackResolver trackResolver;
40+
41+
private final Map<String, AudioTrackSink> audioSinks = new HashMap<>();
42+
43+
// Read on the ADM capture thread, written on the RN executor thread.
44+
private final Set<String> localAudioSinks = ConcurrentHashMap.newKeySet();
45+
46+
// Reused direct buffer for local samples; confined to the ADM capture thread
47+
// (the only caller of onLocalAudioSamplesReady), grown when a chunk is larger.
48+
private ByteBuffer localSamplesBuffer;
49+
50+
// Lazily built from the JS CallInvoker; null on the old architecture (no JSI).
51+
private FJAudioSinkInstaller audioSinkInstaller;
52+
private boolean audioSinkInstallerInitialized;
53+
54+
AudioExtractionController(ReactApplicationContext reactContext, TrackResolver trackResolver) {
55+
this.reactContext = reactContext;
56+
this.trackResolver = trackResolver;
57+
}
58+
59+
private synchronized FJAudioSinkInstaller getInstaller() {
60+
if (audioSinkInstallerInitialized) {
61+
return audioSinkInstaller;
62+
}
63+
audioSinkInstallerInitialized = true;
64+
try {
65+
if (reactContext.getJSCallInvokerHolder() instanceof CallInvokerHolderImpl) {
66+
audioSinkInstaller = new FJAudioSinkInstaller(reactContext);
67+
}
68+
} catch (Throwable t) {
69+
Log.w(TAG, "Audio extraction unavailable: failed to build the JSI installer", t);
70+
}
71+
return audioSinkInstaller;
72+
}
73+
74+
void installAudioSink(Promise promise) {
75+
FJAudioSinkInstaller installer = getInstaller();
76+
if (installer == null) {
77+
promise.reject("E_NO_JSI", "Audio extraction requires the New Architecture.");
78+
return;
79+
}
80+
installer.install(promise);
81+
}
82+
83+
void startExtraction(int pcId, String id, ReadableMap options) {
84+
ThreadUtils.runOnExecutor(() -> {
85+
if (audioSinks.containsKey(id) || localAudioSinks.contains(id)) {
86+
return;
87+
}
88+
// Null without a JSI CallInvoker; JS already rejected at install time.
89+
FJAudioSinkInstaller installer = getInstaller();
90+
if (installer == null) {
91+
return;
92+
}
93+
94+
AudioSinkConfig config = AudioSinkConfig.fromOptions(options);
95+
if (pcId == LOCAL_TRACK_PC_ID) {
96+
startLocalExtraction(id, installer, config);
97+
} else {
98+
startRemoteExtraction(pcId, id, installer, config);
99+
}
100+
});
101+
}
102+
103+
void stopExtraction(int pcId, String id) {
104+
ThreadUtils.runOnExecutor(() -> {
105+
if (pcId == LOCAL_TRACK_PC_ID) {
106+
stopLocalExtraction(id);
107+
} else {
108+
stopRemoteExtraction(pcId, id);
109+
}
110+
});
111+
}
112+
113+
// Local: frames arrive via onLocalAudioSamplesReady, so there is no sink to attach.
114+
private void startLocalExtraction(String id, FJAudioSinkInstaller installer, AudioSinkConfig config) {
115+
config.applyTo(installer, LOCAL_TRACK_PC_ID, id);
116+
localAudioSinks.add(id);
117+
}
118+
119+
private void stopLocalExtraction(String id) {
120+
if (!localAudioSinks.remove(id)) {
121+
return;
122+
}
123+
FJAudioSinkInstaller installer = getInstaller();
124+
if (installer != null) {
125+
installer.removeTrack(id);
126+
}
127+
}
128+
129+
// Remote: frames arrive through an AudioTrackSink attached to the track.
130+
private void startRemoteExtraction(int pcId, String id, FJAudioSinkInstaller installer, AudioSinkConfig config) {
131+
MediaStreamTrack track = trackResolver.getTrack(pcId, id);
132+
if (!(track instanceof AudioTrack)) {
133+
Log.d(TAG, "startAudioExtraction() no audio track for " + id);
134+
return;
135+
}
136+
config.applyTo(installer, pcId, id);
137+
AudioTrackSink sink = new PcmBatchingSink(id, installer);
138+
((AudioTrack) track).addSink(sink);
139+
audioSinks.put(id, sink);
140+
}
141+
142+
private void stopRemoteExtraction(int pcId, String id) {
143+
AudioTrackSink sink = audioSinks.remove(id);
144+
if (sink == null) {
145+
return;
146+
}
147+
MediaStreamTrack track = trackResolver.getTrack(pcId, id);
148+
if (track instanceof AudioTrack) {
149+
((AudioTrack) track).removeSink(sink);
150+
}
151+
FJAudioSinkInstaller installer = getInstaller();
152+
if (installer != null) {
153+
installer.removeTrack(id);
154+
}
155+
}
156+
157+
// ADM samples-ready callback, on the capture thread. Cheap when no local sinks.
158+
void onLocalAudioSamplesReady(JavaAudioDeviceModule.AudioSamples samples) {
159+
if (localAudioSinks.isEmpty()) {
160+
return;
161+
}
162+
FJAudioSinkInstaller installer = getInstaller();
163+
if (installer == null) {
164+
return;
165+
}
166+
byte[] data = samples.getData();
167+
int frames = data.length / (samples.getChannelCount() * 2); // int16 = 2 bytes/sample
168+
ByteBuffer buf = localSamplesBuffer;
169+
if (buf == null || buf.capacity() < data.length) {
170+
buf = ByteBuffer.allocateDirect(data.length);
171+
localSamplesBuffer = buf;
172+
}
173+
buf.clear();
174+
buf.put(data);
175+
buf.flip(); // limit = data.length, so reusing a larger buffer reads only this chunk
176+
for (String trackId : localAudioSinks) {
177+
buf.rewind();
178+
installer.onAudioData(trackId, buf, samples.getSampleRate(), samples.getChannelCount(), frames);
179+
}
180+
}
181+
182+
// Output config for one request, parsed from the JS options (defaults match iOS).
183+
private static class AudioSinkConfig {
184+
final int outRate;
185+
final int outChannels;
186+
final boolean formatF32;
187+
final int lpfOrder;
188+
final double batchMs;
189+
190+
private AudioSinkConfig(int outRate, int outChannels, boolean formatF32, int lpfOrder, double batchMs) {
191+
this.outRate = outRate;
192+
this.outChannels = outChannels;
193+
this.formatF32 = formatF32;
194+
this.lpfOrder = lpfOrder;
195+
this.batchMs = batchMs;
196+
}
197+
198+
static AudioSinkConfig fromOptions(ReadableMap options) {
199+
int outRate = options != null && options.hasKey("sampleRate") ? options.getInt("sampleRate") : 16000;
200+
int outChannels = options != null && options.hasKey("channels") ? options.getInt("channels") : 1;
201+
if (outChannels < 1) {
202+
outChannels = 1;
203+
}
204+
boolean formatF32 =
205+
!(options != null && options.hasKey("format") && "s16".equals(options.getString("format")));
206+
int lpfOrder = options != null && options.hasKey("resampleQuality")
207+
&& "high".equals(options.getString("resampleQuality"))
208+
? MA_MAX_FILTER_ORDER
209+
: 1;
210+
double batchMs =
211+
options != null && options.hasKey("batchDurationMs") ? options.getDouble("batchDurationMs") : 100.0;
212+
if (batchMs <= 0) {
213+
batchMs = 100.0;
214+
}
215+
return new AudioSinkConfig(outRate, outChannels, formatF32, lpfOrder, batchMs);
216+
}
217+
218+
void applyTo(FJAudioSinkInstaller installer, int pcId, String trackId) {
219+
installer.configureTrack(pcId, trackId, outRate, outChannels, formatF32, lpfOrder, batchMs);
220+
}
221+
}
222+
223+
// Thin forwarder: hands each int16 chunk to the native converter.
224+
private static class PcmBatchingSink implements AudioTrackSink {
225+
private final String trackId;
226+
private final FJAudioSinkInstaller installer;
227+
228+
PcmBatchingSink(String trackId, FJAudioSinkInstaller installer) {
229+
this.trackId = trackId;
230+
this.installer = installer;
231+
}
232+
233+
@Override
234+
public void onData(ByteBuffer audioData, int bitsPerSample, int sampleRate, int numberOfChannels,
235+
int numberOfFrames, long absoluteCaptureTimestampMs) {
236+
if (bitsPerSample != 16) {
237+
return;
238+
}
239+
installer.onAudioData(trackId, audioData, sampleRate, numberOfChannels, numberOfFrames);
240+
}
241+
}
242+
}

0 commit comments

Comments
 (0)