Skip to content

Commit e9eb286

Browse files
FCE-3458: Extract Audio to JS from WebRTC (Part 2 — audio extraction) (#58)
* FCE-3458: Extract Audio to JS from WebRTC Adds startAudioExtraction / stopAudioExtraction API that taps remote WebRTC audio tracks, resamples/reformats PCM via miniaudio (Part 1), and delivers batches to JS via a JSI global on both iOS and Android. Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * FCE-3458: Fix formatting Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com> * improve startAudioExtraction API * make api more resilient * fix formatting * improve clarity * lint fix --------- Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent f75b126 commit e9eb286

16 files changed

Lines changed: 1301 additions & 5 deletions

.gitignore

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,5 @@ examples/GumTestApp_macOS/package-lock.json
1414
*.zip
1515
lib/
1616
src/*.js
17-
17+
.cxx
18+
.gradle

FishjamReactNativeWebrtc.podspec

Lines changed: 19 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,12 +14,28 @@ Pod::Spec.new do |s|
1414

1515
s.platforms = { :ios => '13.4', :osx => '10.13', :tvos => '16.0' }
1616

17-
s.preserve_paths = 'ios/**/*'
18-
s.source_files = 'ios/**/*.{h,m,swift}'
17+
s.preserve_paths = 'ios/**/*', 'common/cpp/**/*'
18+
s.source_files = 'ios/**/*.{h,m,mm,c,cpp,hpp,swift}', 'common/cpp/**/*.{h,c,cpp}'
1919
s.public_header_files = 'ios/**/*.h'
20+
# Keep shared C++ headers (miniaudio + JSI core) out of the public umbrella.
21+
s.private_header_files = 'common/cpp/**/*.h'
2022
s.swift_version = '5.0'
2123
s.libraries = 'c', 'sqlite3', 'stdc++'
2224
s.framework = 'AudioToolbox','AVFoundation', 'CoreAudio', 'CoreGraphics', 'CoreVideo', 'GLKit', 'VideoToolbox'
2325
s.dependency 'React-Core'
24-
s.dependency 'FishjamWebRTC', '~> 124.0.2.0'
26+
s.dependency 'FishjamWebRTC', '~> 124.0.2.2'
27+
# JSI audio-sink channel deps. React-jsi is universal; React-callinvoker is
28+
# present in new-arch-capable RN builds (guarded by FJ_HAS_CALL_INVOKER).
29+
s.dependency 'React-jsi'
30+
s.dependency 'React-callinvoker'
31+
32+
# miniaudio conversion-only build. MA_NO_* must be global across all TUs
33+
# (miniaudio is not ABI-compatible across differing configs).
34+
s.pod_target_xcconfig = {
35+
'CLANG_CXX_LANGUAGE_STANDARD' => 'c++20',
36+
'HEADER_SEARCH_PATHS' => '"$(PODS_TARGET_SRCROOT)/common/cpp/vendor" "$(PODS_TARGET_SRCROOT)/common/cpp/fishjam-audio"',
37+
'GCC_PREPROCESSOR_DEFINITIONS' =>
38+
'MA_NO_DEVICE_IO=1 MA_NO_DECODING=1 MA_NO_ENCODING=1 MA_NO_GENERATION=1 ' \
39+
'MA_NO_RESOURCE_MANAGER=1 MA_NO_NODE_GRAPH=1 $(inherited)',
40+
}
2541
end

android/CMakeLists.txt

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
cmake_minimum_required(VERSION 3.22.1)
2+
3+
project(fishjam_webrtc)
4+
5+
add_subdirectory(src/main/cpp)

android/build.gradle

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,13 +12,36 @@ android {
1212

1313
compileSdkVersion safeExtGet('compileSdkVersion', 24)
1414
buildToolsVersion safeExtGet('buildToolsVersion', "23.0.1")
15+
ndkVersion safeExtGet('ndkVersion', '27.1.12297006')
16+
17+
buildFeatures {
18+
prefab true
19+
}
1520

1621
defaultConfig {
1722
minSdkVersion 24
1823
targetSdkVersion safeExtGet('targetSdkVersion', 24)
1924
versionCode 1
2025
versionName "1.0"
2126
consumerProguardFiles 'consumer-rules.pro'
27+
28+
externalNativeBuild {
29+
cmake {
30+
cppFlags "-std=c++20"
31+
arguments "-DANDROID_STL=c++_shared"
32+
}
33+
}
34+
35+
ndk {
36+
abiFilters "armeabi-v7a", "arm64-v8a", "x86", "x86_64"
37+
}
38+
}
39+
40+
externalNativeBuild {
41+
cmake {
42+
path "CMakeLists.txt"
43+
version safeExtGet('cmakeVersion', '3.22.1')
44+
}
2245
}
2346

2447
// WebRTC requires Java 8 features
@@ -31,6 +54,7 @@ android {
3154

3255
dependencies {
3356
implementation "com.facebook.react:react-android:+"
34-
api 'org.jitsi:webrtc:124.+'
57+
api 'com.github.fishjam-cloud:webrtc:v124.0.2.2'
3558
implementation "androidx.core:core:1.7.0"
59+
implementation 'com.facebook.fbjni:fbjni:0.6.0'
3660
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
cmake_minimum_required(VERSION 3.22.1)
2+
3+
# Shared C++ core lives at <module-root>/common/cpp. From this file
4+
# (android/src/main/cpp) that is four directories up: cpp -> main -> src ->
5+
# android -> <react-native-webrtc root>.
6+
set(FJ_COMMON_CPP_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../common/cpp")
7+
get_filename_component(FJ_COMMON_CPP_DIR "${FJ_COMMON_CPP_DIR}" ABSOLUTE)
8+
9+
add_library(fishjam-webrtc-audio
10+
SHARED
11+
# Vendored miniaudio (conversion-only).
12+
"${FJ_COMMON_CPP_DIR}/vendor/miniaudio.c"
13+
# Shared JSI core (CallInvoker-driven global install + deliver).
14+
"${FJ_COMMON_CPP_DIR}/fishjam-audio/FJAudioSinkJSI.cpp"
15+
# fbjni HybridClass that installs the JS global over the CallInvoker.
16+
"${CMAKE_CURRENT_SOURCE_DIR}/FJAudioSinkInstaller.cpp"
17+
)
18+
19+
# miniaudio must be built conversion-only. These MA_NO_* defines must be
20+
# identical across every translation unit that includes miniaudio.h (miniaudio
21+
# is not ABI-compatible across differing configs) — see the iOS podspec, which
22+
# sets them globally. We therefore set them target-wide rather than per-source
23+
# so the Phase-3 JSI core inherits the same config automatically.
24+
target_compile_definitions(fishjam-webrtc-audio
25+
PRIVATE
26+
MA_NO_DEVICE_IO=1
27+
MA_NO_DECODING=1
28+
MA_NO_ENCODING=1
29+
MA_NO_GENERATION=1
30+
MA_NO_RESOURCE_MANAGER=1
31+
MA_NO_NODE_GRAPH=1
32+
)
33+
34+
set_target_properties(fishjam-webrtc-audio PROPERTIES
35+
CXX_STANDARD 20
36+
CXX_STANDARD_REQUIRED ON
37+
)
38+
39+
target_include_directories(fishjam-webrtc-audio
40+
PRIVATE
41+
"${FJ_COMMON_CPP_DIR}/vendor"
42+
"${FJ_COMMON_CPP_DIR}/fishjam-audio"
43+
)
44+
45+
# RN 0.81 prefab packages.
46+
find_package(ReactAndroid REQUIRED CONFIG)
47+
find_package(fbjni REQUIRED CONFIG)
48+
49+
target_link_libraries(fishjam-webrtc-audio
50+
ReactAndroid::jsi
51+
ReactAndroid::reactnative
52+
fbjni::fbjni
53+
android
54+
log
55+
)
Lines changed: 214 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,214 @@
1+
#include "FJAudioSinkInstaller.h"
2+
3+
namespace jni = facebook::jni;
4+
5+
namespace fishjam {
6+
7+
FJAudioSinkInstaller::FJAudioSinkInstaller(jni::alias_ref<jhybridobject> javaThis,
8+
std::shared_ptr<FJAudioSink> sink)
9+
: javaPart_(jni::make_global(javaThis)), sink_(std::move(sink)) {}
10+
11+
jni::local_ref<FJAudioSinkInstaller::jhybriddata> FJAudioSinkInstaller::initHybrid(
12+
jni::alias_ref<jhybridobject> javaThis,
13+
jni::alias_ref<facebook::react::CallInvokerHolder::javaobject> callInvokerHolder) {
14+
// FJAudioSink only needs the JS CallInvoker; it acquires the jsi::Runtime
15+
// itself inside invokeAsync, so no runtime pointer is required here.
16+
auto callInvoker = callInvokerHolder->cthis()->getCallInvoker();
17+
return makeCxxInstance(javaThis, std::make_shared<FJAudioSink>(callInvoker));
18+
}
19+
20+
void FJAudioSinkInstaller::installSink() {
21+
// FJAudioSink::install sets the global on the JS thread and then runs this
22+
// callback (also on the JS thread). We notify the Java peer there so the
23+
// Promise resolves strictly after the global exists. Capturing the
24+
// global_ref keeps the Java peer alive until the callback runs.
25+
auto javaPart = javaPart_;
26+
sink_->install([javaPart] {
27+
static const auto onSinkInstalled = javaPart->getClass()->getMethod<void()>("onSinkInstalled");
28+
onSinkInstalled(javaPart);
29+
});
30+
}
31+
32+
bool FJAudioSinkInstaller::isInstalled() {
33+
return sink_->isInstalled();
34+
}
35+
36+
void FJAudioSinkInstaller::configureTrack(jint pcId, jni::alias_ref<jstring> trackId, jint outRate,
37+
jint outChannels, jboolean formatF32, jint lpfOrder,
38+
jdouble batchMs) {
39+
std::lock_guard<std::mutex> lock(tracksMutex_);
40+
// Replace any existing entry's config; tear down its converter first so a
41+
// re-start with different options doesn't leak the old miniaudio state.
42+
TrackConverter &state = tracks_[trackId->toStdString()];
43+
if (state.ready) {
44+
ma_data_converter_uninit(&state.converter, nullptr);
45+
state.ready = false;
46+
state.inRate = 0;
47+
state.inChannels = 0;
48+
}
49+
state.pcId = pcId;
50+
state.requestedOutRate = outRate;
51+
state.outRate = outRate;
52+
state.outChannels = outChannels;
53+
state.outFormat = formatF32 ? ma_format_f32 : ma_format_s16;
54+
state.lpfOrder = lpfOrder;
55+
state.batchMs = batchMs;
56+
state.inputBuffer.clear();
57+
}
58+
59+
// Mirrors iOS -ensureConverterForRate:channels:. Caller holds tracksMutex_.
60+
void FJAudioSinkInstaller::ensureConverter(TrackConverter &state, int sampleRate, int channels) {
61+
if (state.ready && sampleRate == state.inRate && channels == state.inChannels) {
62+
return;
63+
}
64+
if (state.ready) {
65+
ma_data_converter_uninit(&state.converter, nullptr);
66+
state.ready = false;
67+
}
68+
state.inRate = sampleRate;
69+
state.inChannels = channels;
70+
int outRate = state.requestedOutRate > 0 ? state.requestedOutRate : sampleRate; // outRate 0 => keep input rate
71+
72+
ma_data_converter_config config = ma_data_converter_config_init(
73+
ma_format_s16, state.outFormat, static_cast<ma_uint32>(channels),
74+
static_cast<ma_uint32>(state.outChannels), static_cast<ma_uint32>(sampleRate),
75+
static_cast<ma_uint32>(outRate));
76+
config.resampling.algorithm = ma_resample_algorithm_linear;
77+
config.resampling.linear.lpfOrder = static_cast<ma_uint32>(state.lpfOrder);
78+
79+
state.ready = (ma_data_converter_init(&config, nullptr, &state.converter) == MA_SUCCESS);
80+
state.outRate = outRate;
81+
}
82+
83+
// Mirrors iOS -flush. Caller holds tracksMutex_.
84+
std::optional<FJAudioSinkInstaller::DeliverPayload> FJAudioSinkInstaller::flush(
85+
const std::string &trackId, TrackConverter &state) {
86+
if (!sink_->isInstalled() || !state.ready) {
87+
state.inputBuffer.clear();
88+
return std::nullopt;
89+
}
90+
91+
size_t outBytesPerSample =
92+
(state.outFormat == ma_format_f32) ? sizeof(float) : sizeof(int16_t);
93+
const uint8_t *readPtr = state.inputBuffer.data();
94+
ma_uint64 framesRemaining = state.inputBuffer.size() / (state.inChannels * sizeof(int16_t));
95+
96+
// One process call usually drains everything (the output is sized from the
97+
// expected frame count); loop defensively in case it doesn't.
98+
std::vector<uint8_t> output;
99+
while (framesRemaining > 0) {
100+
ma_uint64 expectedFrames = 0;
101+
ma_data_converter_get_expected_output_frame_count(&state.converter, framesRemaining,
102+
&expectedFrames);
103+
if (expectedFrames == 0) {
104+
break;
105+
}
106+
size_t writeOffset = output.size();
107+
output.resize(writeOffset +
108+
static_cast<size_t>(expectedFrames * state.outChannels * outBytesPerSample));
109+
110+
ma_uint64 framesIn = framesRemaining;
111+
ma_uint64 framesOut = expectedFrames;
112+
if (ma_data_converter_process_pcm_frames(&state.converter, readPtr, &framesIn,
113+
output.data() + writeOffset,
114+
&framesOut) != MA_SUCCESS) {
115+
break;
116+
}
117+
output.resize(writeOffset +
118+
static_cast<size_t>(framesOut * state.outChannels * outBytesPerSample));
119+
120+
if (framesIn == 0) {
121+
break; // made no progress; avoid spinning
122+
}
123+
readPtr += framesIn * state.inChannels * sizeof(int16_t);
124+
framesRemaining -= framesIn;
125+
}
126+
127+
state.inputBuffer.clear();
128+
if (output.empty()) {
129+
return std::nullopt;
130+
}
131+
return DeliverPayload{
132+
state.pcId,
133+
trackId,
134+
state.outRate,
135+
state.outChannels,
136+
state.outFormat == ma_format_f32 ? "f32" : "s16",
137+
std::move(output),
138+
};
139+
}
140+
141+
void FJAudioSinkInstaller::onAudioData(jni::alias_ref<jstring> trackId,
142+
jni::alias_ref<jni::JByteBuffer> audioData, jint sampleRate,
143+
jint channels, jint frames) {
144+
// Wrapped in try/catch so a converter / JSI error drops the batch (mirroring
145+
// iOS dropping on JSError) rather than crossing back into the JVM.
146+
try {
147+
if (channels <= 0 || frames <= 0 || audioData == nullptr) {
148+
return;
149+
}
150+
std::optional<DeliverPayload> payload;
151+
{
152+
std::lock_guard<std::mutex> lock(tracksMutex_);
153+
auto it = tracks_.find(trackId->toStdString());
154+
if (it == tracks_.end()) {
155+
return;
156+
}
157+
TrackConverter &state = it->second;
158+
159+
ensureConverter(state, sampleRate, channels);
160+
161+
// The direct ByteBuffer is backed by native memory that is only valid for
162+
// the duration of this call, so the int16 bytes are copied into inputBuffer.
163+
const uint8_t *src = static_cast<const uint8_t *>(audioData->getDirectAddress());
164+
size_t available = static_cast<size_t>(audioData->getDirectSize());
165+
size_t wanted = static_cast<size_t>(frames) * channels * sizeof(int16_t);
166+
size_t copyLen = wanted < available ? wanted : available;
167+
if (src == nullptr || copyLen == 0) {
168+
return;
169+
}
170+
state.inputBuffer.insert(state.inputBuffer.end(), src, src + copyLen);
171+
172+
size_t bytesPerBatch = static_cast<size_t>(state.inRate * state.batchMs / 1000.0) *
173+
state.inChannels * sizeof(int16_t);
174+
if (bytesPerBatch > 0 && state.inputBuffer.size() >= bytesPerBatch) {
175+
payload = flush(it->first, state);
176+
}
177+
} // tracksMutex_ released before delivery
178+
if (payload) {
179+
sink_->deliver(payload->pcId, payload->trackId, payload->outRate, payload->outChannels,
180+
payload->format.c_str(), std::move(payload->output));
181+
}
182+
} catch (...) {
183+
// Drop this batch.
184+
}
185+
}
186+
187+
void FJAudioSinkInstaller::removeTrack(jni::alias_ref<jstring> trackId) {
188+
std::lock_guard<std::mutex> lock(tracksMutex_);
189+
auto it = tracks_.find(trackId->toStdString());
190+
if (it == tracks_.end()) {
191+
return;
192+
}
193+
if (it->second.ready) {
194+
ma_data_converter_uninit(&it->second.converter, nullptr);
195+
}
196+
tracks_.erase(it);
197+
}
198+
199+
void FJAudioSinkInstaller::registerNatives() {
200+
registerHybrid({
201+
makeNativeMethod("initHybrid", FJAudioSinkInstaller::initHybrid),
202+
makeNativeMethod("installSink", FJAudioSinkInstaller::installSink),
203+
makeNativeMethod("isInstalled", FJAudioSinkInstaller::isInstalled),
204+
makeNativeMethod("configureTrack", FJAudioSinkInstaller::configureTrack),
205+
makeNativeMethod("onAudioData", FJAudioSinkInstaller::onAudioData),
206+
makeNativeMethod("removeTrack", FJAudioSinkInstaller::removeTrack),
207+
});
208+
}
209+
210+
} // namespace fishjam
211+
212+
JNIEXPORT jint JNICALL JNI_OnLoad(JavaVM *vm, void *) {
213+
return facebook::jni::initialize(vm, [] { fishjam::FJAudioSinkInstaller::registerNatives(); });
214+
}

0 commit comments

Comments
 (0)