Skip to content

Commit f180e66

Browse files
davidliudavidliu
committed
feat: preconnect audio buffer
1 parent 2143e4f commit f180e66

13 files changed

Lines changed: 396 additions & 17 deletions

File tree

‎android/src/main/java/com/livekit/reactnative/LivekitReactNativeModule.kt‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,14 +8,19 @@ import com.facebook.react.bridge.ReactApplicationContext
88
import com.facebook.react.bridge.ReactContextBaseJavaModule
99
import com.facebook.react.bridge.ReactMethod
1010
import com.facebook.react.bridge.ReadableMap
11+
import com.facebook.react.modules.core.DeviceEventManagerModule
1112
import com.livekit.reactnative.audio.AudioDeviceKind
1213
import com.livekit.reactnative.audio.AudioManagerUtils
1314
import com.livekit.reactnative.audio.AudioSwitchManager
15+
import com.livekit.reactnative.audio.events.Events
1416
import com.livekit.reactnative.audio.processing.AudioSinkManager
17+
import com.livekit.reactnative.audio.processing.AudioSinkProcessor
1518
import com.livekit.reactnative.audio.processing.MultibandVolumeProcessor
1619
import com.livekit.reactnative.audio.processing.VolumeProcessor
1720
import com.oney.WebRTCModule.WebRTCModuleOptions
1821
import org.webrtc.audio.WebRtcAudioTrackHelper
22+
import java.lang.Thread.sleep
23+
import kotlin.concurrent.thread
1924
import kotlin.time.Duration.Companion.milliseconds
2025

2126

@@ -131,6 +136,22 @@ class LivekitReactNativeModule(reactContext: ReactApplicationContext) : ReactCon
131136
promise.resolve(null)
132137
}
133138

139+
@ReactMethod(isBlockingSynchronousMethod = true)
140+
fun createAudioSinkListener(pcId: Int, trackId: String): String {
141+
val processor = AudioSinkProcessor(reactApplicationContext)
142+
val reactTag = audioSinkManager.registerSink(processor)
143+
audioSinkManager.attachSinkToTrack(processor, pcId, trackId)
144+
processor.reactTag = reactTag
145+
146+
return reactTag
147+
}
148+
149+
@ReactMethod(isBlockingSynchronousMethod = true)
150+
fun deleteAudioSinkListener(reactTag: String, pcId: Int, trackId: String) {
151+
audioSinkManager.detachSinkFromTrack(reactTag, pcId, trackId)
152+
audioSinkManager.unregisterSink(reactTag)
153+
}
154+
134155
@ReactMethod(isBlockingSynchronousMethod = true)
135156
fun createVolumeProcessor(pcId: Int, trackId: String): String {
136157
val processor = VolumeProcessor(reactApplicationContext)

‎android/src/main/java/com/livekit/reactnative/audio/events/Events.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,4 +3,5 @@ package com.livekit.reactnative.audio.events
33
enum class Events {
44
LK_VOLUME_PROCESSED,
55
LK_MULTIBAND_PROCESSED,
6+
LK_AUDIO_DATA,
67
}

‎android/src/main/java/com/livekit/reactnative/audio/processing/AudioSinkProcessor.kt‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.livekit.reactnative.audio.processing
2+
3+
import com.facebook.react.bridge.Arguments
4+
import com.facebook.react.bridge.ReactContext
5+
import com.facebook.react.modules.core.DeviceEventManagerModule
6+
import com.livekit.reactnative.audio.events.Events
7+
import org.webrtc.AudioTrackSink
8+
import java.nio.ByteBuffer
9+
import java.util.Arrays
10+
import kotlin.io.encoding.Base64
11+
import kotlin.io.encoding.ExperimentalEncodingApi
12+
13+
class AudioSinkProcessor(private val reactContext: ReactContext) : BaseAudioSinkProcessor() {
14+
var reactTag: String? = null
15+
16+
@OptIn(ExperimentalEncodingApi::class)
17+
override fun onAudioData(byteArray: ByteArray) {
18+
val reactTag = this.reactTag ?: return
19+
20+
val encodedString = Base64.encode(byteArray)
21+
val event = Arguments.createMap().apply {
22+
putString("data", encodedString)
23+
putString("id", reactTag)
24+
}
25+
reactContext.getJSModule(DeviceEventManagerModule.RCTDeviceEventEmitter::class.java)
26+
.emit(Events.LK_AUDIO_DATA.name, event)
27+
}
28+
}
29+
30+
abstract class BaseAudioSinkProcessor : AudioTrackSink {
31+
abstract fun onAudioData(byteArray: ByteArray)
32+
33+
override fun onData(
34+
audioData: ByteBuffer,
35+
bitsPerSample: Int,
36+
sampleRate: Int,
37+
numberOfChannels: Int,
38+
numberOfFrames: Int,
39+
absoluteCaptureTimestampMs: Long
40+
) {
41+
val byteArray: ByteArray
42+
43+
if (audioData.hasArray()) {
44+
val audioArray = audioData.array()
45+
byteArray = Arrays.copyOfRange(audioArray, audioData.arrayOffset(), audioArray.size)
46+
} else {
47+
audioData.mark()
48+
audioData.position(0)
49+
50+
byteArray = ByteArray(audioData.remaining())
51+
audioData.get(byteArray)
52+
audioData.reset()
53+
}
54+
55+
onAudioData(byteArray)
56+
}
57+
}

‎ci/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@
1313
"dependencies": {
1414
"@livekit/react-native": "*",
1515
"@livekit/react-native-webrtc": "^125.0.10",
16-
"livekit-client": "^2.9.8",
16+
"livekit-client": "^2.15.2",
1717
"react": "18.2.0",
1818
"react-native": "0.74.2"
1919
},

‎ios/LiveKitReactNativeModule.swift‎

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ import React
66
struct LKEvents {
77
static let kEventVolumeProcessed = "LK_VOLUME_PROCESSED";
88
static let kEventMultibandProcessed = "LK_MULTIBAND_PROCESSED";
9+
static let kEventAudioData = "LK_AUDIO_DATA";
910
}
1011

1112
@objc(LivekitReactNativeModule)
@@ -178,6 +179,24 @@ public class LivekitReactNativeModule: RCTEventEmitter {
178179
session.unlockForConfiguration()
179180
}
180181

182+
@objc(createAudioSinkListener:trackId:)
183+
public func createAudioSinkListener(_ pcId: NSNumber, trackId: String) -> String {
184+
let renderer = AudioSinkRenderer(eventEmitter: self)
185+
let reactTag = self.audioRendererManager.registerRenderer(renderer)
186+
renderer.reactTag = reactTag
187+
self.audioRendererManager.attach(renderer: renderer, pcId: pcId, trackId: trackId)
188+
189+
return reactTag
190+
}
191+
192+
@objc(deleteAudioSinkListener:pcId:trackId:)
193+
public func deleteAudioSinkListener(_ reactTag: String, pcId: NSNumber, trackId: String) -> Any? {
194+
self.audioRendererManager.detach(rendererByTag: reactTag, pcId: pcId, trackId: trackId)
195+
self.audioRendererManager.unregisterRenderer(forReactTag: reactTag)
196+
197+
return nil
198+
}
199+
181200
@objc(createVolumeProcessor:trackId:)
182201
public func createVolumeProcessor(_ pcId: NSNumber, trackId: String) -> String {
183202
let renderer = VolumeAudioRenderer(intervalMs: 40.0, eventEmitter: self)
@@ -195,7 +214,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
195214

196215
return nil
197216
}
198-
217+
199218
@objc(createMultibandVolumeProcessor:pcId:trackId:)
200219
public func createMultibandVolumeProcessor(_ options: NSDictionary, pcId: NSNumber, trackId: String) -> String {
201220
let bands = (options["bands"] as? NSNumber)?.intValue ?? 5
@@ -237,6 +256,7 @@ public class LivekitReactNativeModule: RCTEventEmitter {
237256
return [
238257
LKEvents.kEventVolumeProcessed,
239258
LKEvents.kEventMultibandProcessed,
259+
LKEvents.kEventAudioData,
240260
]
241261
}
242262
}

‎ios/LivekitReactNativeModule.m‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,12 @@ @interface RCT_EXTERN_MODULE(LivekitReactNativeModule, RCTEventEmitter)
2121
/// Configure audio config for WebRTC
2222
RCT_EXTERN_METHOD(setAppleAudioConfiguration:(NSDictionary *) configuration)
2323

24+
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createAudioSinkListener:(nonnull NSNumber *)pcId
25+
trackId:(nonnull NSString *)trackId)
26+
27+
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(deleteAudioSinkListener:(nonnull NSString *)reactTag
28+
pcId:(nonnull NSNumber *)pcId
29+
trackId:(nonnull NSString *)trackId)
2430

2531
RCT_EXTERN__BLOCKING_SYNCHRONOUS_METHOD(createVolumeProcessor:(nonnull NSNumber *)pcId
2632
trackId:(nonnull NSString *)trackId)

‎ios/audio/AudioSinkRenderer.swift‎

Lines changed: 51 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
import livekit_react_native_webrtc
2+
import React
3+
4+
@objc
5+
public class AudioSinkRenderer: BaseAudioSinkRenderer {
6+
private let eventEmitter: RCTEventEmitter
7+
8+
@objc
9+
public var reactTag: String? = nil
10+
11+
@objc
12+
public init(eventEmitter: RCTEventEmitter) {
13+
self.eventEmitter = eventEmitter
14+
super.init()
15+
}
16+
17+
override public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
18+
guard pcmBuffer.format.commonFormat == .pcmFormatInt16,
19+
let channelData = pcmBuffer.int16ChannelData else {
20+
return
21+
}
22+
let channelCount = Int(pcmBuffer.format.channelCount)
23+
let channels = UnsafeBufferPointer(start: channelData, count: channelCount)
24+
let length = Int(pcmBuffer.frameCapacity * pcmBuffer.format.streamDescription.pointee.mBytesPerFrame)
25+
let data = NSData(bytes: channels[0], length: length)
26+
let base64 = data.base64EncodedString()
27+
NSLog("AUDIO DATA!!!!")
28+
NSLog("\(data.length)")
29+
NSLog(base64)
30+
NSLog("\(base64.count)")
31+
NSLog("\(length)")
32+
eventEmitter.sendEvent(withName: LKEvents.kEventAudioData, body: [
33+
"data": base64,
34+
"id": reactTag
35+
])
36+
}
37+
}
38+
39+
public class BaseAudioSinkRenderer: NSObject, RTCAudioRenderer {
40+
41+
public override init() {
42+
super.init()
43+
}
44+
45+
public func render(pcmBuffer: AVAudioPCMBuffer) {
46+
onData(pcmBuffer)
47+
}
48+
49+
public func onData(_ pcmBuffer: AVAudioPCMBuffer) {
50+
}
51+
}

‎package.json‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,11 @@
4444
"dependencies": {
4545
"@livekit/components-react": "^2.8.1",
4646
"array.prototype.at": "^1.1.1",
47+
"event-target-shim": "6.0.2",
4748
"events": "^3.3.0",
4849
"loglevel": "^1.8.0",
4950
"promise.allsettled": "^1.0.5",
51+
"react-native-quick-base64": "2.1.1",
5052
"react-native-url-polyfill": "^1.3.0",
5153
"typed-emitter": "^2.1.0",
5254
"web-streams-polyfill": "^4.1.0",
@@ -73,7 +75,7 @@
7375
"eslint-plugin-prettier": "^4.2.1",
7476
"husky": "^7.0.4",
7577
"jest": "^29.6.3",
76-
"livekit-client": "^2.9.8",
78+
"livekit-client": "^2.15.2",
7779
"pod-install": "^0.2.2",
7880
"prettier": "2.8.8",
7981
"react": "18.2.0",
@@ -85,7 +87,7 @@
8587
},
8688
"peerDependencies": {
8789
"@livekit/react-native-webrtc": "^125.0.10",
88-
"livekit-client": "^2.9.0",
90+
"livekit-client": "^2.15.2",
8991
"react": "*",
9092
"react-native": "*"
9193
},

0 commit comments

Comments
 (0)