Add support for react-native preconnect audio (livekit#1598)

Author: davidliu

.changeset/breezy-jobs-yawn.md

@@ -0,0 +1,5 @@
+---
+'livekit-client': patch
+---
+
+Add support for react-native preconnect audio

src/room/participant/LocalParticipant.ts

@@ -558,10 +558,14 @@ export default class LocalParticipant extends Participant {
         }
 
         for (const localTrack of localTracks) {
+          const opts: TrackPublishOptions = {
+            ...this.roomOptions.publishDefaults,
+            ...options,
+          };
           if (
             source === Track.Source.Microphone &&
             isAudioTrack(localTrack) &&
-            publishOptions?.preConnectBuffer
+            opts.preConnectBuffer
           ) {
             this.log.info('starting preconnect buffer for microphone', {
               ...this.logContext,
@@ -1302,6 +1306,7 @@ export default class LocalParticipant extends Participant {
       ti.audioFeatures.includes(AudioTrackFeature.TF_PRECONNECT_BUFFER)
     ) {
       const stream = track.getPreConnectBuffer();
+      const mimeType = track.getPreConnectBufferMimeType();
       // TODO: we're registering the listener after negotiation, so there might be a race
       this.on(ParticipantEvent.LocalTrackSubscribed, (pub) => {
         if (pub.trackSid === ti.sid) {
@@ -1335,7 +1340,7 @@ export default class LocalParticipant extends Participant {
             });
             const writer = await this.streamBytes({
               name: 'preconnect-buffer',
-              mimeType: 'audio/opus',
+              mimeType,
               topic: 'lk.agent.pre-connect-audio-buffer',
               destinationIdentities: [agent.identity],
               attributes: {

src/room/track/LocalTrack.ts

@@ -649,5 +649,9 @@ export default abstract class LocalTrack<
     return this.localTrackRecorder?.byteStream;
   }
 
+  getPreConnectBufferMimeType() {
+    return this.localTrackRecorder?.mimeType;
+  }
+
   protected abstract monitorSender(): void;
 }

src/room/track/record.ts

@@ -51,11 +51,24 @@ export class LocalTrackRecorder<T extends LocalTrack> extends RecorderBase {
       start: (controller) => {
         streamController = controller;
         dataListener = async (event: BlobEvent) => {
-          const arrayBuffer = await event.data.arrayBuffer();
+          let data: Uint8Array;
+
+          if (event.data.arrayBuffer) {
+            const arrayBuffer = await event.data.arrayBuffer();
+            data = new Uint8Array(arrayBuffer);
+
+            // @ts-expect-error react-native passes over Uint8Arrays directly
+          } else if (event.data.byteArray) {
+            // @ts-expect-error
+            data = event.data.byteArray as Uint8Array;
+          } else {
+            throw new Error('no data available!');
+          }
+
           if (isClosed()) {
             return;
           }
-          controller.enqueue(new Uint8Array(arrayBuffer));
+          controller.enqueue(data);
         };
         this.addEventListener('dataavailable', dataListener);
       },