feat: reviewed change for microphone and audio input

Author: iamprashant

node_modules/.package-lock.json

@@ -1,6 +1,6 @@
 {
     "name": "@rapidaai/react",
-    "version": "1.1.60",
+    "version": "1.1.61",
     "description": "An easy to use react client for building generative ai application using Rapida platform.",
     "repository": {
         "type": "git",

package-lock.json

@@ -146,6 +146,21 @@ export class VoiceAgent extends Agent {
           this.disconnect();
         }
       },
+      onConversationEvent: (event) => {
+        this.agentCallbacks.forEach((cb) => {
+          cb.onConversationEvent?.(event);
+        });
+      },
+      onMetric: (metric) => {
+        this.agentCallbacks.forEach((cb) => {
+          cb.onMetric?.(metric);
+        });
+      },
+      onConversationError: (error) => {
+        this.agentCallbacks.forEach((cb) => {
+          cb.onConversationError?.(error);
+        });
+      },
       onInitialization: (config) => {
         // console.log(`${LOG_PREFIX} callback -> onInitialization`, config);
         if (config?.assistantconversationid) {

package.json

@@ -23,7 +23,7 @@
  */
 
 import { AgentConfig } from "@/rapida/types/agent-config";
-import { isChrome, isEdge, isWindows, isFirefox, isSinkIdSupported } from "@/rapida/utils";
+import { isChrome, isEdge, isWindows, isSinkIdSupported } from "@/rapida/utils";
 
 /** Sample rate for Opus */
 const OPUS_SAMPLE_RATE = 48000;
@@ -69,17 +69,30 @@ export class AudioMediaManager {
 
   /** Capture the local microphone stream */
   async setupLocalMedia(): Promise<void> {
+    // Clean up any existing mic resources before re-capturing.
+    // Without this, a second call (e.g. hot reconnect) would overwrite
+    // localStream while the old tracks keep running and the mic indicator stays on.
+    if (this.localStream) {
+      this.localStream.getTracks().forEach(t => t.stop());
+      this.localStream = null;
+    }
+    if (this._inputAnalyser) {
+      this._inputAnalyser.disconnect();
+      this._inputAnalyser = null;
+    }
+
     const constraints = this.getAudioConstraints();
 
     try {
       this.localStream = await navigator.mediaDevices.getUserMedia({ audio: constraints, video: false });
     } catch (error: any) {
-      // OverconstrainedError can occur on any platform when an 'exact' deviceId
-      // constraint cannot be satisfied (e.g. the device was unplugged).
-      // Retry with simplified constraints so the call degrades gracefully
-      // to the default device rather than failing completely.
-      if (error?.name === "OverconstrainedError") {
-        console.warn("[AudioMediaManager] Retrying with simplified audio constraints after OverconstrainedError");
+      // OverconstrainedError — 'exact' deviceId constraint cannot be satisfied
+      //   (Firefox throws this when the device is gone)
+      // NotFoundError — requested device does not exist
+      //   (Chrome throws this for the same situation)
+      // Both mean: fall back to the default device rather than failing completely.
+      if (error?.name === "OverconstrainedError" || error?.name === "NotFoundError") {
+        console.warn("[AudioMediaManager] Retrying with simplified audio constraints:", error.name);
         this.localStream = await navigator.mediaDevices.getUserMedia({
           audio: this.getSimplifiedAudioConstraints(),
           video: false,
@@ -117,6 +130,11 @@ export class AudioMediaManager {
    * Windows browsers (especially Edge) may need special handling.
    */
   private async setupAudioContext(): Promise<void> {
+    // Re-use an existing running context rather than silently abandoning it.
+    // Callers that need a fresh context (e.g. after close()) must null
+    // this.audioContext themselves (disconnectAudio does this).
+    if (this.audioContext && this.audioContext.state !== "closed") return;
+
     const AudioContextClass = getAudioContextClass();
     if (!AudioContextClass) {
       console.warn("[AudioMediaManager] AudioContext not available in this browser");
@@ -458,16 +476,20 @@ export class AudioMediaManager {
   async setInputDevice(deviceId: string): Promise<void> {
     this.agentConfig.inputOptions.device = deviceId;
     this.localStream?.getTracks().forEach(t => t.stop());
+    // Null immediately so that if getUserMedia throws, getLocalAudioTrack()
+    // returns undefined rather than a stream full of stopped (silent) tracks.
+    this.localStream = null;
 
     try {
       this.localStream = await navigator.mediaDevices.getUserMedia({
         audio: this.getAudioConstraints(),
         video: false,
       });
     } catch (error: any) {
-      // On Windows, retry with simplified constraints if device selection fails
-      if (isWindows() && (error?.name === "OverconstrainedError" || error?.name === "NotFoundError")) {
-        console.warn("[AudioMediaManager] Device selection failed, retrying with simplified constraints");
+      // Same cross-platform fallback as setupLocalMedia:
+      // Chrome → NotFoundError, Firefox → OverconstrainedError, for missing 'exact' device.
+      if (error?.name === "OverconstrainedError" || error?.name === "NotFoundError") {
+        console.warn("[AudioMediaManager] Device selection failed, retrying with simplified constraints:", error.name);
         this.localStream = await navigator.mediaDevices.getUserMedia({
           audio: this.getSimplifiedAudioConstraints(),
           video: false,
@@ -478,7 +500,7 @@ export class AudioMediaManager {
     }
 
     // Reconnect analyser (disconnect old source first to avoid leaking audio nodes)
-    if (this.audioContext && this._inputAnalyser) {
+    if (this.audioContext && this._inputAnalyser && this.localStream) {
       this._inputAnalyser.disconnect();
       const source = this.audioContext.createMediaStreamSource(this.localStream);
       this._inputAnalyser = this.audioContext.createAnalyser();

src/agents/voice-agent.ts

@@ -22,23 +22,19 @@
  *  Author: Prashant <prashant@rapida.ai>
  *
  */
-export function isIosDevice() {
+export function isIosDevice(): boolean {
+  if (typeof navigator === "undefined") return false;
   return (
-    [
-      "iPad Simulator",
-      "iPhone Simulator",
-      "iPod Simulator",
-      "iPad",
-      "iPhone",
-      "iPod",
-    ].includes(navigator.platform) ||
-    // iPad on iOS 13 detection
+    /iPad|iPhone|iPod/.test(navigator.userAgent) ||
+    // iPad on iOS 13+ reports as "MacIntel" with touch support
+    (navigator.platform === "MacIntel" && navigator.maxTouchPoints > 1) ||
     (navigator.userAgent.includes("Mac") && "ontouchend" in document)
   );
 }
 
-export function isAndroidDevice() {
-  return /android/i.test(navigator.userAgent);
+export function isAndroidDevice(): boolean {
+  if (typeof navigator === "undefined") return false;
+  return /Android/i.test(navigator.userAgent);
 }
 
 export function isWindowsDevice() {

src/audio/audio-media-manager.ts

@@ -48,6 +48,10 @@ import type { AudioMediaManager } from "./audio-media-manager";
  * This is pure dispatch logic — it holds no state of its own.
  */
 export class MessageProtocolHandler {
+  // Serialization chain ensures signaling messages (CONFIG → OFFER → ICE)
+  // are processed in order even though gRPC callbacks fire without await.
+  private processingChain: Promise<void> = Promise.resolve();
+
   constructor(
     private callbacks: AgentCallback,
     private signaling: GrpcSignalingManager,
@@ -60,6 +64,13 @@ export class MessageProtocolHandler {
   // ---------------------------------------------------------------------------
 
   async handleMessage(response: WebTalkResponse): Promise<void> {
+    this.processingChain = this.processingChain
+      .then(() => this._handleMessage(response))
+      .catch(err => console.error("[Protocol] Message processing error", err));
+    return this.processingChain;
+  }
+
+  private async _handleMessage(response: WebTalkResponse): Promise<void> {
     // Server signaling (SDP, ICE, config, ready, clear, error)
     if (response.hasSignaling()) {
       const sig = response.getSignaling();
@@ -109,6 +120,24 @@ export class MessageProtocolHandler {
       const directive = response.getDirective();
       if (directive) this.callbacks.onDirective?.(directive.toObject());
     }
+
+    // Pipeline conversation event (STT, TTS, LLM, session, etc.)
+    if (response.hasEvent()) {
+      const event = response.getEvent();
+      if (event) this.callbacks.onConversationEvent?.(event.toObject());
+    }
+
+    // Conversation error (server-side error with conversation ID and details)
+    if (response.hasError()) {
+      const error = response.getError();
+      if (error) this.callbacks.onError?.(new Error(error.getMessage()));
+    }
+
+    // Metric (server-side performance/latency data)
+    if (response.hasMetric()) {
+      const metric = response.getMetric();
+      if (metric) this.callbacks.onMetric?.(metric.toObject());
+    }
   }
 
   // ---------------------------------------------------------------------------
@@ -122,9 +151,16 @@ export class MessageProtocolHandler {
     const messageCase = signaling.getMessageCase();
 
     switch (messageCase) {
-      case ServerSignaling.MessageCase.CONFIG:
-        this.peer.setup();
+      case ServerSignaling.MessageCase.CONFIG: {
+        const config = signaling.getConfig();
+        const iceServers = config?.getIceserversList()?.map(srv => ({
+          urls: srv.getUrlsList(),
+          username: srv.getUsername() || undefined,
+          credential: srv.getCredential() || undefined,
+        })) as RTCIceServer[] | undefined;
+        this.peer.setup(iceServers);
         break;
+      }
 
       case ServerSignaling.MessageCase.SDP: {
         try {

src/audio/compatibility.ts

@@ -28,7 +28,7 @@ import { AgentCallback } from "@/rapida/types/agent-callback";
 // WebRTC Constants
 // ============================================================================
 
-const ICE_SERVERS: RTCIceServer[] = [
+const DEFAULT_ICE_SERVERS: RTCIceServer[] = [
   { urls: "stun:stun.l.google.com:19302" },
   { urls: "stun:stun1.l.google.com:19302" },
 ];
@@ -59,17 +59,22 @@ export class WebRTCPeerManager {
   // Lifecycle
   // ---------------------------------------------------------------------------
 
-  /** Create a new RTCPeerConnection */
-  setup(): void {
-    // Close existing connection before creating a new one to prevent leaks
+  /** Create a new RTCPeerConnection, optionally using server-provided ICE servers */
+  setup(iceServers?: RTCIceServer[]): void {
+    // Close existing connection before creating a new one to prevent leaks.
+    // Detach handlers BEFORE closing so the "closed" state change does not
+    // spuriously fire onDisconnected when the peer is simply being replaced.
     if (this.peerConnection) {
+      this.peerConnection.onconnectionstatechange = null;
+      this.peerConnection.ontrack = null;
+      this.peerConnection.onicecandidate = null;
       try { this.peerConnection.close(); } catch { }
       this.peerConnection = null;
       this._isConnected = false;
     }
 
     this.peerConnection = new RTCPeerConnection({
-      iceServers: ICE_SERVERS,
+      iceServers: iceServers?.length ? iceServers : DEFAULT_ICE_SERVERS,
       iceTransportPolicy: ICE_TRANSPORT_POLICY,
       bundlePolicy: BUNDLE_POLICY,
     });
@@ -81,10 +86,14 @@ export class WebRTCPeerManager {
       this.onRemoteTrack(stream);
     };
 
-    // ICE candidates
+    // ICE candidates — Safari may lack toJSON() on RTCIceCandidate
     this.peerConnection.onicecandidate = (event) => {
       if (event.candidate) {
-        this.onICECandidate(event.candidate.toJSON());
+        const c = event.candidate;
+        const json = typeof c.toJSON === "function"
+          ? c.toJSON()
+          : { candidate: c.candidate, sdpMid: c.sdpMid, sdpMLineIndex: c.sdpMLineIndex };
+        this.onICECandidate(json);
       }
     };
 
@@ -150,7 +159,18 @@ export class WebRTCPeerManager {
 
     const track = localStream?.getAudioTracks()[0];
     const transceivers = this.peerConnection!.getTransceivers();
-    const audioTransceiver = transceivers.find(t => t.receiver.track?.kind === "audio");
+
+    // Safari (pre-14.1) leaves receiver.track null after setRemoteDescription
+    // until the first remote packet arrives, so the receiver-track check alone
+    // is unreliable. Use a cascade of fallbacks:
+    //   1. receiver.track.kind  — Chrome, Firefox, Safari 14.1+
+    //   2. sender.track.kind    — if the sender already has a track attached
+    //   3. transceivers[0]      — voice-only sessions always have exactly one
+    //                             audio m-line, so the first transceiver is it
+    const audioTransceiver =
+      transceivers.find(t => t.receiver.track?.kind === "audio") ??
+      transceivers.find(t => t.sender.track?.kind === "audio") ??
+      transceivers[0];
 
     if (!audioTransceiver) {
       console.warn("No audio transceiver found in offer");
@@ -160,13 +180,21 @@ export class WebRTCPeerManager {
     }
 
     if (track) {
-      // Audio mode — bidirectional
+      // Audio mode — bidirectional.
+      // Set direction first so the answer SDP reflects sendrecv intent.
       audioTransceiver.direction = "sendrecv";
+
+      // replaceTrack must succeed — if it throws, revert direction so the
+      // answer SDP does not falsely advertise a send-capable track, and
+      // re-throw so the caller can surface the failure rather than silently
+      // completing negotiation with no mic audio reaching the server.
       try {
         await audioTransceiver.sender.replaceTrack(track);
       } catch (error) {
-        console.error("Failed to replace audio track", error);
+        audioTransceiver.direction = "recvonly";
+        throw error;
       }
+
       this.setCodecPreferences(audioTransceiver);
     } else {
       // Text-only mode — receive only (no microphone)