feat: telemetry changes

Author: iamprashant

package.json

@@ -1,6 +1,6 @@
 {
     "name": "@rapidaai/react",
-    "version": "1.1.62",
+    "version": "1.1.63",
     "description": "An easy to use react client for building generative ai application using Rapida platform.",
     "repository": {
         "type": "git",

src/agents/voice-agent.ts

@@ -94,6 +94,10 @@ export class VoiceAgent extends Agent {
             // console.log(`${LOG_PREFIX} WebRTC peer ${state} but gRPC still alive — staying connected`);
             this.switchToTextModeOnDisconnect();
           } else {
+            // gRPC stream is gone — null the transport so the next connect()
+            // call creates a fresh one instead of returning early on the
+            // `if (this.webrtcTransport) return` guard.
+            this.webrtcTransport = null;
             this.connectionState = ConnectionState.Disconnected;
             this.emit(AgentEvent.ConnectionStateEvent, ConnectionState.Disconnected);
             this.switchToTextModeOnDisconnect();
@@ -664,7 +668,14 @@ export class VoiceAgent extends Agent {
 
     // Add or remove the audio transport
     if (input === Channel.Audio) {
-      await this.webrtcTransport?.reconnectAudio();
+      try {
+        await this.webrtcTransport?.reconnectAudio();
+      } catch (error) {
+        // gRPC stream was lost between ensureConnected and reconnectAudio.
+        // Trigger a full reconnect on the next user action.
+        this.emit(AgentEvent.ErrorEvent, "client", `Audio reconnect failed: ${error}`);
+        this.webrtcTransport = null;
+      }
     } else {
       await this.webrtcTransport?.disconnectAudioOnly();
     }

src/audio/audio-media-manager.ts

@@ -23,7 +23,7 @@
  */
 
 import { AgentConfig } from "@/rapida/types/agent-config";
-import { isChrome, isEdge, isWindows, isSinkIdSupported } from "@/rapida/utils";
+import { isChrome, isEdge, isWindows, isSafari, isIOS, isSinkIdSupported } from "@/rapida/utils";
 
 /** Sample rate for Opus */
 const OPUS_SAMPLE_RATE = 48000;
@@ -165,7 +165,7 @@ export class AudioMediaManager {
     const base: MediaTrackConstraints = {
       echoCancellation: true,
       noiseSuppression: true,
-      autoGainControl: true,
+      autoGainControl: false,
     };
 
     if (this.agentConfig.inputOptions.device) {
@@ -183,7 +183,11 @@ export class AudioMediaManager {
       channelCount: { ideal: 1 },
       echoCancellation: true,
       noiseSuppression: true,
-      autoGainControl: true,
+      // Disable AGC for voice AI: when the assistant finishes speaking and the
+      // user starts talking, AGC would abruptly boost the mic gain (it was
+      // suppressed while output was playing), clipping the first syllable and
+      // causing a jarring volume jump. The STT model handles normalization.
+      autoGainControl: false,
     };
 
     if (this.agentConfig.inputOptions.device) {
@@ -197,8 +201,9 @@ export class AudioMediaManager {
     }
 
     // On Windows, remove sampleRate constraint — WebRTC handles resampling internally,
-    // and forcing 48kHz can conflict with WASAPI on 44100Hz audio hardware
-    if (isWindows()) {
+    // and forcing 48kHz can conflict with WASAPI on 44100Hz audio hardware.
+    // On Safari/iOS, sampleRate is not a supported getUserMedia constraint and must be omitted.
+    if (isWindows() || isSafari() || isIOS()) {
       delete base.sampleRate;
     }
 
@@ -208,11 +213,11 @@ export class AudioMediaManager {
         ...base,
         // @ts-ignore Chrome/Edge-specific
         googEchoCancellation: true,
-        // @ts-ignore
-        googAutoGainControl: true,
+        // @ts-ignore AGC disabled — matches autoGainControl: false above
+        googAutoGainControl: false,
         // @ts-ignore
         googNoiseSuppression: true,
-        // @ts-ignore
+        // @ts-ignore removes low-frequency rumble (fan/AC noise)
         googHighpassFilter: true,
       };
     }

src/audio/grpc-signaling-manager.ts

@@ -87,6 +87,11 @@ export class GrpcSignalingManager {
         });
 
         this.grpcStream.on("end", () => {
+          // Null the stream immediately so isConnected returns false.
+          // Without this, callers checking isConnected after the stream
+          // ends still see true and attempt writes on a dead stream.
+          this.grpcStream = null;
+          this.initializationSent = false;
           this.callbacks.onConnectionStateChange?.("disconnected");
           this.callbacks.onDisconnected?.();
         });

src/audio/webrtc-peer-manager.ts

@@ -86,15 +86,13 @@ export class WebRTCPeerManager {
       this.onRemoteTrack(stream);
     };
 
-    // ICE candidates — Safari may lack toJSON() on RTCIceCandidate
-    this.peerConnection.onicecandidate = (event) => {
-      if (event.candidate) {
-        const c = event.candidate;
-        const json = typeof c.toJSON === "function"
-          ? c.toJSON()
-          : { candidate: c.candidate, sdpMid: c.sdpMid, sdpMLineIndex: c.sdpMLineIndex };
-        this.onICECandidate(json);
-      }
+    // ICE candidates are NOT sent individually — all candidates are embedded
+    // inline in the answer SDP (complete ICE, not trickle ICE). Trickle ICE
+    // candidates sent before the server receives the answer SDP cause
+    // "InvalidStateError: remote description is not set" on the server.
+    // We capture them via waitForIceGathering() instead.
+    this.peerConnection.onicecandidate = (_event) => {
+      // no-op: candidates are captured via localDescription after gathering
     };
 
     // Connection state
@@ -151,14 +149,20 @@ export class WebRTCPeerManager {
    * In text-only mode `localStream` is null — we still complete the WebRTC
    * negotiation (the server always requires it) but set the transceiver to
    * `recvonly` so no microphone is needed.
+   *
+   * After setLocalDescription we wait for local ICE gathering to complete
+   * so that the answer SDP contains all client candidates inline. This pairs
+   * with the server sending a complete offer (all server candidates inline).
+   * Together they eliminate trickle-ICE timing bugs that cause Safari to fail.
    */
   async handleOffer(sdp: string, localStream: MediaStream | null): Promise<void> {
     if (!this.peerConnection) this.setup();
+    const pc = this.peerConnection!;
 
-    await this.peerConnection!.setRemoteDescription({ type: "offer", sdp });
+    await pc.setRemoteDescription({ type: "offer", sdp });
 
     const track = localStream?.getAudioTracks()[0];
-    const transceivers = this.peerConnection!.getTransceivers();
+    const transceivers = pc.getTransceivers();
 
     // Safari (pre-14.1) leaves receiver.track null after setRemoteDescription
     // until the first remote packet arrives, so the receiver-track check alone
@@ -174,25 +178,25 @@ export class WebRTCPeerManager {
 
     if (!audioTransceiver) {
       console.warn("No audio transceiver found in offer");
-      const answer = await this.peerConnection!.createAnswer();
-      await this.peerConnection!.setLocalDescription(answer);
+      const answer = await pc.createAnswer();
+      await pc.setLocalDescription(answer);
+      await this.waitForIceGathering(pc);
       return;
     }
 
     if (track) {
       // Audio mode — bidirectional.
-      // Set direction first so the answer SDP reflects sendrecv intent.
       audioTransceiver.direction = "sendrecv";
 
-      // replaceTrack must succeed — if it throws, revert direction so the
-      // answer SDP does not falsely advertise a send-capable track, and
-      // re-throw so the caller can surface the failure rather than silently
-      // completing negotiation with no mic audio reaching the server.
+      // replaceTrack attaches the mic track to the transceiver's sender.
+      // If it fails (e.g. on older Safari), fall back to recvonly rather
+      // than throwing — the answer is still sent and the server can at least
+      // push audio to the client (user hears the assistant, doesn't send mic).
       try {
         await audioTransceiver.sender.replaceTrack(track);
       } catch (error) {
+        console.warn("[WebRTCPeerManager] replaceTrack failed, falling back to recvonly", error);
         audioTransceiver.direction = "recvonly";
-        throw error;
       }
 
       this.setCodecPreferences(audioTransceiver);
@@ -201,9 +205,37 @@ export class WebRTCPeerManager {
       audioTransceiver.direction = "recvonly";
     }
 
-    // Create and return answer
-    const answer = await this.peerConnection!.createAnswer();
-    await this.peerConnection!.setLocalDescription(answer);
+    const answer = await pc.createAnswer();
+    await pc.setLocalDescription(answer);
+
+    // Wait for local ICE gathering to complete so the answer SDP contains
+    // all client candidates inline. Safari handles the complete SDP reliably;
+    // it often drops or misorders trickle ICE candidates that arrive as
+    // separate addIceCandidate() calls after the offer/answer exchange.
+    await this.waitForIceGathering(pc);
+  }
+
+  /**
+   * Wait for the RTCPeerConnection's ICE gathering to reach "complete".
+   * Resolves immediately if already complete.
+   * Times out after 5 seconds to avoid blocking forever if STUN is unreachable.
+   */
+  private waitForIceGathering(pc: RTCPeerConnection): Promise<void> {
+    if (pc.iceGatheringState === "complete") return Promise.resolve();
+
+    return new Promise<void>((resolve) => {
+      const timeout = setTimeout(resolve, 5000);
+
+      const handler = () => {
+        if (pc.iceGatheringState === "complete") {
+          clearTimeout(timeout);
+          pc.removeEventListener("icegatheringstatechange", handler);
+          resolve();
+        }
+      };
+
+      pc.addEventListener("icegatheringstatechange", handler);
+    });
   }
 
   /** Get local SDP (answer) after handleOffer was called */

src/clients/protos/artifacts

@@ -405,6 +405,142 @@ export namespace GetAllAssistantTelemetryRequest {
   }
 }
 
+export class TelemetryEvent extends jspb.Message {
+  getMessageid(): string;
+  setMessageid(value: string): void;
+
+  getAssistantid(): string;
+  setAssistantid(value: string): void;
+
+  getAssistantconversationid(): string;
+  setAssistantconversationid(value: string): void;
+
+  getProjectid(): string;
+  setProjectid(value: string): void;
+
+  getOrganizationid(): string;
+  setOrganizationid(value: string): void;
+
+  getName(): string;
+  setName(value: string): void;
+
+  getDataMap(): jspb.Map<string, string>;
+  clearDataMap(): void;
+  hasTime(): boolean;
+  clearTime(): void;
+  getTime(): google_protobuf_timestamp_pb.Timestamp | undefined;
+  setTime(value?: google_protobuf_timestamp_pb.Timestamp): void;
+
+  serializeBinary(): Uint8Array;
+  toObject(includeInstance?: boolean): TelemetryEvent.AsObject;
+  static toObject(includeInstance: boolean, msg: TelemetryEvent): TelemetryEvent.AsObject;
+  static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
+  static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
+  static serializeBinaryToWriter(message: TelemetryEvent, writer: jspb.BinaryWriter): void;
+  static deserializeBinary(bytes: Uint8Array): TelemetryEvent;
+  static deserializeBinaryFromReader(message: TelemetryEvent, reader: jspb.BinaryReader): TelemetryEvent;
+}
+
+export namespace TelemetryEvent {
+  export type AsObject = {
+    messageid: string,
+    assistantid: string,
+    assistantconversationid: string,
+    projectid: string,
+    organizationid: string,
+    name: string,
+    dataMap: Array<[string, string]>,
+    time?: google_protobuf_timestamp_pb.Timestamp.AsObject,
+  }
+}
+
+export class TelemetryMetric extends jspb.Message {
+  getContextid(): string;
+  setContextid(value: string): void;
+
+  getAssistantid(): string;
+  setAssistantid(value: string): void;
+
+  getAssistantconversationid(): string;
+  setAssistantconversationid(value: string): void;
+
+  getProjectid(): string;
+  setProjectid(value: string): void;
+
+  getOrganizationid(): string;
+  setOrganizationid(value: string): void;
+
+  getScope(): string;
+  setScope(value: string): void;
+
+  clearMetricsList(): void;
+  getMetricsList(): Array<common_pb.Metric>;
+  setMetricsList(value: Array<common_pb.Metric>): void;
+  addMetrics(value?: common_pb.Metric, index?: number): common_pb.Metric;
+
+  hasTime(): boolean;
+  clearTime(): void;
+  getTime(): google_protobuf_timestamp_pb.Timestamp | undefined;
+  setTime(value?: google_protobuf_timestamp_pb.Timestamp): void;
+
+  serializeBinary(): Uint8Array;
+  toObject(includeInstance?: boolean): TelemetryMetric.AsObject;
+  static toObject(includeInstance: boolean, msg: TelemetryMetric): TelemetryMetric.AsObject;
+  static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
+  static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
+  static serializeBinaryToWriter(message: TelemetryMetric, writer: jspb.BinaryWriter): void;
+  static deserializeBinary(bytes: Uint8Array): TelemetryMetric;
+  static deserializeBinaryFromReader(message: TelemetryMetric, reader: jspb.BinaryReader): TelemetryMetric;
+}
+
+export namespace TelemetryMetric {
+  export type AsObject = {
+    contextid: string,
+    assistantid: string,
+    assistantconversationid: string,
+    projectid: string,
+    organizationid: string,
+    scope: string,
+    metricsList: Array<common_pb.Metric.AsObject>,
+    time?: google_protobuf_timestamp_pb.Timestamp.AsObject,
+  }
+}
+
+export class TelemetryRecord extends jspb.Message {
+  hasEvent(): boolean;
+  clearEvent(): void;
+  getEvent(): TelemetryEvent | undefined;
+  setEvent(value?: TelemetryEvent): void;
+
+  hasMetric(): boolean;
+  clearMetric(): void;
+  getMetric(): TelemetryMetric | undefined;
+  setMetric(value?: TelemetryMetric): void;
+
+  getRecordCase(): TelemetryRecord.RecordCase;
+  serializeBinary(): Uint8Array;
+  toObject(includeInstance?: boolean): TelemetryRecord.AsObject;
+  static toObject(includeInstance: boolean, msg: TelemetryRecord): TelemetryRecord.AsObject;
+  static extensions: {[key: number]: jspb.ExtensionFieldInfo<jspb.Message>};
+  static extensionsBinary: {[key: number]: jspb.ExtensionFieldBinaryInfo<jspb.Message>};
+  static serializeBinaryToWriter(message: TelemetryRecord, writer: jspb.BinaryWriter): void;
+  static deserializeBinary(bytes: Uint8Array): TelemetryRecord;
+  static deserializeBinaryFromReader(message: TelemetryRecord, reader: jspb.BinaryReader): TelemetryRecord;
+}
+
+export namespace TelemetryRecord {
+  export type AsObject = {
+    event?: TelemetryEvent.AsObject,
+    metric?: TelemetryMetric.AsObject,
+  }
+
+  export enum RecordCase {
+    RECORD_NOT_SET = 0,
+    EVENT = 1,
+    METRIC = 2,
+  }
+}
+
 export class GetAllAssistantTelemetryResponse extends jspb.Message {
   getCode(): number;
   setCode(value: number): void;
@@ -413,9 +549,9 @@ export class GetAllAssistantTelemetryResponse extends jspb.Message {
   setSuccess(value: boolean): void;
 
   clearDataList(): void;
-  getDataList(): Array<common_pb.Telemetry>;
-  setDataList(value: Array<common_pb.Telemetry>): void;
-  addData(value?: common_pb.Telemetry, index?: number): common_pb.Telemetry;
+  getDataList(): Array<TelemetryRecord>;
+  setDataList(value: Array<TelemetryRecord>): void;
+  addData(value?: TelemetryRecord, index?: number): TelemetryRecord;
 
   hasError(): boolean;
   clearError(): void;
@@ -441,7 +577,7 @@ export namespace GetAllAssistantTelemetryResponse {
   export type AsObject = {
     code: number,
     success: boolean,
-    dataList: Array<common_pb.Telemetry.AsObject>,
+    dataList: Array<TelemetryRecord.AsObject>,
     error?: common_pb.Error.AsObject,
     paginated?: common_pb.Paginated.AsObject,
   }