refine realtime

Author: Jicheng Lu

src/lib/helpers/pcmProcessor.js src/lib/helpers/realtime/pcmProcessor.jssrc/lib/helpers/pcmProcessor.js renamed to src/lib/helpers/realtime/pcmProcessor.js

@@ -10,9 +10,11 @@ class AudioProcessingWorklet extends AudioWorkletProcessor {
   // current write index
   bufferWriteIndex = 0;
 
+  speaking = false;
+  threshold = 0.1;
+
   constructor() {
     super();
-    this.hasAudio = false;
   }
 
   /**
@@ -22,6 +24,7 @@ class AudioProcessingWorklet extends AudioWorkletProcessor {
   process(inputs) {
     if (inputs[0].length) {
       const channel0 = inputs[0][0];
+      this.speaking = channel0.some(sample => Math.abs(sample) >= this.threshold);
       this.processChunk(channel0);
     }
     return true;
@@ -31,6 +34,7 @@ class AudioProcessingWorklet extends AudioWorkletProcessor {
     this.port.postMessage({
       event: "chunk",
       data: {
+        speaking: this.speaking,
         int16arrayBuffer: this.buffer.slice(0, this.bufferWriteIndex).buffer,
       },
     });

src/lib/services/realtime-chat-service.js

@@ -1,5 +1,5 @@
 import { PUBLIC_SERVICE_URL } from "$env/static/public";
-import { AudioRecordingWorklet } from "$lib/helpers/pcmProcessor";
+import { AudioRecordingWorklet } from "$lib/helpers/realtime/pcmProcessor";
 
 // @ts-ignore
 const AudioContext = window.AudioContext || window.webkitAudioContext;
@@ -15,60 +15,63 @@ let audioQueue = [];
 /** @type {boolean} */
 let isPlaying = false;
 
-export const realtimeChat = {
-    
-    /** @type {WebSocket | null} */
-    socket: null,
+/** @type {WebSocket | null} */
+let socket = null;
 
-    /** @type {MediaStream | null} */
-    mediaStream: null,
+/** @type {MediaStream | null} */
+let mediaStream = null;
 
-    /** @type {AudioWorkletNode | null} */
-    workletNode: null,
+/** @type {AudioWorkletNode | null} */
+let workletNode = null;
 
-    /** @type {MediaStreamAudioSourceNode | null} */
-    micSource: null,
+/** @type {MediaStreamAudioSourceNode | null} */
+let micSource = null;
 
+export const realtimeChat = {
+    
     /**
      * @param {string} agentId
      * @param {string} conversationId
      */
     start(agentId, conversationId) {
         reset();
         const wsUrl = buildWebsocketUrl();
-        this.socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`);
+        socket = new WebSocket(`${wsUrl}/chat/stream/${agentId}/${conversationId}`);
 
-        this.socket.onopen = async () => {
+        socket.onopen = async () => {
             console.log("WebSocket connected");
 
-            this.socket?.send(JSON.stringify({
+            socket?.send(JSON.stringify({
                 event: "start"
             }));
 
-            this.mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
+            mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
             audioCtx = new AudioContext({ sampleRate: sampleRate });
 
             const workletName = "audio-recorder-worklet";
             const src = createWorkletFromSrc(workletName, AudioRecordingWorklet);
             await audioCtx.audioWorklet.addModule(src);
 
-            this.workletNode = new AudioWorkletNode(audioCtx, workletName);
-            this.micSource = audioCtx.createMediaStreamSource(this.mediaStream);
-            this.micSource.connect(this.workletNode);
+            workletNode = new AudioWorkletNode(audioCtx, workletName);
+            micSource = audioCtx.createMediaStreamSource(mediaStream);
+            micSource.connect(workletNode);
 
-            this.workletNode.port.onmessage = event => {
+            workletNode.port.onmessage = event => {
                 const arrayBuffer = event.data.data.int16arrayBuffer;
-                if (arrayBuffer && this.socket?.readyState === WebSocket.OPEN) {
+                if (arrayBuffer && socket?.readyState === WebSocket.OPEN) {
+                    if (event.data.data.speaking) {
+                        reset();
+                    }
                     const arrayBufferString = arrayBufferToBase64(arrayBuffer);
-                    this.socket.send(JSON.stringify({
+                    socket.send(JSON.stringify({
                         event: 'media',
                         payload: arrayBufferString
                     }));
                 }
             };
         };
 
-        this.socket.onmessage = (/** @type {MessageEvent} */ e) => {
+        socket.onmessage = (/** @type {MessageEvent} */ e) => {
             try {
                 const json = JSON.parse(e.data);
                 if (json.event === 'media' && !!json.media.payload) {
@@ -80,34 +83,37 @@ export const realtimeChat = {
             }
         };
 
-        this.socket.onclose = () => {
+        socket.onclose = () => {
             console.log("Websocket closed");
         };
 
-        this.socket.onerror = (/** @type {Event} */ e) => {
+        socket.onerror = (/** @type {Event} */ e) => {
             console.error('WebSocket error', e);
         };
     },
 
     stop() {
         reset();
 
-        if (this.mediaStream) {
-            this.mediaStream.getTracks().forEach(t => t.stop());
-            this.mediaStream = null;
+        if (mediaStream) {
+            mediaStream.getTracks().forEach(t => t.stop());
+            mediaStream = null;
         }
 
-        if (this.workletNode) {
-            this.micSource?.disconnect(this.workletNode);
-            this.workletNode.port.close();
-            this.workletNode.disconnect();
+        if (workletNode) {
+            micSource?.disconnect(workletNode);
+            workletNode.port.close();
+            workletNode.disconnect();
+            micSource = null;
+            workletNode = null;
         }
 
-        if (this.socket?.readyState === WebSocket.OPEN) {
-            this.socket.send(JSON.stringify({
+        if (socket?.readyState === WebSocket.OPEN) {
+            socket.send(JSON.stringify({
                 event: 'disconnect'
             }));
-            this.socket.close();
+            socket.close();
+            socket = null;
         }
     }
 };
@@ -143,7 +149,7 @@ function enqueueAudioChunk(base64Audio) {
     audioQueue.push(audioBuffer);
 
     if (!isPlaying) {
-      playNext();
+        playNext();
     }
 }
 
@@ -159,7 +165,6 @@ function playNext() {
     const source = audioCtx.createBufferSource();
     source.buffer = buffer;
     source.connect(audioCtx.destination);
-  
     source.onended = () => {
         playNext();
     };
@@ -221,7 +226,7 @@ function convert16BitPCMToFloat32(buffer) {
     for (let i = 0; i< chunk.length / 2; i++) {
         try {
             const int16 = dataView.getInt16(i * 2, true);
-            output[i] = int16 / 32767;
+            output[i] = int16 / 32768;
         } catch (e) {
             console.error(e);
         }