chore: refactor agent microphone button

Author: yaroslav8765

custom/ChatSurface.vue

@@ -173,7 +173,7 @@
                   </button>
                 </div>
               </div>
-              <MicrophoneButton />
+              <MicrophoneButton v-if="props.meta.hasAudioAdapter" />
               <template v-if="!agentStore.isAudioChatMode">
                 <Button 
                   v-if="!agentStore.isResponseInProgress"
@@ -226,6 +226,7 @@ const props = defineProps<{
     }>;
     defaultModeName: string | null;
     stickByDefault: boolean;
+    hasAudioAdapter: boolean;
   }
   adminUser: any
 }>();

custom/composables/agentAudio/agent-processing.mp3

@@ -14,11 +14,34 @@ type StreamingAudioState = {
   isDone: boolean;
 };
 
+const standByAudio = new Audio('./agentAudio/agent-processing.mp3');
+
+function playStandByAudio() {
+  standByAudio.currentTime = 0;
+  standByAudio.play()
+}
+
+function stopStandByAudio() {
+  standByAudio.pause();
+  standByAudio.currentTime = 0;
+}
+
+function restartStandByAudio() {
+  standByAudio.currentTime = 0;
+  playStandByAudio();
+}
+
+standByAudio.addEventListener('ended', () => {
+  if (!standByAudio.paused) {
+    restartStandByAudio();
+  }
+});
+
 export const useAgentAudio = defineStore('agentAudio', () => {
   const agentStore = useAgentStore();
-
+  const agentAudioMode = ref<'transcribing' | 'streaming' | 'fetchingAudio' | 'playingAgentResponse' | null>(null);
   const isStreamingResponse = ref(false);
-  
+    
   let currentAbortController: AbortController | null = null;
   let isPlaying = false;
   let currentAudio: HTMLAudioElement | null = null;
@@ -28,9 +51,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
   let bufferedAudioMimeType = 'audio/mpeg';
 
   function stopGenerationAndAudio() {
-    if (isStreamingResponse.value) {
-      isStreamingResponse.value = false;
-    }
+    agentAudioMode.value = null;
     stopCurrentAudioPlayback();
     currentAbortController?.abort();
   }
@@ -45,7 +66,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     formData.append('currentPage', JSON.stringify(getCurrentPageContext()));
     const fullPath = `${import.meta.env.VITE_ADMINFORTH_PUBLIC_PATH || ''}/adminapi/v1/agent/speech-response`;
     try {
-      isStreamingResponse.value = true;
+      agentAudioMode.value = 'transcribing';
       const res = await fetch(fullPath, {
         method: 'POST',
         body: formData,
@@ -54,7 +75,9 @@ export const useAgentAudio = defineStore('agentAudio', () => {
         },
         signal: currentAbortController!.signal,
       });
+      isStreamingResponse.value = true;
       if (res.ok) {
+        agentAudioMode.value = 'streaming';
         await readSpeechResponseStream(res);
       } else {
         console.error('Failed to transcribe audio:', res.statusText);
@@ -68,6 +91,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       }
     } finally {
       isStreamingResponse.value = false;
+      agentAudioMode.value = null;
     }
   }
 
@@ -140,12 +164,16 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
 
     if (event.type === 'speech-response') {
+      // stopStandByAudio();
       agentStore.setCurrentChatStatus('ready');
       agentStore.addAgentMessage(event.data.response.text);
+      agentAudioMode.value = 'playingAgentResponse';
       return;
     }
 
     if (event.type === 'audio-start') {
+      isStreamingResponse.value = false;
+      agentAudioMode.value = 'fetchingAudio';
       initializeAudioStream(event.data.mimeType);
       return;
     }
@@ -161,6 +189,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     }
 
     if (event.type === 'data-tool-call') {
+      // playStandByAudio();
       agentStore.addDataToolCallMessage(event.data);
     }
   }
@@ -177,7 +206,7 @@ export const useAgentAudio = defineStore('agentAudio', () => {
       currentAudio.currentTime = 0;
       return;
     }
-
+    agentAudioMode.value = 'playingAgentResponse';
     void currentAudio.play().catch((error) => {
       console.error('Failed to play audio:', error);
     });
@@ -300,14 +329,18 @@ export const useAgentAudio = defineStore('agentAudio', () => {
     isPlaying = false;
   }
 
-  function stopCurrentAudioPlayback() {
+  function stopCurrentAudioPlayback(dontResetMode = false) {
     bufferedAudioChunks = [];
     bufferedAudioMimeType = 'audio/mpeg';
     detachStreamingAudio();
     destroyCurrentAudioElement();
+    if (!dontResetMode) {
+      agentAudioMode.value = null;
+    }
   }
 
   function handleAudioEnded() {
+    agentAudioMode.value = null;
     stopCurrentAudioPlayback();
   }
 
@@ -350,9 +383,10 @@ export const useAgentAudio = defineStore('agentAudio', () => {
 
   return {
     sendAudioToServerAndHandleResponse,
-    isStreamingResponse,
     stopGenerationAndAudio,
-    playBeep
+    stopCurrentAudioPlayback,
+    playBeep,
+    agentAudioMode
   };
 
 });

custom/composables/useAgentAudio.ts

@@ -7,10 +7,10 @@
     @click="toggleChatMode"
   >
     <div class="w-5 h-5 flex items-center justify-center">
-      <div v-if="!showButtonSpinner" class="flex justify-evenly items-center gap-[0.1rem]">
-        <AudioLines :showAnimation="showAnimation" :isRecording="isAudioChatMode" />
+      <div v-if="microphoneButtonMode === 'listen' || microphoneButtonMode === 'off'" class="flex justify-evenly items-center gap-[0.1rem]">
+        <AudioLines :showAnimation="showAudioWavesAnimation" :isRecording="microphoneButtonMode === 'listen'" />
       </div>
-      <div v-else-if="showStopGenerationMessage" class="flex items-center justify-center gap-2 text-white text-sm">
+      <div v-else-if="microphoneButtonMode === 'generating'" class="flex items-center justify-center gap-2 text-white text-sm">
         <span class="w-3 h-3 bg-white rounded-sm" />
         {{ $t('Stop') }}
       </div>
@@ -22,49 +22,57 @@
 
 
 <script setup lang="ts">
-import { computed, onBeforeUnmount, ref, watch } from 'vue';
+import { computed, onMounted, onBeforeUnmount, ref, watch } from 'vue';
 import debounce from 'lodash/debounce';
 import { requestMicAndStartVAD, stopUserMedia, getRecorder, CALIBRATION_DURATION } from './voiceActivityDetection';
 import { Spinner } from '@/afcl'
-import { useAdminforth } from '@/adminforth';
 import { storeToRefs } from 'pinia';
 import { useAgentStore } from '../composables/useAgentStore';
 import { useAgentAudio } from '../composables/useAgentAudio';
 import AudioLines from './AudioLines.vue';
 
-const adminforth = useAdminforth();
 const agentStore = useAgentStore();
 const agentAudio = useAgentAudio();
 const { sendAudioToServerAndHandleResponse } = agentAudio;
 const { stopGenerationAndAudio } = agentAudio;
-const { isStreamingResponse } = storeToRefs(agentAudio);
-
-agentStore.registerOnBeforeChatCloseCallback(async () => {
-  if(agentStore.isAudioChatMode) {
-    onStopRecording();
-    agentStore.setIsAudioChatMode(false);
-  }
-});
-
-const showAnimation = ref(false);
-const showButtonSpinner = ref(false);
-const showStopGenerationMessage = ref(false);
+const { stopCurrentAudioPlayback } = agentAudio;
+const { agentAudioMode } = storeToRefs(agentAudio);
+const microphoneButtonMode = ref<'off' | 'calibrating' | 'listen' | 'transcribing' | 'generating'>('off');
+const showAudioWavesAnimation = ref(false);
 const hideAnimationDebounced = debounce(() => {
-  showAnimation.value = false;
+  showAudioWavesAnimation.value = false;
 }, 100);
 const sendUserRecordDebounced = debounce(() => {
   sendRecordForTranscription();
-}, 1000);
+}, 500);
 
 const isAudioChatMode = computed(() => agentStore.isAudioChatMode);
 
-watch(isStreamingResponse, (newVal) => {
-  if(!newVal) {
-    showStopGenerationMessage.value = false;
-    showButtonSpinner.value = false;
+onMounted(() => {
+  agentStore.registerOnBeforeChatCloseCallback(async () => {
+    if(agentStore.isAudioChatMode) {
+      onStopRecording();
+      agentStore.setIsAudioChatMode(false);
+    }
+  });
+});
+
+watch(agentAudioMode, (newVal) => {
+  if(newVal === 'streaming') {
+    stopCurrentAudioPlayback(true);
+    microphoneButtonMode.value = 'generating';
+  } else if (newVal === 'transcribing') {
+    microphoneButtonMode.value = 'transcribing';
+  } else if (newVal === 'fetchingAudio') {
+    //Generation is done, waiting for audio to be ready
+  } else if (newVal === 'playingAgentResponse') {
+    // Audio is playing
   } else {
-    showButtonSpinner.value = true;
-    showStopGenerationMessage.value = true;
+    if(isAudioChatMode.value) {
+      microphoneButtonMode.value = 'listen';
+    } else {
+      microphoneButtonMode.value = 'off';
+    }
   }
 })
 
@@ -79,34 +87,31 @@ function toggleChatMode() {
 }
 
 async function onStartRecording() {
-  showButtonSpinner.value = true;
+  microphoneButtonMode.value = 'calibrating';
   await requestMicAndStartVAD(saidSomething, stopRecording, onAnySound);
   setTimeout(() => {
-    showButtonSpinner.value = false;
+    microphoneButtonMode.value = 'listen';
     agentAudio.playBeep(1000);
-    //Play a sound to indicate that recording has started
   }, CALIBRATION_DURATION);
 }
 
 function onStopRecording() {
   agentAudio.playBeep(600);
   stopUserMedia();
-  showAnimation.value = false;
-  // Play a sound to indicate that recording has stopped
+  showAudioWavesAnimation.value = false;
 }
 
 function resetAll() {
   stopGenerationAndAudio();
-  showAnimation.value = false;
-  showButtonSpinner.value = false;
-  showStopGenerationMessage.value = false;
+  microphoneButtonMode.value = 'off';
+  showAudioWavesAnimation.value = false;
   hideAnimationDebounced.cancel();
   sendUserRecordDebounced.cancel();
 }
 
 
 function saidSomething() {
-  showAnimation.value = true;
+  showAudioWavesAnimation.value = true;
   hideAnimationDebounced();
   sendUserRecordDebounced();
 }
@@ -118,23 +123,15 @@ function stopRecording() {
 
 function onAnySound(amplitude: number) {
   if(amplitude < 0.01) {
-    showAnimation.value = false;
+    showAudioWavesAnimation.value = false;
     return;
   }
-  showAnimation.value = true;
+  showAudioWavesAnimation.value = true;
   hideAnimationDebounced.cancel();
 }
 
-onBeforeUnmount(() => {
-  stopUserMedia();
-  agentStore.setIsAudioChatMode(false);
-  onStopRecording();
-  hideAnimationDebounced.cancel();
-  sendUserRecordDebounced.cancel();
-});
-
 async function sendRecordForTranscription() {
-  showAnimation.value = false;
+  showAudioWavesAnimation.value = false;
   const recordBlob = await getRecorder();
   if (recordBlob) {
     onStopRecording();
@@ -147,6 +144,13 @@ async function sendRecordForTranscription() {
   }
 }
 
+onBeforeUnmount(() => {
+  stopUserMedia();
+  agentStore.setIsAudioChatMode(false);
+  onStopRecording();
+  hideAnimationDebounced.cancel();
+  sendUserRecordDebounced.cancel();
+});
 
 
 </script>