feat(agents): expose interrupted speech data on SpeechHandle

.changeset/expose-interrupted-speech-data.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": patch
+---
+
+feat(agents): expose `generatedText` and `spokenText` on `SpeechHandle` for interrupted speeches, allowing apps to recover undelivered content

agents/src/voice/agent_activity.ts

@@ -2249,6 +2249,8 @@ export class AgentActivity implements RecognitionHooks {
         }
       }
 
+      speechHandle._setInterruptionData(llmGenData.generatedText, forwardedText);
+
       if (forwardedText) {
         hasSpeechMessage = true;
         const message = ChatMessage.create({
@@ -2716,10 +2718,14 @@ export class AgentActivity implements RecognitionHooks {
       replyAbortController.abort();
       await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
 
+      let generatedText = '';
+      let forwardedText = '';
+
       if (messageOutputs.length > 0) {
         // there should be only one message
         const [msgId, textOut, audioOut, msgModalities] = messageOutputs[0]!;
-        let forwardedText = textOut?.text || '';
+        generatedText = textOut?.text || '';
+        forwardedText = generatedText;
 
         if (audioOutput) {
           audioOutput.clearBuffer();
@@ -2768,6 +2774,8 @@ export class AgentActivity implements RecognitionHooks {
           'playout completed with interrupt',
         );
       }
+
+      speechHandle._setInterruptionData(generatedText, forwardedText);
       speechHandle._markGenerationDone();
       await executeToolsTask.cancelAndWait(AgentActivity.REPLY_TASK_CANCEL_TIMEOUT);
 

agents/src/voice/speech_handle.ts

@@ -69,6 +69,8 @@ export class SpeechHandle {
   private doneFut = new Future<void>();
   private generations: Future<void>[] = [];
   private _chatItems: ChatItem[] = [];
+  private _generatedText: string | null = null;
+  private _spokenText: string | null = null;
 
   /** @internal */
   _tasks: Task<void>[] = [];
@@ -159,6 +161,22 @@ export class SpeechHandle {
     return this._chatItems;
   }
 
+  /** Full text the model generated before the speech was interrupted. Null if not interrupted or no text was generated. */
+  get generatedText(): string | null {
+    return this._generatedText;
+  }
+
+  /** Text that was actually spoken (heard by the listener) before interruption. Null if not interrupted or nothing was spoken. */
+  get spokenText(): string | null {
+    return this._spokenText;
+  }
+
+  /** @internal */
+  _setInterruptionData(generatedText: string, spokenText: string): void {
+    this._generatedText = generatedText || null;
+    this._spokenText = spokenText || null;
+  }
+
   /**
    * Interrupt the current speech generation.
    *