From 84a259fed482e2d08df1aa95adef6cacc0d739bf Mon Sep 17 00:00:00 2001 From: enrique Date: Mon, 29 Jun 2026 17:17:05 -0400 Subject: [PATCH 1/3] fix(voice): resume speech interrupted before its first frame (#1909) Port of livekit/agents#5039 (Python issue #5038) to agents-js. When the agent is in the "thinking" state and the user makes a brief sound before the first TTS audio frame is forwarded, `onStartOfSpeech` pauses the not-yet-playing speech. That thinking-state pause is intentional and is preserved. The frames are still captured into the paused output buffer, but `forwardAudio`'s finally block rejected `firstFrameFut` (and removed its PLAYBACK_STARTED listener) whenever no frame had played yet. When a false interruption then cleared and the output resumed, the buffered first frame played but nothing was listening, so the future stayed rejected. Because the reply tasks gate transcript preservation on `firstFrameFut.done && !firstFrameFut.rejected`, the resumed turn was dropped from history even though audio reached the user. Fix: - Move the PLAYBACK_STARTED listener from `forwardAudio` to `performAudioForwarding` so it outlives the forwarding task; a late first frame (e.g. after a `resumeFalseInterruption` resume) can still resolve `firstFrameFut`. - Stop rejecting `firstFrameFut` in `forwardAudio`'s finally. - Settle the future in the reply tasks (say / pipeline / realtime) after playout finishes or is interrupted, which also removes the listener. JS note: unlike Python, the JS `Future` has no `cancel()` distinct from `reject()` (reject sets `rejected = true`), so the fix preserves audio on the no-first-frame path by relocating resolution rather than relying on a cancel/reject distinction in the downstream gate. Adds a regression test reproducing the thinking-state pause before the first frame for both a false interruption (resumes and plays, transcript preserved) and a genuine interruption after a resume (partial transcript kept, turn not lost). Both fail on main and pass with this change. --- .changeset/interrupt-before-first-frame.md | 9 + agents/src/voice/agent_activity.ts | 27 ++- agents/src/voice/generation.ts | 61 ++++-- ...ation_interrupt_before_first_frame.test.ts | 192 ++++++++++++++++++ .../src/voice/generation_tts_timeout.test.ts | 8 +- 5 files changed, 271 insertions(+), 26 deletions(-) create mode 100644 .changeset/interrupt-before-first-frame.md create mode 100644 agents/src/voice/generation_interrupt_before_first_frame.test.ts diff --git a/.changeset/interrupt-before-first-frame.md b/.changeset/interrupt-before-first-frame.md new file mode 100644 index 000000000..f280fefac --- /dev/null +++ b/.changeset/interrupt-before-first-frame.md @@ -0,0 +1,9 @@ +--- +'@livekit/agents': patch +--- + +Fix agent speech being silently dropped when interrupted before its first audio frame plays (#1909, port of livekit/agents#5039). + +When the agent is in the "thinking" state and the user makes a brief sound before the first TTS frame is forwarded, `onStartOfSpeech` pauses the not-yet-playing speech (this thinking-state pause is intentional and preserved). The frames were still captured into the paused output buffer, but `forwardAudio`'s `finally` block rejected `firstFrameFut` (and removed its `PLAYBACK_STARTED` listener) whenever no frame had played yet. So when a false interruption cleared and the output resumed, the buffered first frame played but nothing was listening — the future stayed rejected, and because the reply tasks gate transcript preservation on `firstFrameFut.done && !firstFrameFut.rejected`, the resumed turn was dropped from history even though audio reached the user. + +The `PLAYBACK_STARTED` listener now lives in `performAudioForwarding` so it outlives the forwarding task, and `forwardAudio` no longer rejects the future; a late first frame (e.g. after a `resumeFalseInterruption` resume) can still resolve it. The reply tasks settle the future after playout finishes or is interrupted to remove the listener. A genuine interruption after a resume now keeps its partial synchronized transcript instead of losing the turn. diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index b93e5cce9..fd58e7615 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -2394,6 +2394,7 @@ export class AgentActivity implements RecognitionHooks { } }; + let audioOut: _AudioOut | null = null; if (!audioOutput) { if (textOut) { textOut.firstTextFut.await @@ -2401,7 +2402,6 @@ export class AgentActivity implements RecognitionHooks { .catch(() => this.logger.debug('firstTextFut cancelled before first frame')); } } else { - let audioOut: _AudioOut | null = null; if (!audio) { // generate audio using TTS const [ttsTask, ttsGenData] = performTTSInference( @@ -2490,6 +2490,23 @@ export class AgentActivity implements RecognitionHooks { } this.restoreInterruptionByAudioActivity(); } + + // Settle firstFrameFut to remove its PLAYBACK_STARTED listener now that playout is + // done/interrupted. forwardAudio no longer rejects it (#1909). + this.settleFirstFrameFut(audioOut); + } + + /** + * Reject `firstFrameFut` if it is still pending, to drop the PLAYBACK_STARTED listener + * registered in `performAudioForwarding`. Called by reply tasks once playout has + * finished or been interrupted. `forwardAudio` no longer rejects the future itself so + * that a frame which plays late (e.g. after a false-interruption resume) can still + * resolve it; see livekit/agents-js#1909 (port of livekit/agents#5039). + */ + private settleFirstFrameFut(audioOut: _AudioOut | null | undefined): void { + if (audioOut && !audioOut.firstFrameFut.done) { + audioOut.firstFrameFut.reject(new Error('audio forwarding finished before first frame')); + } } private _pipelineReplyTaskImpl = async ({ @@ -2832,6 +2849,11 @@ export class AgentActivity implements RecognitionHooks { } finally { replyAbortController.signal.removeEventListener('abort', abortSegment); await cancelAndWait(forwardTasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); + // Settle firstFrameFut once playout is done/interrupted so its PLAYBACK_STARTED + // listener is removed. forwardAudio no longer rejects it (#1909): a frame that + // played after a false-interruption resume must keep the synchronized transcript + // (gated on `firstFrameFut.done && !rejected` above) instead of being dropped. + this.settleFirstFrameFut(output.audioOut); } }; @@ -3389,6 +3411,9 @@ export class AgentActivity implements RecognitionHooks { } finally { abortController.signal.removeEventListener('abort', abortMessage); await cancelAndWait(forwardTasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); + // See note in the pipeline reply task: settle firstFrameFut to remove its + // PLAYBACK_STARTED listener now that playout is done/interrupted (#1909). + this.settleFirstFrameFut(output.audioOut); } }; diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 6cf12d6ea..16937986d 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -829,6 +829,13 @@ export interface _AudioOut { * metric. */ startedForwardingAt?: number; + /** + * Whether this segment has captured at least one of its own frames. Read by the + * PLAYBACK_STARTED listener registered in `performAudioForwarding` to ignore a + * stray event emitted by another overlapping segment before this one has played. + * @internal + */ + _hasCapturedOwnFrame?: boolean; } async function forwardAudio( @@ -842,20 +849,7 @@ async function forwardAudio( const reader = ttsStream.getReader(); let resampler: AudioResampler | null = null; - // The audio output is shared across overlapping segments, so ignore a - // PLAYBACK_STARTED from another segment until we capture our own first frame. - // Resolving `firstFrameFut` early skips resampler creation and pushes an - // unresampled frame (`RtcError: sample_rate and num_channels don't match`). - let hasCapturedOwnFrame = false; - - const onPlaybackStarted = (ev: { createdAt: number }) => { - if (hasCapturedOwnFrame && !out.firstFrameFut.done) { - out.firstFrameFut.resolve(ev.createdAt); - } - }; - try { - audioOutput.on(AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted); audioOutput.resume(); while (true) { @@ -876,8 +870,9 @@ async function forwardAudio( } // Mark before capturing so the PLAYBACK_STARTED emitted synchronously inside - // the first captureFrame is attributed to this segment. - hasCapturedOwnFrame = true; + // the first captureFrame is attributed to this segment (see the listener in + // `performAudioForwarding`). + out._hasCapturedOwnFrame = true; if (resampler) { for (const f of resampler.push(frame)) { @@ -900,12 +895,15 @@ async function forwardAudio( throw e; } } finally { - audioOutput.off(AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted); - - if (!out.firstFrameFut.done) { - out.firstFrameFut.reject(new Error('audio forwarding cancelled before playback started')); - } - + // NOTE: `firstFrameFut` is intentionally NOT rejected here. The frames may have + // been captured into a paused output (agent interrupted in the thinking state + // before its first frame played); playback then starts only once the output is + // resumed — possibly after this forwarding task has already finished. The + // PLAYBACK_STARTED listener registered in `performAudioForwarding` outlives this + // task and resolves the future when that late first frame finally plays, so a + // false interruption resumes and plays instead of dropping the turn. The caller + // settles the future (and removes the listener) after playout finishes or is + // interrupted. See livekit/agents-js#1909 (port of livekit/agents#5039). reader?.releaseLock(); audioOutput.flush(); if (signal?.aborted) { @@ -926,6 +924,27 @@ export function performAudioForwarding( firstFrameFut: new Future(), }; + // Resolve `firstFrameFut` from the output's PLAYBACK_STARTED event. Registered here + // (rather than inside `forwardAudio`) so the listener outlives the forwarding task: + // when a not-yet-playing speech is paused in the thinking state its frames are + // buffered and the forwarding task may finish before playback ever starts. The late + // first frame — e.g. once a false interruption clears and the output resumes — must + // still resolve the future instead of being dropped (livekit/agents-js#1909). + const onPlaybackStarted = (ev: { createdAt: number }) => { + // Ignore a PLAYBACK_STARTED from another overlapping segment until this segment + // has captured its own first frame; resolving early would skip resampler creation. + if (out._hasCapturedOwnFrame && !out.firstFrameFut.done) { + out.firstFrameFut.resolve(ev.createdAt); + } + }; + audioOutput.on(AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted); + // Remove the listener once the future settles — resolved by playback above, or + // rejected by the caller (via `firstFrameFut.reject`) after playout finishes or is + // interrupted without a frame ever playing. + const removeListener = () => + audioOutput.off(AudioOutput.EVENT_PLAYBACK_STARTED, onPlaybackStarted); + out.firstFrameFut.await.then(removeListener).catch(removeListener); + return [ Task.from( (controller) => forwardAudio(ttsStream, audioOutput, out, idleTimeout, controller.signal), diff --git a/agents/src/voice/generation_interrupt_before_first_frame.test.ts b/agents/src/voice/generation_interrupt_before_first_frame.test.ts new file mode 100644 index 000000000..202eaa600 --- /dev/null +++ b/agents/src/voice/generation_interrupt_before_first_frame.test.ts @@ -0,0 +1,192 @@ +// SPDX-FileCopyrightText: 2026 LiveKit, Inc. +// +// SPDX-License-Identifier: Apache-2.0 + +/** + * Regression tests for livekit/agents-js#1909 (JS port of livekit/agents#5038/#5039). + * + * When the agent is in the "thinking" state and the user makes a brief sound + * *before the first TTS audio frame has played*, `AgentActivity.onStartOfSpeech` + * pauses the (not-yet-playing) speech. The TTS frames are still captured into the + * paused output buffer and the forwarding task finishes — but playback has not + * started yet, so `firstFrameFut` is still unresolved when the task ends. + * + * The bug: `forwardAudio` used to *reject* `firstFrameFut` (and remove its + * PLAYBACK_STARTED listener) in its `finally` block whenever no frame had played. + * So when the false interruption cleared and the output resumed, the buffered first + * frame played but nothing was listening — `firstFrameFut` stayed rejected forever. + * The reply task gates transcript preservation on + * `firstFrameFut.done && !firstFrameFut.rejected`, so the resumed turn was dropped + * from history even though audio reached the user. + * + * The fix moves the PLAYBACK_STARTED listener to `performAudioForwarding` (so it + * outlives the forwarding task) and stops rejecting the future in `forwardAudio`; + * the caller settles it after playout finishes/interrupts. A late first frame can + * therefore still resolve the future and keep the turn. + * + * NOTE: in the JS `Future`, "cancel" is `reject` (it sets `rejected = true`), so a + * literal transcription of #5039 would not have changed the `!rejected` gate — these + * tests assert on `rejected`/`done` directly to lock in the corrected behavior. + */ +import { AudioFrame } from '@livekit/rtc-node'; +import { ReadableStream } from 'stream/web'; +import { describe, expect, it } from 'vitest'; +import { initializeLogger } from '../log.js'; +import type { _AudioOut } from './generation.js'; +import { performAudioForwarding } from './generation.js'; +import { AudioOutput } from './io.js'; + +function createSilentFrame(sampleRate = 24000, channels = 1, durationMs = 20): AudioFrame { + const samplesPerChannel = Math.floor((sampleRate * durationMs) / 1000); + const data = new Int16Array(samplesPerChannel * channels); + return new AudioFrame(data, sampleRate, channels, samplesPerChannel); +} + +/** + * Mock output that models a real audio sink's pause semantics: while paused, frames + * are buffered but PLAYBACK_STARTED is NOT emitted; it fires only once the first + * frame actually plays (on the first un-paused capture, or when a paused buffer is + * resumed). `clearBuffer` drops buffered frames (a real interruption). + */ +class PausableMockAudioOutput extends AudioOutput { + capturedFrames: AudioFrame[] = []; + clearBufferCalls = 0; + private paused = false; + private startedPlaying = false; + + constructor() { + super(24000, undefined, { pause: true }); + } + + async captureFrame(frame: AudioFrame): Promise { + await super.captureFrame(frame); + this.capturedFrames.push(frame); + if (!this.paused) { + this.maybeStartPlayback(); + } + } + + pause(): void { + this.paused = true; + } + + resume(): void { + const wasPaused = this.paused; + this.paused = false; + // On resume, a buffered (but never-played) first frame begins playing. + if (wasPaused && this.capturedFrames.length > 0) { + this.maybeStartPlayback(); + } + } + + clearBuffer(): void { + this.clearBufferCalls++; + this.capturedFrames = []; + } + + private maybeStartPlayback(): void { + if (!this.startedPlaying) { + this.startedPlaying = true; + this.onPlaybackStarted(Date.now()); + } + } +} + +/** + * Mirrors the gate used by the reply tasks in `agent_activity.ts` to decide whether + * the synchronized transcript (i.e. the turn) is preserved after an interruption. + */ +function wouldPreserveTranscript(audioOut: _AudioOut): boolean { + return audioOut.firstFrameFut.done && !audioOut.firstFrameFut.rejected; +} + +describe('interruption before the first audio frame (#1909)', () => { + initializeLogger({ pretty: false, level: 'silent' }); + + it('resumes and plays a speech paused before its first frame (false interruption)', async () => { + // A controlled stream so we can pause the output *after* forwardAudio has + // started (and called its initial resume()) but before any frame is read. + let enqueue!: (frame: AudioFrame) => void; + let close!: () => void; + const stream = new ReadableStream({ + start(controller) { + enqueue = (frame) => controller.enqueue(frame); + close = () => controller.close(); + }, + }); + + const audioOutput = new PausableMockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + + // User barges in during the thinking state, before the first frame has played. + audioOutput.pause(); + + // TTS frames arrive and are captured into the paused buffer; the forwarding + // task then finishes (TTS stream closed) while playback has not started. + enqueue(createSilentFrame()); + enqueue(createSilentFrame()); + close(); + await task.result; + + expect(audioOutput.capturedFrames.length).toBe(2); + // The discriminating assertion: on `main`, forwardAudio's finally rejected the + // future here, dropping the turn. The fix leaves it pending until playback. + expect(audioOut.firstFrameFut.rejected).toBe(false); + expect(audioOut.firstFrameFut.done).toBe(false); + + // The false interruption clears: the output resumes and the buffered first + // frame finally plays. + audioOutput.resume(); + const startedAt = await audioOut.firstFrameFut.await; + + expect(typeof startedAt).toBe('number'); + expect(audioOut.firstFrameFut.done).toBe(true); + expect(audioOut.firstFrameFut.rejected).toBe(false); + // The reply task would keep the (full) turn instead of blanking it. + expect(wouldPreserveTranscript(audioOut)).toBe(true); + }); + + it('keeps the partial transcript on a genuine interruption after a resume', async () => { + let enqueue!: (frame: AudioFrame) => void; + let close!: () => void; + const stream = new ReadableStream({ + start(controller) { + enqueue = (frame) => controller.enqueue(frame); + close = () => controller.close(); + }, + }); + + const audioOutput = new PausableMockAudioOutput(); + const controller = new AbortController(); + const [task, audioOut] = performAudioForwarding(stream, audioOutput, controller); + + // Paused in the thinking state before the first frame; frames buffer. + audioOutput.pause(); + enqueue(createSilentFrame()); + enqueue(createSilentFrame()); + close(); + await task.result; + + expect(audioOut.firstFrameFut.rejected).toBe(false); + + // The pause clears and playback starts (firstFrameFut resolves)... + audioOutput.resume(); + await audioOut.firstFrameFut.await; + + // ...then the user genuinely interrupts mid-playback: the reply task clears the + // buffer and evaluates its gate. + audioOutput.clearBuffer(); + expect(audioOutput.clearBufferCalls).toBe(1); + + // The turn is NOT silently lost: a frame did play, so the gate preserves the + // synchronized (partial) transcript. On `main` the future was already rejected + // during the paused finish, so this gate would be false and the turn dropped. + expect(wouldPreserveTranscript(audioOut)).toBe(true); + + // Caller-side cleanup (mirrors AgentActivity.settleFirstFrameFut) is a no-op + // here because the future already resolved. + expect(audioOut.firstFrameFut.done).toBe(true); + expect(audioOut.firstFrameFut.rejected).toBe(false); + }); +}); diff --git a/agents/src/voice/generation_tts_timeout.test.ts b/agents/src/voice/generation_tts_timeout.test.ts index cbe636ebe..1237f42ab 100644 --- a/agents/src/voice/generation_tts_timeout.test.ts +++ b/agents/src/voice/generation_tts_timeout.test.ts @@ -115,9 +115,6 @@ describe('TTS stream idle timeout', () => { const controller = new AbortController(); const [task, audioOut] = performAudioForwarding(stalledStream, audioOutput, controller, 500); - // Reject path is expected (no first frame ever captured). - audioOut.firstFrameFut.await.catch(() => {}); - vi.useFakeTimers(); audioOutput.onPlaybackStarted(Date.now()); @@ -130,7 +127,10 @@ describe('TTS stream idle timeout', () => { vi.useRealTimers(); expect(audioOutput.capturedFrames.length).toBe(0); - expect(audioOut.firstFrameFut.rejected).toBe(true); + // forwardAudio no longer rejects firstFrameFut on the no-first-frame path + // (#1909); the stray event was ignored, so the future is simply still pending. + // The caller settles it after playout finishes/interrupts. + expect(audioOut.firstFrameFut.done).toBe(false); }); it('resamples a rate-mismatched frame even after a stray PLAYBACK_STARTED', async () => { From 56a30efafb1f7de785d7e757dd7ccc665f2834a8 Mon Sep 17 00:00:00 2001 From: enrique Date: Mon, 29 Jun 2026 17:27:44 -0400 Subject: [PATCH 2/3] fix(voice): settle firstFrameFut in finally in ttsTask (#1909) Wrap the post-forwarding body of ttsTask in try/finally so settleFirstFrameFut runs even if an await throws between the performAudioForwarding call and the end of the method. Otherwise the PLAYBACK_STARTED listener registered in performAudioForwarding would leak on the shared audioOutput EventEmitter on the exception path. Mirrors the finally-block pattern already used in forwardSegment and processOneMessage. --- agents/src/voice/agent_activity.ts | 86 ++++++++++++++++-------------- 1 file changed, 45 insertions(+), 41 deletions(-) diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index fd58e7615..56d0e89ad 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -2442,58 +2442,62 @@ export class AgentActivity implements RecognitionHooks { .catch(() => this.logger.debug('firstFrameFut cancelled before first frame')); } - await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result)); - - if (audioOutput) { - await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]); - } + try { + await speechHandle.waitIfNotInterrupted(tasks.map((task) => task.result)); - if (speechHandle.interrupted) { - replyAbortController.abort(); - await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); if (audioOutput) { - audioOutput.clearBuffer(); - await audioOutput.waitForPlayout(); + await speechHandle.waitIfNotInterrupted([audioOutput.waitForPlayout()]); } - } - if (addToChatCtx) { - const replyStoppedSpeakingAt = Date.now(); - const replyAssistantMetrics: MetricsReport = {}; - if (replyTtsGenData?.ttfb !== undefined) { - replyAssistantMetrics.ttsNodeTtfb = replyTtsGenData.ttfb; + if (speechHandle.interrupted) { + replyAbortController.abort(); + await cancelAndWait(tasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); + if (audioOutput) { + audioOutput.clearBuffer(); + await audioOutput.waitForPlayout(); + } } - if (replyStartedSpeakingAt !== undefined) { - replyAssistantMetrics.startedSpeakingAt = replyStartedSpeakingAt / 1000; // ms -> seconds - replyAssistantMetrics.stoppedSpeakingAt = replyStoppedSpeakingAt / 1000; // ms -> seconds - if (replyStartedForwardingAt !== undefined) { - replyAssistantMetrics.playbackLatency = - (replyStartedSpeakingAt - replyStartedForwardingAt) / 1000; // ms -> seconds + if (addToChatCtx) { + const replyStoppedSpeakingAt = Date.now(); + const replyAssistantMetrics: MetricsReport = {}; + if (replyTtsGenData?.ttfb !== undefined) { + replyAssistantMetrics.ttsNodeTtfb = replyTtsGenData.ttfb; } - } + if (replyStartedSpeakingAt !== undefined) { + replyAssistantMetrics.startedSpeakingAt = replyStartedSpeakingAt / 1000; // ms -> seconds + replyAssistantMetrics.stoppedSpeakingAt = replyStoppedSpeakingAt / 1000; // ms -> seconds - const message = ChatMessage.create({ - role: 'assistant', - content: textOut?.text || '', - interrupted: speechHandle.interrupted, - metrics: replyAssistantMetrics, - }); - this.agent._chatCtx.insert(message); - this.agentSession._conversationItemAdded(message); - } + if (replyStartedForwardingAt !== undefined) { + replyAssistantMetrics.playbackLatency = + (replyStartedSpeakingAt - replyStartedForwardingAt) / 1000; // ms -> seconds + } + } - if (this.agentSession.agentState === 'speaking') { - this.agentSession._updateAgentState('listening'); - if (this.audioRecognition) { - this.audioRecognition.onEndOfAgentSpeech(Date.now()); + const message = ChatMessage.create({ + role: 'assistant', + content: textOut?.text || '', + interrupted: speechHandle.interrupted, + metrics: replyAssistantMetrics, + }); + this.agent._chatCtx.insert(message); + this.agentSession._conversationItemAdded(message); } - this.restoreInterruptionByAudioActivity(); - } - // Settle firstFrameFut to remove its PLAYBACK_STARTED listener now that playout is - // done/interrupted. forwardAudio no longer rejects it (#1909). - this.settleFirstFrameFut(audioOut); + if (this.agentSession.agentState === 'speaking') { + this.agentSession._updateAgentState('listening'); + if (this.audioRecognition) { + this.audioRecognition.onEndOfAgentSpeech(Date.now()); + } + this.restoreInterruptionByAudioActivity(); + } + } finally { + // Settle firstFrameFut to remove its PLAYBACK_STARTED listener now that playout is + // done/interrupted. In a finally so the listener is dropped even if an await above + // throws — otherwise it would leak on the shared audioOutput. forwardAudio no + // longer rejects firstFrameFut itself (#1909). + this.settleFirstFrameFut(audioOut); + } } /** From 2e087f7d01417427d37b6821bad2f9e1e207540c Mon Sep 17 00:00:00 2001 From: enrique Date: Mon, 29 Jun 2026 17:30:25 -0400 Subject: [PATCH 3/3] docs(voice): trim issue-reference comments to codebase style (#1909) Use the bare #1909 short form referenced once at the core fix points, matching existing comments (e.g. #1662, #1430, #1124), instead of the verbose cross-repo 'livekit/agents-js#1909 (port of livekit/agents#5039)' form and the per-call-site repetition. The port context lives in the commit/PR/changeset. --- agents/src/voice/agent_activity.ts | 17 ++++++----------- agents/src/voice/generation.ts | 4 ++-- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/agents/src/voice/agent_activity.ts b/agents/src/voice/agent_activity.ts index 56d0e89ad..d3c425e59 100644 --- a/agents/src/voice/agent_activity.ts +++ b/agents/src/voice/agent_activity.ts @@ -2492,10 +2492,8 @@ export class AgentActivity implements RecognitionHooks { this.restoreInterruptionByAudioActivity(); } } finally { - // Settle firstFrameFut to remove its PLAYBACK_STARTED listener now that playout is - // done/interrupted. In a finally so the listener is dropped even if an await above - // throws — otherwise it would leak on the shared audioOutput. forwardAudio no - // longer rejects firstFrameFut itself (#1909). + // In a finally so the listener is dropped even if an await above throws — + // otherwise it would leak on the shared audioOutput. this.settleFirstFrameFut(audioOut); } } @@ -2505,7 +2503,7 @@ export class AgentActivity implements RecognitionHooks { * registered in `performAudioForwarding`. Called by reply tasks once playout has * finished or been interrupted. `forwardAudio` no longer rejects the future itself so * that a frame which plays late (e.g. after a false-interruption resume) can still - * resolve it; see livekit/agents-js#1909 (port of livekit/agents#5039). + * resolve it (#1909). */ private settleFirstFrameFut(audioOut: _AudioOut | null | undefined): void { if (audioOut && !audioOut.firstFrameFut.done) { @@ -2853,10 +2851,9 @@ export class AgentActivity implements RecognitionHooks { } finally { replyAbortController.signal.removeEventListener('abort', abortSegment); await cancelAndWait(forwardTasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); - // Settle firstFrameFut once playout is done/interrupted so its PLAYBACK_STARTED - // listener is removed. forwardAudio no longer rejects it (#1909): a frame that - // played after a false-interruption resume must keep the synchronized transcript - // (gated on `firstFrameFut.done && !rejected` above) instead of being dropped. + // A frame that played after a false-interruption resume keeps the synchronized + // transcript (gated on `firstFrameFut.done && !rejected` above) instead of being + // dropped. this.settleFirstFrameFut(output.audioOut); } }; @@ -3415,8 +3412,6 @@ export class AgentActivity implements RecognitionHooks { } finally { abortController.signal.removeEventListener('abort', abortMessage); await cancelAndWait(forwardTasks, AgentActivity.REPLY_TASK_CANCEL_TIMEOUT); - // See note in the pipeline reply task: settle firstFrameFut to remove its - // PLAYBACK_STARTED listener now that playout is done/interrupted (#1909). this.settleFirstFrameFut(output.audioOut); } }; diff --git a/agents/src/voice/generation.ts b/agents/src/voice/generation.ts index 16937986d..79b2262dc 100644 --- a/agents/src/voice/generation.ts +++ b/agents/src/voice/generation.ts @@ -903,7 +903,7 @@ async function forwardAudio( // task and resolves the future when that late first frame finally plays, so a // false interruption resumes and plays instead of dropping the turn. The caller // settles the future (and removes the listener) after playout finishes or is - // interrupted. See livekit/agents-js#1909 (port of livekit/agents#5039). + // interrupted. See #1909. reader?.releaseLock(); audioOutput.flush(); if (signal?.aborted) { @@ -929,7 +929,7 @@ export function performAudioForwarding( // when a not-yet-playing speech is paused in the thinking state its frames are // buffered and the forwarding task may finish before playback ever starts. The late // first frame — e.g. once a false interruption clears and the output resumes — must - // still resolve the future instead of being dropped (livekit/agents-js#1909). + // still resolve the future instead of being dropped (#1909). const onPlaybackStarted = (ev: { createdAt: number }) => { // Ignore a PLAYBACK_STARTED from another overlapping segment until this segment // has captured its own first frame; resolving early would skip resampler creation.