Skip to content

Commit ff61aac

Browse files
committed
feat(call): coalesce fragmented transcript finals in TTY display
Chunked TTS providers like Cartesia Sonic stream each assistant utterance as 2-4 separate `transcript` events with `transcriptType: "final"`, often split mid-sentence at punctuation. Before this change, each fragment rendered on its own `🤖 Assistant:` line, so a single turn like "Hi {seller}, this is William. This call may be recorded for quality purposes. Is now a good time for a quick two-minute chat?" displayed as three separate lines. Introduce a time-based coalescing buffer keyed by role. Consecutive finals from the same speaker are appended to a `FinalBuffer` and a flush timer is reset on each arrival. Once `COALESCE_TIMEOUT_MS` (600ms) passes without a new final, the fragments join into one line and print with a single `🤖 Assistant:` / `🎤 You:` prefix. Flush points cover the natural boundaries: - Role change on a new `final` (e.g. user barges in mid-assistant response) flushes the held buffer before starting the new one. - `speech-update` with status `started` from the opposite role flushes before announcing the new speaker, so the banner doesn't appear above still-pending transcript from the last turn. - `call-ended` and WebSocket `onclose` flush before printing the ended line, so the final words match what the speaker actually said. - `cleanup` (SIGINT/SIGTERM) flushes so Ctrl+C doesn't swallow the last utterance. Refactor the `lastTranscript` + `setLastTranscript` setter pair into a single mutable `CallDisplayState` object shared between `handleControlMessage` and the outer WebSocket closure. This keeps the coalescing timer and partial-clearing state consistent in one place instead of threading multiple getters/setters. No network or agent behavior changes. Regression-safe: short single-fragment finals print as before (one fragment → flush → one line). Non-TTY output (CI logs, file redirects) still skips the live partial overwrite but now also benefits from coalesced finals.
1 parent dfd1815 commit ff61aac

1 file changed

Lines changed: 95 additions & 20 deletions

File tree

src/call.ts

Lines changed: 95 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -407,6 +407,50 @@ type ControlMessage =
407407
| CallEndedMessage
408408
| { type: string };
409409

410+
// Cartesia Sonic and other chunked TTS providers stream each final
411+
// transcript as 2-4 separate events per utterance, often split mid-sentence.
412+
// Rather than rendering each fragment on its own `🤖 Assistant:` line, we
413+
// buffer consecutive finals from the same role and flush them as a single
414+
// merged line once the speaker pauses for `COALESCE_TIMEOUT_MS`. The same
415+
// mechanism helps with user transcriber micro-pauses (Soniox, Gladia).
416+
interface FinalBuffer {
417+
role: "user" | "assistant";
418+
fragments: string[];
419+
flushTimer: NodeJS.Timeout | null;
420+
}
421+
422+
interface CallDisplayState {
423+
// The most recent partial transcript line written to stdout (so we can
424+
// erase it before writing a new line — see `clearWrittenLine`).
425+
lastTranscript: string;
426+
// Pending buffered finals, waiting to be coalesced into a single line.
427+
finalBuffer: FinalBuffer | null;
428+
}
429+
430+
// Max wait between consecutive finals from the same role before the buffer
431+
// auto-flushes. 600ms empirically captures most Cartesia chunk gaps without
432+
// introducing noticeable display latency for the developer. Tune if your
433+
// voice provider streams slower or faster.
434+
const COALESCE_TIMEOUT_MS = 600;
435+
436+
function flushFinalBuffer(state: CallDisplayState): void {
437+
const buf = state.finalBuffer;
438+
if (!buf) return;
439+
440+
if (buf.flushTimer) {
441+
clearTimeout(buf.flushTimer);
442+
}
443+
state.finalBuffer = null;
444+
445+
const merged = buf.fragments.join(" ").replace(/\s+/g, " ").trim();
446+
if (merged.length === 0) return;
447+
448+
const prefix = buf.role === "user" ? "🎤 You" : "🤖 Assistant";
449+
clearWrittenLine(process.stdout, state.lastTranscript);
450+
state.lastTranscript = "";
451+
console.log(`${prefix}: ${merged}`);
452+
}
453+
410454
async function connectWebSocket(
411455
websocketUrl: string,
412456
config: CallConfig,
@@ -431,12 +475,16 @@ async function connectWebSocket(
431475
let audioContext: ReturnType<typeof createAudioContext> | null = null;
432476
let micStream: ReturnType<typeof createMicrophoneStream> | null = null;
433477
let isConnected = false;
434-
let lastTranscript = "";
478+
const state: CallDisplayState = {
479+
lastTranscript: "",
480+
finalBuffer: null,
481+
};
435482

436483
// Graceful shutdown
437484
const cleanup = () => {
438-
clearWrittenLine(process.stdout, lastTranscript);
439-
lastTranscript = "";
485+
flushFinalBuffer(state);
486+
clearWrittenLine(process.stdout, state.lastTranscript);
487+
state.lastTranscript = "";
440488
console.log("👋 Ending call...");
441489
if (micStream) {
442490
micStream.stop();
@@ -493,9 +541,7 @@ async function connectWebSocket(
493541
// Control message (JSON)
494542
try {
495543
const message = JSON.parse(event.data as string) as ControlMessage;
496-
handleControlMessage(message, lastTranscript, (t) => {
497-
lastTranscript = t;
498-
});
544+
handleControlMessage(message, state);
499545
} catch {
500546
// Ignore parse errors
501547
}
@@ -510,8 +556,9 @@ async function connectWebSocket(
510556
};
511557

512558
ws.onclose = (event) => {
513-
clearWrittenLine(process.stdout, lastTranscript);
514-
lastTranscript = "";
559+
flushFinalBuffer(state);
560+
clearWrittenLine(process.stdout, state.lastTranscript);
561+
state.lastTranscript = "";
515562
console.log(`📴 Call ended (code: ${event.code})`);
516563
cleanup();
517564
};
@@ -575,41 +622,69 @@ function clearWrittenLine(stream: NodeJS.WriteStream, text: string): void {
575622

576623
function handleControlMessage(
577624
message: ControlMessage,
578-
lastTranscript: string,
579-
setLastTranscript: (t: string) => void,
625+
state: CallDisplayState,
580626
): void {
581627
switch (message.type) {
582628
case "transcript": {
583629
const tm = message as TranscriptMessage;
584630
const prefix = tm.role === "user" ? "🎤 You" : "🤖 Assistant";
585-
const line = `${prefix}: ${tm.transcript}`;
586631

587632
if (tm.transcriptType === "final") {
588-
clearWrittenLine(process.stdout, lastTranscript);
589-
console.log(line);
590-
setLastTranscript("");
633+
// A role change means we have a held final buffer from the other
634+
// speaker (e.g. assistant buffered fragments, user barges in and
635+
// starts emitting finals) — flush it immediately so the previous
636+
// turn prints as a complete line before the new one accumulates.
637+
if (state.finalBuffer && state.finalBuffer.role !== tm.role) {
638+
flushFinalBuffer(state);
639+
}
640+
641+
if (!state.finalBuffer) {
642+
state.finalBuffer = {
643+
role: tm.role,
644+
fragments: [],
645+
flushTimer: null,
646+
};
647+
}
648+
state.finalBuffer.fragments.push(tm.transcript);
649+
650+
if (state.finalBuffer.flushTimer) {
651+
clearTimeout(state.finalBuffer.flushTimer);
652+
}
653+
state.finalBuffer.flushTimer = setTimeout(() => {
654+
flushFinalBuffer(state);
655+
}, COALESCE_TIMEOUT_MS);
591656
} else if (process.stdout.isTTY) {
592657
// Live partial overwrite only makes sense in a TTY. In non-TTY
593658
// output (piped to a file, CI logs, etc.) every partial would
594659
// print as its own line and produce huge spam — skip them and
595660
// wait for the final.
596-
clearWrittenLine(process.stdout, lastTranscript);
661+
const line = `${prefix}: ${tm.transcript}`;
662+
clearWrittenLine(process.stdout, state.lastTranscript);
597663
process.stdout.write(line);
598-
setLastTranscript(line);
664+
state.lastTranscript = line;
599665
}
600666
break;
601667
}
602668
case "speech-update": {
603669
const sm = message as SpeechUpdateMessage;
604670
if (sm.status === "started") {
671+
// If there's a held final buffer from the other speaker, flush it
672+
// before announcing the new speaker so the banner doesn't appear
673+
// above still-pending transcript from the last turn.
674+
if (state.finalBuffer && state.finalBuffer.role !== sm.role) {
675+
flushFinalBuffer(state);
676+
}
605677
const who = sm.role === "user" ? "You" : "Assistant";
606-
clearWrittenLine(process.stdout, lastTranscript);
607-
if (lastTranscript) setLastTranscript("");
678+
clearWrittenLine(process.stdout, state.lastTranscript);
679+
state.lastTranscript = "";
608680
console.log(`💬 ${who} started speaking...`);
609681
}
610682
break;
611683
}
612684
case "call-ended": {
685+
// Flush any held fragments before the call-ended line so the last
686+
// words printed match what the speaker actually said.
687+
flushFinalBuffer(state);
613688
const cm = message as CallEndedMessage;
614689
const reasonLabels: Record<string, string> = {
615690
"silence-timed-out": "Silence timeout (no speech detected)",
@@ -624,8 +699,8 @@ function handleControlMessage(
624699
"assistant-not-found": "Assistant not found",
625700
};
626701
const label = cm.reason ? (reasonLabels[cm.reason] ?? cm.reason) : "unknown reason";
627-
clearWrittenLine(process.stdout, lastTranscript);
628-
if (lastTranscript) setLastTranscript("");
702+
clearWrittenLine(process.stdout, state.lastTranscript);
703+
state.lastTranscript = "";
629704
console.log(`📞 Call ended: ${label}`);
630705
break;
631706
}

0 commit comments

Comments
 (0)