feat(inference/tts): port aligned transcript / output_timestamps from Python (#5534) (#1311)

Co-authored-by: Claude <noreply@anthropic.com>

Author: toubatbrian

.changeset/inference-tts-aligned-timestamps.md

@@ -0,0 +1,5 @@
+---
+"@livekit/agents": minor
+---
+
+feat(inference/tts): detect aligned transcript capability from provider `modelOptions` (`cartesia.add_timestamps`, `elevenlabs.sync_alignment`, `inworld.timestamp_type`) and forward the gateway's `output_timestamps` WebSocket events as `TimedString` word/character timings attached to the next synthesized audio frame. Ported from livekit/agents#5534.

agents/src/inference/api_protos.test.ts

@@ -2,7 +2,7 @@
 //
 // SPDX-License-Identifier: Apache-2.0
 import { describe, expect, it } from 'vitest';
-import { sttServerEventSchema } from './api_protos.js';
+import { sttServerEventSchema, ttsServerEventSchema } from './api_protos.js';
 
 describe('sttServerEventSchema', () => {
   it('accepts numeric error codes from STT server events', () => {
@@ -15,3 +15,53 @@ describe('sttServerEventSchema', () => {
     expect(result.success).toBe(true);
   });
 });
+
+describe('ttsServerEventSchema', () => {
+  it('extracts output_timestamps words payload', () => {
+    const result = ttsServerEventSchema.safeParse({
+      type: 'output_timestamps',
+      session_id: 's1',
+      words: [
+        { word: 'hello', start: 0.1, end: 0.4 },
+        { word: 'world', start: 0.4, end: 0.8 },
+      ],
+    });
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.type).toBe('output_timestamps');
+      expect(result.data.session_id).toBe('s1');
+      expect(result.data.words?.map((w) => w.word)).toEqual(['hello', 'world']);
+      expect(result.data.chars).toBeUndefined();
+    }
+  });
+
+  it('extracts output_timestamps chars payload', () => {
+    const result = ttsServerEventSchema.safeParse({
+      type: 'output_timestamps',
+      session_id: 's2',
+      chars: [
+        { char: 'h', start: 0.1, end: 0.2 },
+        { char: 'i', start: 0.2, end: 0.3 },
+      ],
+    });
+
+    expect(result.success).toBe(true);
+    if (result.success) {
+      expect(result.data.type).toBe('output_timestamps');
+      expect(result.data.session_id).toBe('s2');
+      expect(result.data.chars?.map((c) => c.char)).toEqual(['h', 'i']);
+      expect(result.data.words).toBeUndefined();
+    }
+  });
+
+  it('rejects malformed output_timestamps entries', () => {
+    const result = ttsServerEventSchema.safeParse({
+      type: 'output_timestamps',
+      session_id: 's3',
+      words: [{ word: 'oops', start: 'bad', end: 0.2 }],
+    });
+
+    expect(result.success).toBe(false);
+  });
+});

agents/src/inference/api_protos.ts

@@ -54,6 +54,25 @@ export const ttsErrorEventSchema = z.object({
   session_id: z.string().optional(),
 });
 
+export const ttsWordTimestampSchema = z.object({
+  word: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+export const ttsCharTimestampSchema = z.object({
+  char: z.string(),
+  start: z.number(),
+  end: z.number(),
+});
+
+export const ttsOutputTimestampsEventSchema = z.object({
+  type: z.literal('output_timestamps'),
+  session_id: z.string().optional(),
+  words: z.array(ttsWordTimestampSchema).optional(),
+  chars: z.array(ttsCharTimestampSchema).optional(),
+});
+
 export const ttsClientEventSchema = z.discriminatedUnion('type', [
   ttsSessionCreateEventSchema,
   ttsInputTranscriptEventSchema,
@@ -64,6 +83,7 @@ export const ttsClientEventSchema = z.discriminatedUnion('type', [
 export const ttsServerEventSchema = z.discriminatedUnion('type', [
   ttsSessionCreatedEventSchema,
   ttsOutputAudioEventSchema,
+  ttsOutputTimestampsEventSchema,
   ttsDoneEventSchema,
   ttsSessionClosedEventSchema,
   ttsErrorEventSchema,
@@ -75,6 +95,9 @@ export type TtsSessionFlushEvent = z.infer<typeof ttsSessionFlushEventSchema>;
 export type TtsSessionCloseEvent = z.infer<typeof ttsSessionCloseEventSchema>;
 export type TtsSessionCreatedEvent = z.infer<typeof ttsSessionCreatedEventSchema>;
 export type TtsOutputAudioEvent = z.infer<typeof ttsOutputAudioEventSchema>;
+export type TtsWordTimestamp = z.infer<typeof ttsWordTimestampSchema>;
+export type TtsCharTimestamp = z.infer<typeof ttsCharTimestampSchema>;
+export type TtsOutputTimestampsEvent = z.infer<typeof ttsOutputTimestampsEventSchema>;
 export type TtsDoneEvent = z.infer<typeof ttsDoneEventSchema>;
 export type TtsSessionClosedEvent = z.infer<typeof ttsSessionClosedEventSchema>;
 export type TtsErrorEvent = z.infer<typeof ttsErrorEventSchema>;

agents/src/inference/tts.test.ts

@@ -1,11 +1,17 @@
 // SPDX-FileCopyrightText: 2025 LiveKit, Inc.
 //
 // SPDX-License-Identifier: Apache-2.0
-import { beforeAll, describe, expect, it } from 'vitest';
+import { beforeAll, describe, expect, it, vi } from 'vitest';
 import { normalizeLanguage } from '../language.js';
 import { initializeLogger } from '../log.js';
 import { type APIConnectOptions, DEFAULT_API_CONNECT_OPTIONS } from '../types.js';
-import { TTS, type TTSFallbackModel, normalizeTTSFallback, parseTTSModelString } from './tts.js';
+import {
+  TTS,
+  type TTSFallbackModel,
+  hasAlignedTranscript,
+  normalizeTTSFallback,
+  parseTTSModelString,
+} from './tts.js';
 
 beforeAll(() => {
   initializeLogger({ level: 'silent', pretty: false });
@@ -352,3 +358,114 @@ describe('TTS provider modelOptions parity', () => {
     expect(tts['opts'].modelOptions).toEqual(modelOptions);
   });
 });
+
+describe('hasAlignedTranscript', () => {
+  it('returns false for unknown provider', () => {
+    expect(hasAlignedTranscript('rime/mistv2', { add_timestamps: true })).toBe(false);
+    expect(hasAlignedTranscript('deepgram/aura-2', { sync_alignment: true })).toBe(false);
+  });
+
+  it('returns false for an empty options payload', () => {
+    expect(hasAlignedTranscript('cartesia/sonic', {})).toBe(false);
+    expect(hasAlignedTranscript('elevenlabs/eleven_flash_v2', undefined)).toBe(false);
+    expect(hasAlignedTranscript(undefined, { add_timestamps: true })).toBe(false);
+  });
+
+  it('detects Cartesia add_timestamps opt-in', () => {
+    expect(hasAlignedTranscript('cartesia/sonic', { add_timestamps: true })).toBe(true);
+    expect(hasAlignedTranscript('cartesia/sonic-3', { add_timestamps: false })).toBe(false);
+  });
+
+  it('detects ElevenLabs sync_alignment opt-in', () => {
+    expect(hasAlignedTranscript('elevenlabs/eleven_flash_v2', { sync_alignment: true })).toBe(true);
+    expect(
+      hasAlignedTranscript('elevenlabs/eleven_multilingual_v2', { sync_alignment: false }),
+    ).toBe(false);
+  });
+
+  it('detects Inworld WORD/CHARACTER timestamp types', () => {
+    expect(hasAlignedTranscript('inworld/inworld-tts-1', { timestamp_type: 'WORD' })).toBe(true);
+    expect(hasAlignedTranscript('inworld/inworld-tts-1', { timestamp_type: 'CHARACTER' })).toBe(
+      true,
+    );
+    expect(
+      hasAlignedTranscript('inworld/inworld-tts-1', {
+        timestamp_type: 'TIMESTAMP_TYPE_UNSPECIFIED',
+      }),
+    ).toBe(false);
+  });
+});
+
+describe('TTS alignedTranscript capability', () => {
+  it('defaults to alignedTranscript=false when no opt-in is provided', () => {
+    const tts = makeTts();
+    expect(tts.capabilities.alignedTranscript).toBe(false);
+  });
+
+  it('reports alignedTranscript=true when Cartesia add_timestamps is set', () => {
+    const tts = makeTts({
+      model: 'cartesia/sonic',
+      modelOptions: { add_timestamps: true },
+    });
+    expect(tts.capabilities.alignedTranscript).toBe(true);
+  });
+
+  it('reports alignedTranscript=true when ElevenLabs sync_alignment is set', () => {
+    const tts = makeTts({
+      model: 'elevenlabs/eleven_flash_v2',
+      modelOptions: { sync_alignment: true },
+    });
+    expect(tts.capabilities.alignedTranscript).toBe(true);
+  });
+
+  it('reports alignedTranscript=true when Inworld timestamp_type is WORD', () => {
+    const tts = makeTts({
+      model: 'inworld/inworld-tts-1',
+      modelOptions: { timestamp_type: 'WORD' },
+    });
+    expect(tts.capabilities.alignedTranscript).toBe(true);
+  });
+
+  it('recomputes alignedTranscript when updateOptions changes modelOptions', () => {
+    const tts = makeTts({ model: 'cartesia/sonic' });
+    expect(tts.capabilities.alignedTranscript).toBe(false);
+
+    tts.updateOptions({ modelOptions: { add_timestamps: true } });
+    expect(tts.capabilities.alignedTranscript).toBe(true);
+
+    tts.updateOptions({ modelOptions: { add_timestamps: false } });
+    expect(tts.capabilities.alignedTranscript).toBe(false);
+  });
+
+  it('recomputes alignedTranscript when updateOptions changes the model', () => {
+    const tts = makeTts({
+      model: 'cartesia/sonic',
+      modelOptions: { sync_alignment: true },
+    });
+    expect(tts.capabilities.alignedTranscript).toBe(false);
+
+    tts.updateOptions({ model: 'elevenlabs/eleven_flash_v2' });
+    expect(tts.capabilities.alignedTranscript).toBe(true);
+  });
+
+  it('invalidates the connection pool when session-affecting options change', () => {
+    const tts = makeTts({ model: 'cartesia/sonic' });
+    const invalidateSpy = vi.spyOn(tts.pool, 'invalidate');
+
+    tts.updateOptions({ modelOptions: { add_timestamps: true } });
+    expect(invalidateSpy).toHaveBeenCalledTimes(1);
+
+    tts.updateOptions({ model: 'elevenlabs/eleven_flash_v2' });
+    expect(invalidateSpy).toHaveBeenCalledTimes(2);
+
+    tts.updateOptions({ voice: 'narrator' });
+    expect(invalidateSpy).toHaveBeenCalledTimes(3);
+
+    tts.updateOptions({ language: 'en' });
+    expect(invalidateSpy).toHaveBeenCalledTimes(4);
+
+    // Empty update should not churn the pool.
+    tts.updateOptions({});
+    expect(invalidateSpy).toHaveBeenCalledTimes(4);
+  });
+});