diff --git a/README.md b/README.md index 0a34cdd22..795152f49 100644 --- a/README.md +++ b/README.md @@ -168,6 +168,80 @@ async for chat_completion in stream: print(chat_completion.choices) ``` +## Realtime transcription + +Transcribe live audio — a microphone, a phone call, a meeting — as it is being +spoken. You stream PCM audio over a WebSocket and receive interim text within +moments and a finalized transcript for each utterance. Requires the `realtime` +extra: + +```sh +uv add "together[realtime]" +``` + +Use `client.beta.realtime.transcription()`: a session built for live sources. If +the connection drops mid-conversation, the session holds on to the speech the +server hadn't transcribed yet, reconnects automatically, and picks up where +the transcript left off — words spoken during the outage still come back as +text. + +```python +from together import AsyncTogether +from together.realtime import TranscriptDelta, TranscriptCompleted + +client = AsyncTogether() + +async with client.beta.realtime.transcription( + model="openai/whisper-large-v3", + sample_rate=16_000, +) as session: + # feed audio from your capture source as it arrives (any chunk size) + await session.append(pcm_chunk) # 16 kHz mono 16-bit PCM + + async for event in session: + if isinstance(event, TranscriptDelta): + print("interim:", event.text) # updates while a phrase is spoken + elif isinstance(event, TranscriptCompleted): + print("final:", event.text) # one per finished utterance + + transcript = await session.flush() # finalize whatever was said last +``` + +What to know before integrating: + +- **Audio in**: 16 kHz mono 16-bit PCM (`pcm_s16le_16000`). Resample your + source before appending — passing `sample_rate=` lets the SDK reject a + mismatch loudly instead of incorrecly transcribing different sample rate . + `append()` never blocks on network state, so it is safe to call from a capture loop. +- **Utterance boundaries**: detected server-side by default — final + transcripts arrive on their own as the speaker pauses. To control + segmentation yourself, pass `turn_detection={"type": "none"}` and call + `await session.commit()` when each segment ends. +- **Tuning**: `language`, `prompt`, and turn-detection parameters + (`min_silence_duration_ms`, `max_speech_duration_s`, ...) are passed + through to the service. +- **When the connection drops**: The session emits + `Reconnecting`/`Reconnected` events and tries to retry the connection. + Transcripts recomputed from speech that was carried across the reconnect are marked + `replayed=True` and may overlap text you already received; voice agents + that act on each final can set `buffer={"max_replay_seconds": 0}` to + resume live with no re-emission instead. +- **If audio is ever dropped** (an outage longer than the retention window), + the session tells you with a `BufferGap` event. +- **If an endpoint fails for good**, calls raise `RealtimeConnectionError`. + To keep a conversation alive across endpoint outages, run a failover ring — + see [`examples/realtime_failover.py`](examples/realtime_failover.py): + on failure, `session.pending_audio()` hands you the un-transcribed speech + to seed a new session on another endpoint. +- **Sync applications**: `Together().beta.realtime.transcription(...)` mirrors the + async API on a background thread — good for a handful of sessions; use the + async client for high concurrency. +- **Full manual control** (raw wire events, no automatic recovery): + `client.beta.realtime.connect()`. + +See [`examples/realtime_transcription.py`](examples/realtime_transcription.py) +for a runnable end-to-end example. + ## Using types Nested request parameters are [TypedDicts](https://docs.python.org/3/library/typing.html#typing.TypedDict). Responses are [Pydantic models](https://docs.pydantic.dev) which also provide helper methods for things like: diff --git a/examples/realtime_failover.py b/examples/realtime_failover.py new file mode 100644 index 000000000..71be2a4b0 --- /dev/null +++ b/examples/realtime_failover.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +"""Realtime transcription with failover across multiple endpoints. + +Stream live audio to a primary endpoint; when it fails beyond recovery, +resume on the next endpoint in the ring, carrying over any speech that was +never transcribed so nothing is lost across the switch. + +The SDK handles transient failures on a single endpoint internally. +When an endpoint is unrecoverable, SDK calls raise RealtimeConnectionError. +Moving to another endpoint is application code: the loop below handles failover. + +Usage: + uv add "together[realtime]" + export TOGETHER_API_KEY=... + uv run ./examples/realtime_failover.py audio.wav # 16 kHz mono s16le WAV +""" + +from __future__ import annotations + +import sys +import wave +import asyncio +from pathlib import Path + +from together import AsyncTogether +from together.realtime import ( + BufferGap, + SessionStarted, + TranscriptDelta, + TranscriptCompleted, + RealtimeSessionEvent, + RealtimeConnectionError, +) + +# Independent deployments of the same model. Order = preference; on failure +# the next entry takes over, wrapping around. Each entry is (base_url, model); +# base_url is the API root and may point at region-specific hosts so the ring +# entries share no failure domain. +ENDPOINTS = [ + ("https://api.together.ai/v1", "openai/whisper-large-v3-endpoint1"), + ("https://api.together.ai/v1", "openai/whisper-large-v3-endpoint2"), +] + +SAMPLE_RATE = 16_000 +CHUNK_BYTES = SAMPLE_RATE * 2 // 10 # 100 ms per append, like a live source +MAX_ATTEMPTS = 3 * len(ENDPOINTS) # give up after several full ring cycles + + +def on_event(event: RealtimeSessionEvent) -> None: + """Handle session events as they arrive.""" + if isinstance(event, SessionStarted): + # Log the server-assigned session id — useful for correlating a session. + print(f"session {event.session_id} started on {event.model}") + elif isinstance(event, TranscriptDelta): + print(f"interim: {event.text}", end="\r") + elif isinstance(event, TranscriptCompleted): + # `replayed` marks re-transcriptions of carried-over audio; they may + # overlap text you already received. + marker = " (replayed)" if event.replayed else "" + print(f"final: {event.text}{marker}") + elif isinstance(event, BufferGap): + # Audio dropped beyond recovery. + print(f"[gap: {event.dropped_seconds:.1f}s lost]") + + +async def transcribe_with_failover(audio: bytes) -> str: + # One client per endpoint: base_url is client-level configuration. + # Clients are lightweight config objects — nothing connects until a + # session starts. + clients = [(AsyncTogether(base_url=base_url), model) for base_url, model in ENDPOINTS] + + transcripts: list[str] = [] + carry_over = b"" # audio the failed endpoint received but never transcribed + position = 0 # progress through the source; survives endpoint switches + + for attempt in range(MAX_ATTEMPTS): + client, model = clients[attempt % len(clients)] + print(f"--- streaming to {model}") + + session = client.beta.realtime.transcription( + model=model, + sample_rate=SAMPLE_RATE, + event_callback=on_event, + # Switch endpoints immediately on failure. Raise max_attempts to + # let the SDK retry the same endpoint first. + reconnect={"max_attempts": 0}, + ) + try: + async with session: + if carry_over: + # The un-transcribed speech from the failed endpoint is transmitted + # as fast as the connection allows, so the new endpoint catches up to live immediately. + await session.append(carry_over) + + while position < len(audio): + await session.append(audio[position : position + CHUNK_BYTES]) + position += CHUNK_BYTES + await asyncio.sleep(0.1) # simulate a live capture cadence + transcripts.append(await session.flush()) + return " ".join(t for t in transcripts if t) + except RealtimeConnectionError as exc: + # Endpoint is unrecoverable: keep its transcripts, take back the + # audio it never transcribed, and move to the next endpoint. + transcripts.extend(session.transcripts) + carry_over = session.pending_audio() + print(f"--- {model} failed ({exc}); carrying over {len(carry_over) / (SAMPLE_RATE * 2):.1f}s of audio") + + raise SystemExit("all endpoints failed repeatedly; giving up") + + +def load_pcm(path: Path) -> bytes: + with wave.open(str(path), "rb") as w: + if (w.getnchannels(), w.getsampwidth(), w.getframerate()) != (1, 2, SAMPLE_RATE): + raise SystemExit(f"expected mono 16-bit {SAMPLE_RATE} Hz WAV") + return w.readframes(w.getnframes()) + + +async def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + transcript = await transcribe_with_failover(load_pcm(Path(sys.argv[1]))) + print(f"\nfull transcript: {transcript}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/realtime_single_endpoint_failover.py b/examples/realtime_single_endpoint_failover.py new file mode 100644 index 000000000..9b4a6cbf2 --- /dev/null +++ b/examples/realtime_single_endpoint_failover.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Realtime transcription that rides out failures on a single endpoint. + +The SDK recovers from transient failures on its own: if the connection drops +mid-conversation, it reconnects with backoff and replays the speech the +server never transcribed — the application just keeps feeding audio. You'll +see Reconnecting/Reconnected events; there is nothing to handle. + +Only when the endpoint stays unrecoverable past the retry budget do SDK +calls raise RealtimeConnectionError. To keep a conversation alive across +endpoint outages, add an endpoint ring on top — see +examples/realtime_failover.py. + +Usage: + uv add "together[realtime]" + export TOGETHER_API_KEY=... + uv run ./examples/realtime_single_endpoint_failover.py audio.wav # 16 kHz mono s16le WAV +""" + +from __future__ import annotations + +import sys +import wave +import asyncio +from pathlib import Path + +from together import AsyncTogether +from together.lib.realtime import ( + BufferGap, + Reconnected, + Reconnecting, + SessionStarted, + TranscriptDelta, + TranscriptFailed, + TranscriptCompleted, + RealtimeSessionEvent, +) + +MODEL = "openai/whisper-large-v3-endpoint1" + +SAMPLE_RATE = 16_000 +CHUNK_BYTES = SAMPLE_RATE * 2 // 10 # 100 ms per append, like a live source + + +def on_event(event: RealtimeSessionEvent) -> None: + """Handle session events as they arrive.""" + if isinstance(event, SessionStarted): + # Log the server-assigned session id — useful when correlating a + # stream with server-side logs or support requests. + print(f"session {event.session_id} started on {event.model}") + elif isinstance(event, TranscriptDelta): + print(f"interim: {event.text}", end="\r") + elif isinstance(event, TranscriptCompleted): + # `replayed` marks re-transcriptions of speech replayed after a + # reconnect; they may overlap text you already received. + marker = " (replayed)" if event.replayed else "" + print(f"final: {event.text}{marker}") + elif isinstance(event, Reconnecting): + # Transient failure: the SDK is recovering on its own — no action + # needed, shown here only to make the recovery visible. + print(f"[reconnecting, attempt {event.attempt}: {event.reason}]") + elif isinstance(event, Reconnected): + print(f"[reconnected; {event.replayed_seconds:.1f}s of speech replayed]") + elif isinstance(event, TranscriptFailed): + # One utterance failed server-side; the session continues. + print(f"[utterance failed: {event.message}]") + elif isinstance(event, BufferGap): + # Audio dropped beyond recovery — always announced, never silent. + print(f"[gap: {event.dropped_seconds:.1f}s lost]") + + +async def transcribe(audio: bytes) -> str: + client = AsyncTogether() + + # Default reconnect settings retry this endpoint with backoff; failures + # you see as Reconnecting/Reconnected events are handled entirely by the + # SDK, with un-transcribed speech replayed after each reconnect. + async with client.beta.realtime.transcription( + model=MODEL, + sample_rate=SAMPLE_RATE, + event_callback=on_event, + ) as session: + position = 0 + while position < len(audio): + await session.append(audio[position : position + CHUNK_BYTES]) + position += CHUNK_BYTES + await asyncio.sleep(0.1) # simulate a live capture cadence + + return await session.flush() + + +def load_pcm(path: Path) -> bytes: + with wave.open(str(path), "rb") as w: + if (w.getnchannels(), w.getsampwidth(), w.getframerate()) != (1, 2, SAMPLE_RATE): + raise SystemExit(f"expected mono 16-bit {SAMPLE_RATE} Hz WAV") + return w.readframes(w.getnframes()) + + +async def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + transcript = await transcribe(load_pcm(Path(sys.argv[1]))) + print(f"\nfull transcript: {transcript}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/examples/realtime_transcription.py b/examples/realtime_transcription.py new file mode 100644 index 000000000..4f9f000b4 --- /dev/null +++ b/examples/realtime_transcription.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +"""Realtime transcription of live audio. + +Stream PCM audio as it is captured and receive interim text while a phrase +is being spoken, plus a finalized transcript per utterance. The WAV file +here stands in for a live source (microphone, phone call, meeting audio). + +Usage: + uv add "together[realtime]" + export TOGETHER_API_KEY=... + uv run ./examples/realtime_transcription.py audio.wav # 16 kHz mono s16le WAV +""" + +from __future__ import annotations + +import sys +import wave +import asyncio +from pathlib import Path + +from together import AsyncTogether +from together.realtime import ( + SessionStarted, + TranscriptDelta, + TranscriptCompleted, + RealtimeSessionEvent, +) + +MODEL = "openai/whisper-large-v3" + +SAMPLE_RATE = 16_000 +CHUNK_BYTES = SAMPLE_RATE * 2 // 10 # 100 ms per append, like a live source + + +def on_event(event: RealtimeSessionEvent) -> None: + """Handle session events as they arrive.""" + if isinstance(event, SessionStarted): + # Log the server-assigned session id — useful when correlating a + # stream with server-side logs or support requests. + print(f"session {event.session_id} started on {event.model}") + elif isinstance(event, TranscriptDelta): + print(f"interim: {event.text}", end="\r") + elif isinstance(event, TranscriptCompleted): + print(f"final: {event.text}") + + +async def transcribe(audio: bytes) -> str: + client = AsyncTogether() + + async with client.beta.realtime.transcription( + model=MODEL, + sample_rate=SAMPLE_RATE, + event_callback=on_event, + ) as session: + position = 0 + while position < len(audio): + await session.append(audio[position : position + CHUNK_BYTES]) + position += CHUNK_BYTES + await asyncio.sleep(0.1) # simulate a live capture cadence + + return await session.flush() # finalize whatever was said last + + +def load_pcm(path: Path) -> bytes: + with wave.open(str(path), "rb") as w: + if (w.getnchannels(), w.getsampwidth(), w.getframerate()) != (1, 2, SAMPLE_RATE): + raise SystemExit(f"expected mono 16-bit {SAMPLE_RATE} Hz WAV") + return w.readframes(w.getnframes()) + + +async def main() -> None: + if len(sys.argv) != 2: + raise SystemExit(__doc__) + transcript = await transcribe(load_pcm(Path(sys.argv[1]))) + print(f"\nfull transcript: {transcript}") + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/pyproject.toml b/pyproject.toml index 52961f7ce..47001f69a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,8 @@ aiohttp = ["aiohttp", "httpx_aiohttp>=0.1.9"] aiofiles = ["aiofiles>=25.0.0", "types-aiofiles"] pyarrow = ["pyarrow>=16.1.0", "pyarrow-stubs>=10.0.1.7"] cli = ["PyYAML", "questionary>=2.1.1"] +# websockets>=13 is the floor for both websockets.sync.client and websockets.asyncio.client +realtime = ["websockets>=13,<16"] [project.scripts] together = "together.lib.cli:main" @@ -88,6 +90,7 @@ dev = [ "pytest-xdist>=3.6.1", "pytest-mock>=3.14.0", "aiohttp>=3.12.13", + "websockets>=13,<16", ] pydantic-v1 = [ "pydantic>=1.9.0,<2", diff --git a/src/together/lib/realtime/__init__.py b/src/together/lib/realtime/__init__.py new file mode 100644 index 000000000..f79114223 --- /dev/null +++ b/src/together/lib/realtime/__init__.py @@ -0,0 +1,71 @@ +"""Realtime transcription over WebSocket for the Together SDK. + +Public import surface for the realtime package. Everything not exported here +is private and may change without notice. +""" + +from ._types import ( + BufferGap, + TurnEvent, + Reconnected, + Reconnecting, + UnknownEvent, + BufferOptions, + SessionStarted, + TranscriptDelta, + ReconnectOptions, + TranscriptFailed, + EchoResponseEvent, + RealtimeErrorInfo, + TurnDetectionParam, + RealtimeServerEvent, + SessionCreatedEvent, + TranscriptCompleted, + RealtimeSessionEvent, + TranscriptionDeltaEvent, + TranscriptionFailedEvent, + TranscriptionCompletedEvent, + InputAudioBufferProcessedEvent, +) +from ._exceptions import ( + RealtimeError, + RealtimeSessionError, + RealtimeConnectionError, + RealtimeIdleTimeoutError, + RealtimeInvalidStateError, + RealtimeBufferOverflowError, +) + +__all__ = [ + # events (wire) + "RealtimeServerEvent", + "SessionCreatedEvent", + "TranscriptionDeltaEvent", + "TranscriptionCompletedEvent", + "TranscriptionFailedEvent", + "TurnEvent", + "InputAudioBufferProcessedEvent", + "EchoResponseEvent", + "UnknownEvent", + "RealtimeErrorInfo", + # events (normalized session) + "RealtimeSessionEvent", + "SessionStarted", + "TranscriptDelta", + "TranscriptCompleted", + "TranscriptFailed", + "Reconnecting", + "Reconnected", + "BufferGap", + # options + "TurnDetectionParam", + "ReconnectOptions", + "BufferOptions", + # exceptions + "RealtimeError", + "RealtimeConnectionError", + "RealtimeSessionError", + "RealtimeIdleTimeoutError", + "RealtimeBufferOverflowError", + "RealtimeInvalidStateError", +] diff --git a/src/together/lib/realtime/_connection.py b/src/together/lib/realtime/_connection.py new file mode 100644 index 000000000..37aa78274 --- /dev/null +++ b/src/together/lib/realtime/_connection.py @@ -0,0 +1,351 @@ +from __future__ import annotations + +import json +from typing import TYPE_CHECKING, Any, Dict, Mapping, Iterator, Optional, AsyncIterator + +import httpx + +from ._types import ( + DEFAULT_AUDIO_FORMAT, + TurnDetectionParam, + echo_event, + clear_event, + append_event, + commit_event, + parse_server_event, + session_update_event, +) + +if TYPE_CHECKING: + from websockets.sync.client import ClientConnection as SyncClientConnection + from websockets.asyncio.client import ClientConnection as AsyncClientConnection + + from ..._client import Together, AsyncTogether + +__all__ = [ + "RealtimeConnection", + "AsyncRealtimeConnection", + "RealtimeConnectionManager", + "AsyncRealtimeConnectionManager", + "build_realtime_url", + "build_realtime_headers", + "handshake_status_of", +] + +_INSTALL_HINT = 'The realtime API requires the `websockets` package; install it with `uv add "together[realtime]"`' + + +def _require_websockets() -> None: + import importlib.util + + if importlib.util.find_spec("websockets") is None: + raise RuntimeError(_INSTALL_HINT) + + +def handshake_status_of(exc: BaseException) -> Optional[int]: + """HTTP status of a rejected WebSocket handshake, if the exception carries one.""" + try: + from websockets.exceptions import InvalidStatus + except ImportError: + return None + if isinstance(exc, InvalidStatus): + response = getattr(exc, "response", None) + return getattr(response, "status_code", None) + return None + + +def build_realtime_url( + base_url: httpx.URL, + *, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + turn_detection: Optional[TurnDetectionParam] = None, + extra_query: Optional[Mapping[str, Any]] = None, +) -> str: + """Derive the wss:// realtime URL from the client's HTTP base_url. + + The model MUST be in the query string before any audio is appended — + without it the server queues appends and never starts the session. + """ + scheme = {"https": "wss", "http": "ws"}.get(base_url.scheme, base_url.scheme) + path = base_url.path.rstrip("/") + "/realtime" + params: Dict[str, Any] = {"model": model, "input_audio_format": input_audio_format} + if turn_detection: + detection = dict(turn_detection) + detection_type = detection.pop("type", None) + if detection_type is not None: + params["turn_detection"] = detection_type + params.update(detection) + if extra_query: + params.update(dict(extra_query)) + url = base_url.copy_with(scheme=scheme, path=path, params=params) + return str(url) + + +def build_realtime_headers( + auth_headers: Mapping[str, str], + *, + extra_headers: Optional[Mapping[str, str]] = None, +) -> Dict[str, str]: + headers: Dict[str, str] = {**auth_headers, "OpenAI-Beta": "realtime=v1"} + if extra_headers: + headers.update(extra_headers) + return headers + + +def _session_config( + *, + language: Optional[str], + prompt: Optional[str], + rolling_prompt: Optional[bool], + energy_gate_rms: Optional[float], + session_params: Optional[Mapping[str, Any]], +) -> Dict[str, Any]: + """Session-level params delivered via `transcription_session.updated`.""" + config: Dict[str, Any] = {} + if language is not None: + config["language"] = language + if prompt is not None: + config["prompt"] = prompt + if rolling_prompt is not None: + config["rolling_prompt"] = rolling_prompt + if energy_gate_rms is not None: + config["energy_gate_rms"] = energy_gate_rms + if session_params: + config.update(dict(session_params)) + return config + + +class _InputAudioBuffer: + """`conn.input_audio_buffer.append/commit/clear` helpers (sync).""" + + def __init__(self, connection: RealtimeConnection) -> None: + self._connection = connection + + def append(self, pcm: bytes) -> None: + self._connection.send(append_event(pcm)) + + def commit(self) -> None: + self._connection.send(commit_event()) + + def clear(self) -> None: + self._connection.send(clear_event()) + + +class RealtimeConnection: + """A live, typed realtime WebSocket connection (no retry logic). + + Iterate to receive parsed server events; use `send` / `input_audio_buffer` + helpers to talk to the server. Audio is always sent base64-encoded inside + `input_audio_buffer.append` — the server terminates sessions on binary frames. + """ + + def __init__(self, ws: SyncClientConnection) -> None: + self._ws = ws + self.input_audio_buffer = _InputAudioBuffer(self) + + def send(self, event: Mapping[str, Any]) -> None: + self._ws.send(json.dumps(dict(event))) + + def recv(self, timeout: Optional[float] = None) -> object: + return parse_server_event(self._ws.recv(timeout=timeout)) + + def session_update(self, session: Mapping[str, Any]) -> None: + self.send(session_update_event(session)) + + def echo(self, echo_id: Any = None) -> None: + self.send(echo_event(echo_id=echo_id)) + + def close(self) -> None: + self._ws.close() + + def __iter__(self) -> Iterator[object]: + while True: + try: + yield self.recv() + except Exception: + return + + +class _AsyncInputAudioBuffer: + """`conn.input_audio_buffer.append/commit/clear` helpers (async).""" + + def __init__(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + + async def append(self, pcm: bytes) -> None: + await self._connection.send(append_event(pcm)) + + async def commit(self) -> None: + await self._connection.send(commit_event()) + + async def clear(self) -> None: + await self._connection.send(clear_event()) + + +class AsyncRealtimeConnection: + """Async counterpart of RealtimeConnection.""" + + def __init__(self, ws: AsyncClientConnection) -> None: + self._ws = ws + self.input_audio_buffer = _AsyncInputAudioBuffer(self) + + def is_closing(self) -> bool: + """True once the close handshake has started in either direction. + + A graceful server close can stall for the server's close_timeout when + the client is mid-burst; callers polling recv() should treat CLOSING + as connection-lost instead of waiting the handshake out. + """ + from websockets.protocol import State + + return self._ws.state is not State.OPEN + + async def send(self, event: Mapping[str, Any]) -> None: + await self._ws.send(json.dumps(dict(event))) + + async def recv(self) -> object: + return parse_server_event(await self._ws.recv()) + + async def session_update(self, session: Mapping[str, Any]) -> None: + await self.send(session_update_event(session)) + + async def echo(self, echo_id: Any = None) -> None: + await self.send(echo_event(echo_id=echo_id)) + + async def close(self) -> None: + await self._ws.close() + + def __aiter__(self) -> AsyncIterator[object]: + return self._iterate() + + async def _iterate(self) -> AsyncIterator[object]: + while True: + try: + yield await self.recv() + except Exception: + return + + +class RealtimeConnectionManager: + """Context manager returned by `client.beta.realtime.connect(...)` (sync).""" + + def __init__( + self, + *, + client: Together, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + turn_detection: Optional[TurnDetectionParam] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + open_timeout: Optional[float] = 10.0, + ) -> None: + self._client = client + self._model = model + self._input_audio_format = input_audio_format + self._turn_detection = turn_detection + self._session_config = _session_config( + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + ) + self._extra_query = extra_query + self._extra_headers = extra_headers + self._open_timeout = open_timeout + self._connection: Optional[RealtimeConnection] = None + + def connect(self) -> RealtimeConnection: + _require_websockets() + from websockets.sync.client import connect as ws_connect + + url = build_realtime_url( + self._client.base_url, + model=self._model, + input_audio_format=self._input_audio_format, + turn_detection=self._turn_detection, + extra_query=self._extra_query, + ) + headers = build_realtime_headers(self._client.auth_headers, extra_headers=self._extra_headers) + ws = ws_connect(url, additional_headers=headers, open_timeout=self._open_timeout) + self._connection = RealtimeConnection(ws) + if self._session_config: + self._connection.session_update(self._session_config) + return self._connection + + def __enter__(self) -> RealtimeConnection: + return self.connect() + + def __exit__(self, *_exc: object) -> None: + if self._connection is not None: + self._connection.close() + self._connection = None + + +class AsyncRealtimeConnectionManager: + """Awaitable context manager returned by `client.beta.realtime.connect(...)` (async).""" + + def __init__( + self, + *, + client: AsyncTogether, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + turn_detection: Optional[TurnDetectionParam] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + open_timeout: Optional[float] = 10.0, + ) -> None: + self._client = client + self._model = model + self._input_audio_format = input_audio_format + self._turn_detection = turn_detection + self._session_config = _session_config( + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + ) + self._extra_query = extra_query + self._extra_headers = extra_headers + self._open_timeout = open_timeout + self._connection: Optional[AsyncRealtimeConnection] = None + + async def connect(self) -> AsyncRealtimeConnection: + _require_websockets() + from websockets.asyncio.client import connect as ws_connect + + url = build_realtime_url( + self._client.base_url, + model=self._model, + input_audio_format=self._input_audio_format, + turn_detection=self._turn_detection, + extra_query=self._extra_query, + ) + headers = build_realtime_headers(self._client.auth_headers, extra_headers=self._extra_headers) + ws = await ws_connect(url, additional_headers=headers, open_timeout=self._open_timeout) + self._connection = AsyncRealtimeConnection(ws) + if self._session_config: + await self._connection.session_update(self._session_config) + return self._connection + + async def __aenter__(self) -> AsyncRealtimeConnection: + return await self.connect() + + async def __aexit__(self, *_exc: object) -> None: + if self._connection is not None: + await self._connection.close() + self._connection = None diff --git a/src/together/lib/realtime/_exceptions.py b/src/together/lib/realtime/_exceptions.py new file mode 100644 index 000000000..5c5702590 --- /dev/null +++ b/src/together/lib/realtime/_exceptions.py @@ -0,0 +1,70 @@ +from __future__ import annotations + +from typing import Any, Optional + +from ..._exceptions import TogetherError + +__all__ = [ + "RealtimeError", + "RealtimeConnectionError", + "RealtimeSessionError", + "RealtimeIdleTimeoutError", + "RealtimeBufferOverflowError", + "RealtimeInvalidStateError", +] + + +class RealtimeError(TogetherError): + """Base class for all realtime transcription errors.""" + + +class RealtimeConnectionError(RealtimeError): + """Raised when the websocket connection could not be (re)established. + + Carries the number of attempts made and the last underlying cause. + """ + + def __init__(self, message: str, *, attempts: int = 0, cause: Optional[BaseException] = None) -> None: + super().__init__(message) + self.attempts = attempts + self.__cause__ = cause + + +class RealtimeSessionError(RealtimeError): + """A fatal, non-retryable error reported by the server. + + Examples: invalid model, unsupported audio format, revoked access. + """ + + def __init__( + self, + message: str, + *, + error_type: Optional[str] = None, + code: Optional[str] = None, + raw: Optional[Any] = None, + ) -> None: + super().__init__(message) + self.error_type = error_type + self.code = code + self.raw = raw + + +class RealtimeIdleTimeoutError(RealtimeError): + """The server closed the session because no audio was appended for its idle window. + + The server is healthy; the client simply went silent. Not retried by default — + reconnect only happens automatically when un-replayed audio is pending. + """ + + +class RealtimeBufferOverflowError(RealtimeError): + """The SDK-side recovery buffer exceeded its limit with overflow policy "error".""" + + def __init__(self, message: str, *, dropped_seconds: float = 0.0) -> None: + super().__init__(message) + self.dropped_seconds = dropped_seconds + + +class RealtimeInvalidStateError(RealtimeError): + """An operation was attempted in an invalid state (e.g. append after close).""" diff --git a/src/together/lib/realtime/_session.py b/src/together/lib/realtime/_session.py new file mode 100644 index 000000000..e01554888 --- /dev/null +++ b/src/together/lib/realtime/_session.py @@ -0,0 +1,902 @@ +from __future__ import annotations + +import atexit +import random +import asyncio +import logging +import threading +from typing import TYPE_CHECKING, Any, List, Mapping, Callable, Iterator, Optional, AsyncIterator + +from ._state import ( + BufferPool, + FailureKind, + BackoffPolicy, + RecoveryState, + iter_frames, + classify_fatal_error, + classify_handshake_status, +) +from ._types import ( + DEFAULT_AUDIO_FORMAT, + BufferGap, + Reconnected, + Reconnecting, + BufferOptions, + SessionStarted, + TranscriptDelta, + ReconnectOptions, + TranscriptFailed, + EchoResponseEvent, + TurnDetectionParam, + SessionCreatedEvent, + TranscriptCompleted, + RealtimeSessionEvent, + TranscriptionDeltaEvent, + TranscriptionFailedEvent, + TranscriptionCompletedEvent, + InputAudioBufferProcessedEvent, + echo_event, + clear_event, + append_event, + commit_event, + sample_rate_of, + session_update_event, +) +from ._connection import ( + AsyncRealtimeConnection, + AsyncRealtimeConnectionManager, + _session_config, + handshake_status_of, +) +from ._exceptions import ( + RealtimeError, + RealtimeSessionError, + RealtimeConnectionError, + RealtimeIdleTimeoutError, + RealtimeInvalidStateError, +) + +if TYPE_CHECKING: + from ..._client import AsyncTogether + +__all__ = [ + "AsyncRealtimeTranscriptionSession", + "RealtimeTranscriptionSession", + "DEFAULT_RECONNECT_THROTTLE", +] + +log = logging.getLogger("together.realtime") + +_SENTINEL = object() + +# Caps concurrent reconnect handshakes process-wide so a shared network blip +# with thousands of sessions doesn't stampede the server. +DEFAULT_RECONNECT_THROTTLE = threading.BoundedSemaphore(32) + +_DEFAULT_POOL = BufferPool() + +# Internal liveness cadence: probe every ~5s (jittered), declare the path dead +# if a probe goes unanswered for 2s, and force-reconnect if audio is flowing +# but no server events arrive for 30s. +_ECHO_INTERVAL = 5.0 +_ECHO_TIMEOUT = 2.0 +_STALE_STREAM_TIMEOUT = 30.0 + + +class AsyncRealtimeTranscriptionSession: + """Auto-reconnecting realtime transcription session (async). + + Maintains an SDK-side recovery buffer; on retryable failures it reconnects + with backoff and replays un-acknowledged audio from the last transcribed + position (the `completed` event's start + duration, minus a safety margin). + Iterate the session to consume normalized events. + """ + + def __init__( + self, + *, + client: AsyncTogether, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + sample_rate: Optional[int] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + turn_detection: Optional[TurnDetectionParam] = None, + reconnect: Optional[ReconnectOptions] = None, + buffer: Optional[BufferOptions] = None, + keepalive_silence: bool = False, + reprime_prompt: bool = False, + max_chunk_ms: float = 1000.0, + pool: Optional[BufferPool] = None, + event_callback: Optional[Callable[[RealtimeSessionEvent], None]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + reconnect_throttle: Optional[threading.BoundedSemaphore] = None, + ) -> None: + expected_rate = sample_rate_of(input_audio_format) + if sample_rate is not None and sample_rate != expected_rate: + raise ValueError( + f"audio source is {sample_rate}Hz but input_audio_format={input_audio_format!r} " + f"expects {expected_rate}Hz — resample before appending" + ) + self._manual_commit = turn_detection is not None and turn_detection.get("type") == "none" + if keepalive_silence and not self._manual_commit: + raise ValueError( + "keepalive_silence requires turn_detection={'type': 'none'}: with server VAD " + "enabled, injected silence would finalize a user utterance at a natural pause" + ) + + reconnect = reconnect or {} + buffer = buffer or {} + + self._client = client + self._model = model + self._format = input_audio_format + self._language = language + self._prompt = prompt + self._rolling_prompt = rolling_prompt + self._energy_gate_rms = energy_gate_rms + self._session_params = session_params + self._turn_detection = turn_detection + self._extra_query = extra_query + self._extra_headers = extra_headers + self._keepalive_silence = keepalive_silence + self._reprime_prompt = reprime_prompt + self._event_callback = event_callback + self._throttle = reconnect_throttle or DEFAULT_RECONNECT_THROTTLE + + self._backoff = BackoffPolicy( + initial=reconnect.get("backoff_initial", 0.5), + maximum=reconnect.get("backoff_max", 15.0), + max_attempts=reconnect.get("max_attempts", 10), + max_elapsed=reconnect.get("max_elapsed", 120.0), + ) + pool = pool if pool is not None else _DEFAULT_POOL + self.state = RecoveryState( + input_audio_format=input_audio_format, + replay_margin=buffer.get("replay_margin", 0.0), + max_replay_seconds=buffer.get("max_replay_seconds", 5.0), + max_seconds=buffer.get("max_seconds", 120.0), + overflow=buffer.get("overflow", "drop_oldest"), + pool=pool, + ) + self._pool = pool + pool.register(self, lambda: self.state.buffer.size, self.state.reclaim) + + # Liveness probing is internal (not part of the public API): a small + # application-level ping catches silent failures — connection looks + # open but the server stopped responding — within a few seconds. + self._echo_interval = _ECHO_INTERVAL + self._echo_timeout = _ECHO_TIMEOUT + self._stale_stream_timeout = _STALE_STREAM_TIMEOUT + + # audio passes through with the caller's own chunk boundaries; this + # cap only splits oversized chunks so one websocket message never + # exceeds what the server will parse + self._max_chunk_bytes = max(2, int(self.state.bps * max_chunk_ms / 1000.0)) + self._events: asyncio.Queue[Any] = asyncio.Queue() + self._connection: Optional[AsyncRealtimeConnection] = None + self._reader_task: Optional[asyncio.Task[None]] = None + self._watchdog_task: Optional[asyncio.Task[None]] = None + self._reconnect_task: Optional[asyncio.Task[None]] = None + self._writer_task: Optional[asyncio.Task[None]] = None + self._writer_wakeup = asyncio.Event() + self._sent_offset = 0 + # control events (e.g. commit) ordered by buffer offset: sent by the + # writer only once all audio before the offset has been sent + self._pending_controls: List[Any] = [] # list[tuple[int, dict]] + self._replaying_until = 0 + + self._closed = False + self._failure: Optional[BaseException] = None + self._echo_pending_since: Optional[float] = None + self._last_server_event = 0.0 + self._last_transcript_event = 0.0 + self._last_append_at = 0.0 + self._connected_at = 0.0 + self.transcripts: List[str] = [] + self._warned_riff = False + self._warned_silence = False + + # -- lifecycle --------------------------------------------------------- + + async def __aenter__(self) -> "AsyncRealtimeTranscriptionSession": + await self.start() + return self + + async def __aexit__(self, exc_type: object, *_exc: object) -> None: + if exc_type is None and self._failure is None and not self._closed: + try: + await self.flush() + except RealtimeError: + pass + await self.close() + + async def start(self) -> None: + if self._connection is not None: + return + try: + connection = await self._open_connection() + except Exception as exc: + status = handshake_status_of(exc) + detail = f"HTTP {status}" if status is not None else exc.__class__.__name__ + raise RealtimeConnectionError( + f"could not open realtime connection to {self._client.base_url} ({detail}); " + "check that base_url / TOGETHER_BASE_URL points at an API root that " + "serves /realtime, e.g. https://api.together.ai/v1", + cause=exc, + ) from exc + created = await self._await_session_created(connection) + self.state.begin_epoch(self.state.write_head) + self._sent_offset = self.state.write_head + self._attach(connection) + self._emit( + SessionStarted( + session_id=created.session.id if created.session else None, + model=self.model, + epoch=self.state.epoch, + ) + ) + self._writer_task = asyncio.create_task(self._writer_loop()) + self._watchdog_task = asyncio.create_task(self._watchdog_loop()) + + async def close(self) -> None: + if self._closed: + return + self._closed = True + self._writer_wakeup.set() + for task in (self._watchdog_task, self._writer_task, self._reconnect_task, self._reader_task): + if task is not None: + task.cancel() + if self._connection is not None: + try: + await self._connection.close() + except Exception: + pass + self._connection = None + self._pool.unregister(self) + self._events.put_nowait(_SENTINEL) + + # -- public API ---------------------------------------------------------- + + async def append(self, pcm: bytes) -> None: + """Buffer PCM audio for transmission. Never blocks on reconnection. + + If the session has failed terminally, raises that failure (e.g. + RealtimeConnectionError) so a plain feed loop can drive failover + without a separate consumer task watching for errors. + """ + self._raise_if_failed() + if self._closed: + raise RealtimeInvalidStateError("append() after close()") + self._sanity_check_audio(pcm) + self.state.record_append(pcm) + self._flush_gap_events() + self._last_append_at = self._now() + self._writer_wakeup.set() + + async def commit(self) -> None: + """Finalize the buffered tail (required when turn_detection='none').""" + self._raise_if_failed() + if self._closed: + raise RealtimeInvalidStateError("commit() after close()") + self.state.record_commit() + self._pending_controls.append((self.state.write_head, commit_event())) + self._writer_wakeup.set() + + async def clear(self) -> None: + """Drop buffered audio on the server and in the SDK recovery buffer.""" + self._raise_if_failed() + if self._closed: + raise RealtimeInvalidStateError("clear() after close()") + self.state.record_clear() + self._pending_controls.clear() + self._sent_offset = self.state.write_head + if self._connection is not None: + await self._connection.send(clear_event()) + + async def flush(self, *, quiescence: float = 2.0, timeout: float = 30.0) -> str: + """Commit the buffered tail and wait until transcript events go quiet. + + Commit works with server VAD too — it force-finalizes in-progress + speech, so the tail utterance gets a final transcript. Bounded by + `timeout`; uses event quiescence rather than any exact completeness + signal (a commit can legitimately yield zero transcripts). Returns the + concatenation of all final transcripts seen so far. + """ + if self.state.write_head > (self.state.anchor or 0): + await self.commit() + self._raise_if_failed() + started = self._now() + deadline = started + timeout + while self._now() < deadline: + if self._failure is not None: + raise self._failure + if self._closed: + break + fully_sent = self._sent_offset >= self.state.write_head and not self._pending_controls + # the quiescence clock starts at flush time so the commit sent + # above always gets a response window: a final for the tail + # utterance may land seconds after the last transcript event + quiet_for = self._now() - max(self._last_transcript_event, self._connected_at, started) + if fully_sent and quiet_for >= quiescence and self._reconnect_task is None: + break + await asyncio.sleep(min(0.1, quiescence / 4)) + return " ".join(self.transcripts) + + def pending_audio(self) -> bytes: + """Audio appended but not yet covered by a final transcript. + + The recovery primitive for orchestrating failover OUTSIDE the SDK: + when this session fails terminally (RealtimeConnectionError after + retries), feed this audio — together with a prompt built from + `self.transcripts` — into a fresh session on an alternate endpoint to + resume where this one left off. + """ + plan = self.state.replay_plan() + return b"".join(self.state.buffer.read_from(plan.start_offset)) + + def context_prompt(self, max_chars: int = 200) -> str: + """Tail of the delivered transcripts, suitable as `prompt` for a + successor session (mirrors the server's rolling-prompt context).""" + return " ".join(self.transcripts)[-max_chars:] + + def _raise_if_failed(self) -> None: + if self._failure is not None: + raise self._failure + + async def drain(self) -> str: + """Alias for flush() then close(); returns the final transcript.""" + text = await self.flush() + await self.close() + return text + + @property + def metrics(self) -> Any: + return self.state.metrics + + def __aiter__(self) -> AsyncIterator[RealtimeSessionEvent]: + return self._iterate() + + async def _iterate(self) -> AsyncIterator[RealtimeSessionEvent]: + while True: + item = await self._events.get() + if item is _SENTINEL: + break + yield item + if self._failure is not None: + raise self._failure + + # -- internals: connection ------------------------------------------------ + + @property + def model(self) -> str: + return self._model + + def _manager(self) -> AsyncRealtimeConnectionManager: + return AsyncRealtimeConnectionManager( + client=self._client, + model=self.model, + input_audio_format=self._format, + turn_detection=self._turn_detection, + extra_query=self._extra_query, + extra_headers=self._extra_headers, + ) + + async def _open_connection(self) -> AsyncRealtimeConnection: + return await self._manager().connect() + + async def _await_session_created(self, connection: AsyncRealtimeConnection) -> SessionCreatedEvent: + try: + event = await asyncio.wait_for(connection.recv(), timeout=10.0) + except asyncio.TimeoutError: + raise RealtimeConnectionError("timed out waiting for session.created") from None + if not isinstance(event, SessionCreatedEvent): + raise RealtimeConnectionError(f"expected session.created, got {getattr(event, 'type', event)!r}") + await self._configure_session(connection) + return event + + async def _configure_session(self, connection: AsyncRealtimeConnection) -> None: + # single source of truth for the session-param field mapping lives in + # _connection._session_config; only the reprime tail is layered on here + session = _session_config( + language=self._language, + prompt=self._effective_prompt(), + rolling_prompt=self._rolling_prompt, + energy_gate_rms=self._energy_gate_rms, + session_params=self._session_params, + ) + if session: + await connection.send(session_update_event(session)) + + def _effective_prompt(self) -> Optional[str]: + prompt = self._prompt + if self._reprime_prompt and self.transcripts: + # restore the decode context the server lost with the old connection + tail = " ".join(self.transcripts)[-200:] + prompt = f"{prompt} {tail}".strip() if prompt else tail + return prompt + + def _attach(self, connection: AsyncRealtimeConnection) -> None: + self._connection = connection + self._connected_at = self._now() + self._echo_pending_since = None + self._last_server_event = self._now() + epoch = self.state.epoch + self._reader_task = asyncio.create_task(self._reader_loop(connection, epoch)) + + # -- internals: reader ------------------------------------------------------ + + async def _reader_loop(self, connection: AsyncRealtimeConnection, epoch: int) -> None: + try: + while True: + try: + event = await asyncio.wait_for(connection.recv(), timeout=0.5) + except asyncio.TimeoutError: + # A graceful server close can stall for the server's + # close_timeout while we're bursting appends; don't wait + # the handshake out — reconnect as soon as CLOSING shows. + if connection.is_closing(): + raise ConnectionError("server initiated close") from None + continue + if self._closed: + return + if self.state.epoch != epoch: + return # stale connection still draining; drop its events + self._handle_server_event(event) + except asyncio.CancelledError: + raise + except Exception as exc: + if self._closed or self.state.epoch != epoch: + return + self._schedule_reconnect(f"connection lost: {exc.__class__.__name__}") + + def _handle_server_event(self, event: object) -> None: + now = self._now() + self._last_server_event = now + + if isinstance(event, TranscriptionDeltaEvent): + self._last_transcript_event = now + seg = self.state.segment_for(event.item_id, replayed=self._is_replayed_now()) + self.state.metrics.delta_events += 1 + self._emit( + TranscriptDelta( + segment_id=seg.segment_id, + text=event.delta, + audio_start=seg.audio_start, + audio_end=self.state.to_seconds(self.state.write_head), + replayed=seg.replayed, + raw=event, + ) + ) + elif isinstance(event, TranscriptionCompletedEvent): + self._last_transcript_event = now + seg = self.state.segment_for(event.item_id, replayed=self._is_replayed_now()) + processed_seconds = ( + event.start + event.duration if event.start is not None and event.duration is not None else None + ) + self.state.record_completed(processed_seconds) + self.state.close_segment(event.item_id) + if event.transcript: + self.transcripts.append(event.transcript) + self._emit( + TranscriptCompleted( + segment_id=seg.segment_id, + text=event.transcript, + audio_start=seg.audio_start, + audio_end=self.state.to_seconds(self.state.anchor or self.state.write_head), + replayed=seg.replayed, + raw=event, + ) + ) + self._flush_gap_events() + elif isinstance(event, TranscriptionFailedEvent): + if event.is_fatal: + self._handle_fatal_failed(event) + else: + seg = self.state.segment_for(event.item_id, replayed=False) + self.state.close_segment(event.item_id) + self._emit( + TranscriptFailed( + segment_id=seg.segment_id, + message=event.error.message if event.error else None, + raw=event, + ) + ) + elif isinstance(event, InputAudioBufferProcessedEvent): + self.state.record_processed(event.processed_ms) + elif isinstance(event, EchoResponseEvent): + self._echo_pending_since = None + # turn events / unknown events are intentionally not surfaced by the + # normalized stream; use client.beta.realtime.connect() for raw access. + + def _is_replayed_now(self) -> bool: + """Events for audio at or before the replay watermark came from replayed bytes.""" + return self.state.write_head <= self._replaying_until + + def _handle_fatal_failed(self, event: TranscriptionFailedEvent) -> None: + kind = classify_fatal_error(event.error) + message = (event.error.message if event.error else None) or "realtime session failed" + if kind is FailureKind.RETRYABLE: + self._schedule_reconnect(message) + elif kind is FailureKind.IDLE_TIMEOUT: + plan = self.state.replay_plan() + if plan.start_offset < self.state.write_head or plan.resend_commit: + self._schedule_reconnect("idle timeout with unreplayed audio") + else: + self._fail( + RealtimeIdleTimeoutError( + "server closed the session after 300s without audio appends; " + "enable keepalive_silence or append audio continuously" + ) + ) + else: + self._fail( + RealtimeSessionError( + message, + error_type=event.error.type if event.error else None, + code=event.error.code if event.error else None, + raw=event, + ) + ) + + # -- internals: writer ------------------------------------------------------- + + async def _writer_loop(self) -> None: + """Drain the buffer cursor to the current connection. + + Replay and live streaming are the same code path: reconnect simply + moves `_sent_offset` back to the replay start. + """ + try: + while not self._closed: + # clear before checking for work so an append landing mid-drain + # re-sets it and the wait below returns immediately + self._writer_wakeup.clear() + connection = self._connection + progressed = False + if connection is not None and self._reconnect_task is None: + while self._sent_offset < self.state.write_head: + # chunk boundaries are the caller's own append() chunks + # (the buffer stores them as appended); a chunk is only + # split when it exceeds the per-message safety cap + chunks = list(self.state.buffer.read_from(self._sent_offset)) + if not chunks: + # cursor points below retained data (trimmed); skip ahead + self._sent_offset = max(self._sent_offset, self.state.buffer.start_offset) + if self._sent_offset >= self.state.write_head: + break + continue + for chunk in chunks: + for piece in iter_frames(chunk, self._max_chunk_bytes): + await connection.send(append_event(piece)) + self._sent_offset += len(piece) + await self._maybe_send_controls(connection) + progressed = True + await self._maybe_send_controls(connection) + if not progressed: + # No progress possible right now (nothing to send, no + # connection, or a reconnect is in flight). ALWAYS yield + # here: unsent audio accumulating during a reconnect must + # not turn this loop into a busy-spin that starves the + # event loop (and with it the reconnect task itself). + try: + await asyncio.wait_for(self._writer_wakeup.wait(), timeout=0.25) + except asyncio.TimeoutError: + pass + except asyncio.CancelledError: + raise + except Exception as exc: + if not self._closed: + self._schedule_reconnect(f"send failed: {exc.__class__.__name__}") + # writer restarts with the new connection + self._writer_task = asyncio.create_task(self._writer_loop()) + + async def _maybe_send_controls(self, connection: AsyncRealtimeConnection) -> None: + while self._pending_controls and self._pending_controls[0][0] <= self._sent_offset: + _offset, control = self._pending_controls.pop(0) + await connection.send(control) + + # -- internals: reconnect ------------------------------------------------------ + + def _schedule_reconnect(self, reason: str) -> None: + if self._closed or self._failure is not None: + return + if self._reconnect_task is not None and not self._reconnect_task.done(): + return # single-flight + self._reconnect_task = asyncio.create_task(self._reconnect(reason)) + + async def _reconnect(self, reason: str) -> None: + old_connection = self._connection + started = self._now() + attempt = 0 + try: + while not self._closed: + self._emit(Reconnecting(attempt=attempt + 1, reason=reason, model=self.model)) + delay = self._backoff.delay(attempt) + if delay: + await asyncio.sleep(delay) + attempt += 1 + if attempt > self._backoff.max_attempts or (self._now() - started) > self._backoff.max_elapsed: + self._fail( + RealtimeConnectionError( + f"reconnect failed after {attempt - 1} attempts ({reason})", + attempts=attempt - 1, + ) + ) + return + try: + await self._acquire_throttle() + try: + connection = await self._open_connection() + created = await self._await_session_created(connection) + finally: + self._throttle.release() + except Exception as exc: + status = handshake_status_of(exc) + kind = classify_handshake_status(status) if status is not None else FailureKind.RETRYABLE + if kind is FailureKind.FATAL_AUTH or kind is FailureKind.FATAL: + self._fail( + RealtimeConnectionError( + f"reconnect rejected with HTTP {status}", attempts=attempt, cause=exc + ) + ) + return + reason = f"handshake failed: {exc.__class__.__name__}" + continue + + # make-before-break: the new connection is live before the old + # socket is torn down; the epoch bump quarantines stale frames. + plan = self.state.replay_plan() + gap = self.state.consume_pending_gap() + if plan.gap_bytes or gap: + self._emit( + BufferGap( + dropped_seconds=self.state.to_seconds(plan.gap_bytes + gap), + reason="replay window capped", + ) + ) + self.state.begin_epoch(plan.start_offset) + self._sent_offset = plan.start_offset + self._replaying_until = self.state.write_head + self.state.metrics.replayed_bytes += max(0, self.state.write_head - plan.start_offset) + if plan.resend_commit and self.state.outstanding_commits: + boundary = max(self.state.outstanding_commits) + self._pending_controls = [(boundary, commit_event())] + self._attach(connection) + if old_connection is not None: + try: + await old_connection.close() + except Exception: + pass + self._emit( + Reconnected( + attempt=attempt, + replayed_seconds=self.state.to_seconds(self.state.write_head - plan.start_offset), + model=self.model, + ) + ) + self._writer_wakeup.set() + _created = created # session id available if callers need it later + return + except asyncio.CancelledError: + raise + finally: + self._reconnect_task = None + + async def _acquire_throttle(self) -> None: + while not self._throttle.acquire(blocking=False): + await asyncio.sleep(0.05 + random.random() * 0.1) + + # -- internals: watchdog ------------------------------------------------------- + + async def _watchdog_loop(self) -> None: + try: + next_echo = self._now() + self._jittered(self._echo_interval) + while not self._closed: + await asyncio.sleep(0.5) + now = self._now() + connection = self._connection + if connection is None or self._reconnect_task is not None: + continue + + # echo liveness probe: a live heartbeat means the server path is + # healthy; a missed one means it isn't — reconnect proactively + # (make-before-break) instead of waiting for TCP to notice. + if self._echo_pending_since is not None: + if now - self._echo_pending_since > self._echo_timeout: + self._echo_pending_since = None + self._schedule_reconnect("echo timeout") + # reconnect in flight: stale-stream/keepalive checks below + # are intentionally skipped this tick + continue + elif now >= next_echo: + next_echo = now + self._jittered(self._echo_interval) + self._echo_pending_since = now + try: + await connection.send(echo_event(echo_id=int(now * 1000))) + except Exception: + self._schedule_reconnect("echo send failed") + continue + + # stale stream: audio flowing, echo answering, but zero events — + # the decode path is wedged + audio_in_flight = self._sent_offset > 0 and self._last_append_at > self._last_server_event + if ( + audio_in_flight + and now - max(self._last_server_event, self._connected_at) > self._stale_stream_timeout + ): + self._schedule_reconnect("stale stream (audio flowing, no server events)") + continue # skip keepalive this tick; reconnect owns the connection now + + # keepalive silence (manual-commit mode only): defeat the server's + # 300s no-append idle timeout + if self._keepalive_silence and self._last_append_at: + if now - self._last_append_at > 200.0: + await self.append(b"\x00" * int(self.state.bps * 0.1)) # 100ms of silence + except asyncio.CancelledError: + raise + + def _jittered(self, interval: float) -> float: + return interval * (0.8 + 0.4 * random.random()) + + # -- internals: misc -------------------------------------------------------- + + def _now(self) -> float: + return asyncio.get_running_loop().time() + + def _emit(self, event: RealtimeSessionEvent) -> None: + if self._event_callback is not None: + try: + self._event_callback(event) + except Exception: + log.exception("realtime event callback raised") + else: + self._events.put_nowait(event) + + def _fail(self, error: BaseException) -> None: + if self._failure is None: + self._failure = error + self._closed = True + self._events.put_nowait(_SENTINEL) + for task in (self._watchdog_task, self._writer_task): + if task is not None: + task.cancel() + if self._connection is not None: + connection = self._connection + self._connection = None + asyncio.ensure_future(_close_quietly(connection)) + self._pool.unregister(self) + + def _flush_gap_events(self) -> None: + gap = self.state.consume_pending_gap() + if gap: + self._emit(BufferGap(dropped_seconds=self.state.to_seconds(gap), reason="recovery buffer overflow")) + + def _sanity_check_audio(self, pcm: bytes) -> None: + if not self._warned_riff and pcm[:4] == b"RIFF": + self._warned_riff = True + log.warning( + "append() received what looks like a WAV file (RIFF header); " + "the realtime API expects raw PCM frames — strip the WAV header first" + ) + if not self._warned_silence and len(pcm) >= self.state.bps and pcm.count(0) == len(pcm): + self._warned_silence = True + log.warning("append() has received >=1s of all-zero audio — is the microphone live?") + + +async def _close_quietly(connection: AsyncRealtimeConnection) -> None: + try: + await connection.close() + except Exception: + pass + + +class RealtimeTranscriptionSession: + """Sync facade over AsyncRealtimeTranscriptionSession. + + Runs the async session on a dedicated background event loop thread — one + thread per session, so this surface is intended for low session counts; + use the async client for high-concurrency deployments. + """ + + def __init__(self, **kwargs: Any) -> None: + self._loop = asyncio.new_event_loop() + self._thread = threading.Thread(target=self._run_loop, name="together-realtime", daemon=True) + self._thread.start() + self._kwargs = kwargs + self._session: Optional[AsyncRealtimeTranscriptionSession] = None + atexit.register(self.close) + + def _run_loop(self) -> None: + asyncio.set_event_loop(self._loop) + self._loop.run_forever() + + def _call(self, coro: Any, timeout: Optional[float] = None) -> Any: + future = asyncio.run_coroutine_threadsafe(coro, self._loop) + return future.result(timeout) + + def start(self) -> None: + async def _create() -> AsyncRealtimeTranscriptionSession: + session = AsyncRealtimeTranscriptionSession(**self._kwargs) + await session.start() + return session + + self._session = self._call(_create()) + + def __enter__(self) -> "RealtimeTranscriptionSession": + self.start() + return self + + def __exit__(self, exc_type: object, *_exc: object) -> None: + if self._session is not None and exc_type is None: + try: + self.flush() + except RealtimeError: + pass + self.close() + + def append(self, pcm: bytes) -> None: + assert self._session is not None, "session not started" + self._call(self._session.append(pcm)) + + def commit(self) -> None: + assert self._session is not None, "session not started" + self._call(self._session.commit()) + + def clear(self) -> None: + assert self._session is not None, "session not started" + self._call(self._session.clear()) + + def flush(self, *, quiescence: float = 2.0, timeout: float = 30.0) -> str: + assert self._session is not None, "session not started" + result = self._call(self._session.flush(quiescence=quiescence, timeout=timeout), timeout=timeout + 5) + return str(result) + + @property + def metrics(self) -> Any: + assert self._session is not None, "session not started" + return self._session.metrics + + def pending_audio(self) -> bytes: + assert self._session is not None, "session not started" + return self._session.pending_audio() + + def context_prompt(self, max_chars: int = 200) -> str: + assert self._session is not None, "session not started" + return self._session.context_prompt(max_chars) + + @property + def transcripts(self) -> List[str]: + assert self._session is not None, "session not started" + return self._session.transcripts + + def __iter__(self) -> Iterator[RealtimeSessionEvent]: + assert self._session is not None, "session not started" + session = self._session + while True: + try: + item = self._call(session._events.get()) + except Exception: + return + if item is _SENTINEL: + if session._failure is not None: + raise session._failure + return + yield item + + def close(self) -> None: + atexit.unregister(self.close) + if self._session is not None: + try: + self._call(self._session.close(), timeout=10) + except Exception: + pass + self._session = None + if self._loop.is_running(): + self._loop.call_soon_threadsafe(self._loop.stop) + if self._thread.is_alive(): + self._thread.join(timeout=5) diff --git a/src/together/lib/realtime/_state.py b/src/together/lib/realtime/_state.py new file mode 100644 index 000000000..ec6172afa --- /dev/null +++ b/src/together/lib/realtime/_state.py @@ -0,0 +1,565 @@ +from __future__ import annotations + +import enum +import random +import threading +from typing import Dict, List, Deque, Tuple, Callable, Iterator, Optional +from collections import deque +from dataclasses import dataclass + +from ._types import ( + RealtimeErrorInfo, + bytes_per_second, +) +from ._exceptions import RealtimeBufferOverflowError + +__all__ = [ + "FailureKind", + "classify_handshake_status", + "classify_fatal_error", + "BackoffPolicy", + "BufferPool", + "AudioBuffer", + "ReplayPlan", + "SegmentInfo", + "SessionMetrics", + "RecoveryState", + "iter_frames", +] + + +# --------------------------------------------------------------------------- +# Failure classification +# +# The server always closes with WS code 1000, so fatal-vs-retryable must be +# derived from (a) the HTTP status of a failed handshake and (b) the last +# fatal `...transcription.failed` frame's error payload. Some fatal +# conditions (bad audio format) are wrapped in a generic +# `service_unavailable_error` and are only identifiable by message text. +# --------------------------------------------------------------------------- + + +class FailureKind(enum.Enum): + RETRYABLE = "retryable" + FATAL = "fatal" + FATAL_AUTH = "fatal_auth" + IDLE_TIMEOUT = "idle_timeout" + + +_FATAL_CODES = {"model_not_available", "model_not_accessible"} +_AUTH_CODES = {"invalid_api_key", "missing_api_key"} +_FATAL_MESSAGE_MARKERS = ( + "unsupported format", + "unsupported input sample rate", + "invalid audio data format", +) +_IDLE_TIMEOUT_CODES = {"idle_timeout"} + + +def classify_handshake_status(status_code: Optional[int]) -> FailureKind: + """Classify a failed WebSocket handshake by its HTTP status (None = network error).""" + if status_code in (401, 403): + return FailureKind.FATAL_AUTH + # 429 and 5xx are transient; unknown statuses default to retryable so that a + # flaky proxy can't permanently kill a session. + return FailureKind.RETRYABLE + + +def classify_fatal_error(error: Optional[RealtimeErrorInfo]) -> FailureKind: + """Classify the error payload of a fatal `failed` frame (or a bare close).""" + if error is None: + return FailureKind.RETRYABLE + + code = (error.code or "").lower() + if code in _AUTH_CODES: + return FailureKind.FATAL_AUTH + if code in _FATAL_CODES: + return FailureKind.FATAL + if code in _IDLE_TIMEOUT_CODES: + return FailureKind.IDLE_TIMEOUT + + message = (error.message or "").lower() + if any(marker in message for marker in _FATAL_MESSAGE_MARKERS): + return FailureKind.FATAL + # The realtime idle-timeout path currently surfaces as a generic + # service_unavailable_error with "Timeout" in the message. + if "timeout" in message and (error.type or "") in ("service_unavailable_error", "request_timeout", ""): + return FailureKind.IDLE_TIMEOUT + + if (error.type or "") == "invalid_request_error": + return FailureKind.FATAL + return FailureKind.RETRYABLE + + +# --------------------------------------------------------------------------- +# Backoff +# --------------------------------------------------------------------------- + + +@dataclass +class BackoffPolicy: + """Exponential backoff with full jitter and deterministic-test injection.""" + + initial: float = 0.5 + maximum: float = 15.0 + max_attempts: int = 10 + max_elapsed: float = 120.0 + rng: Callable[[], float] = random.random # returns [0, 1) + + def delay(self, attempt: int) -> float: + """Full-jitter delay for a 0-indexed attempt number.""" + cap = min(self.maximum, self.initial * (2.0**attempt)) + return cap * self.rng() + + +# --------------------------------------------------------------------------- +# Global buffer pool +# +# One pool per client bounds total recovery-buffer memory across all sessions +# in the process. On exhaustion it reclaims from the largest holders first; +# each holder's reclaim callback trims its own buffer and surfaces a +# BufferGap to its consumer. +# --------------------------------------------------------------------------- + + +class BufferPool: + def __init__(self, max_bytes: Optional[int] = 256 * 1024 * 1024) -> None: + self.max_bytes = max_bytes + self._used = 0 + self._lock = threading.RLock() + # holder -> (held_bytes_fn, reclaim_fn(bytes_to_free) -> bytes_freed) + self._holders: Dict[object, Tuple[Callable[[], int], Callable[[int], int]]] = {} + + @property + def used_bytes(self) -> int: + return self._used + + def register(self, holder: object, held_bytes: Callable[[], int], reclaim: Callable[[int], int]) -> None: + with self._lock: + self._holders[holder] = (held_bytes, reclaim) + + def unregister(self, holder: object) -> None: + with self._lock: + self._holders.pop(holder, None) + + def charge(self, n: int) -> None: + if n <= 0: + return + with self._lock: + if self.max_bytes is not None and self._used + n > self.max_bytes: + self._reclaim_locked(self._used + n - self.max_bytes) + self._used += n + + def release(self, n: int) -> None: + if n <= 0: + return + with self._lock: + self._used = max(0, self._used - n) + + def _reclaim_locked(self, deficit: int) -> None: + # Largest holders first; each reclaim releases pool bytes via the + # holder's own buffer, which calls back into release(). + holders = sorted(self._holders.values(), key=lambda h: h[0](), reverse=True) + for _held, reclaim in holders: + if deficit <= 0: + return + deficit -= reclaim(deficit) + + +# --------------------------------------------------------------------------- +# Audio ring buffer with global offsets +# --------------------------------------------------------------------------- + + +class AudioBuffer: + """Rolling buffer of appended PCM keyed by global byte offsets. + + Offsets are monotonically increasing over the life of the session; the + buffer retains the byte range [start_offset, end_offset). Raw PCM is + stored (base64 encoding happens only at send time). + """ + + def __init__(self, pool: Optional[BufferPool] = None) -> None: + self._chunks: Deque[Tuple[int, bytes]] = deque() # (global_offset, data) + self._start = 0 + self._end = 0 + self._pool = pool + + @property + def start_offset(self) -> int: + return self._start + + @property + def end_offset(self) -> int: + return self._end + + @property + def size(self) -> int: + return self._end - self._start + + def append(self, data: bytes) -> int: + """Append PCM bytes; returns the global offset of the first byte.""" + offset = self._end + if data: + self._chunks.append((offset, data)) + self._end += len(data) + if self._pool is not None: + self._pool.charge(len(data)) + return offset + + def trim_to(self, offset: int) -> int: + """Drop retained bytes below `offset`; returns bytes freed.""" + target = min(max(offset, self._start), self._end) + freed = 0 + while self._chunks: + chunk_start, data = self._chunks[0] + chunk_end = chunk_start + len(data) + if chunk_end <= target: + self._chunks.popleft() + freed += len(data) + elif chunk_start < target: + keep = data[target - chunk_start :] + self._chunks[0] = (target, keep) + freed += len(data) - len(keep) + break + else: + break + self._start = max(self._start, target) + if self._chunks: + # start_offset never exceeds the first retained chunk + self._start = min(self._start, self._chunks[0][0]) + else: + self._start = self._end + if freed and self._pool is not None: + self._pool.release(freed) + return freed + + def read_from(self, offset: int) -> Iterator[bytes]: + """Yield retained data from `offset` (clamped to what's available).""" + for chunk_start, data in self._chunks: + chunk_end = chunk_start + len(data) + if chunk_end <= offset: + continue + if chunk_start >= offset: + yield data + else: + yield data[offset - chunk_start :] + + def clear(self) -> int: + return self.trim_to(self._end) + + +# --------------------------------------------------------------------------- +# Recovery state +# --------------------------------------------------------------------------- + + +@dataclass +class ReplayPlan: + start_offset: int + """Global byte offset replay should start from.""" + gap_bytes: int = 0 + """Un-recovered bytes that were lost (trimmed/skipped) before start_offset.""" + resend_commit: bool = False + """Whether an outstanding manual commit must be re-issued after replay.""" + + +@dataclass +class SegmentInfo: + segment_id: str + audio_start: float + replayed: bool = False + + +@dataclass +class SessionMetrics: + """Counters for observability. + + Ownership is split by layer: the state maintains reconnects (via + begin_epoch), completed/delta counts, buffered/watermark byte gauges and + gap_bytes; the session layer maintains replayed_bytes (it owns replay). + """ + + reconnects: int = 0 + replayed_bytes: int = 0 + gap_bytes: int = 0 + completed_events: int = 0 + delta_events: int = 0 + buffered_bytes: int = 0 + watermark_lag_bytes: int = 0 + + +_ANON_ITEM = "~anonymous~" + + +class RecoveryState: + """Sans-I/O bookkeeping shared by the sync and async transcription sessions. + + The offset line (global appended-audio byte offsets, all sample-aligned): + + 0 .. buffer.start <= replay/trim point <= watermark <= anchor <= write_head + | | | | + | | | newest appended byte + | | append offset when the last + | | `completed` ARRIVED + | transcribed position from the last + | completed's start+duration + max(watermark_or_anchor - replay_margin, + write_head - max_replay_seconds) + + `replay_plan` computes the replay start and `_safe_trim_point` the trim + floor from the same precedence: watermark > anchor > everything retained, + clamped so outstanding manual commits stay replayable. + + Owns: the audio buffer, the last-completed replay anchor, the optional + server `processed_ms` watermark, the manual-commit ledger, connection + epochs, and segment identity. All byte offsets are global (survive + reconnects); conversions to seconds use the negotiated audio format. + """ + + def __init__( + self, + *, + input_audio_format: str, + replay_margin: float = 0.0, + max_replay_seconds: Optional[float] = 5.0, + max_seconds: float = 120.0, + overflow: str = "drop_oldest", + pool: Optional[BufferPool] = None, + ) -> None: + self.bps = bytes_per_second(input_audio_format) + self.block_align = 2 # 16-bit mono PCM: every cut must land on a sample + self.buffer = AudioBuffer(pool) + self.replay_margin_bytes = int(replay_margin * self.bps) + self.max_replay_seconds = max_replay_seconds + self.max_bytes = int(max_seconds * self.bps) + self.overflow = overflow + + self.epoch = 0 + self.conn_base = 0 # global offset of the current connection's timeline zero + self.anchor: Optional[int] = None # append offset at last completed arrival + self.watermark: Optional[int] = None # global offset from processed_ms (supersedes anchor) + self.outstanding_commits: List[int] = [] # append offsets of un-retired manual commits + self.metrics = SessionMetrics() + self.pending_gap_bytes = 0 # overflow-trimmed unsafe bytes not yet surfaced + + self._segments: Dict[Tuple[int, str], SegmentInfo] = {} + self._next_segment = 0 + + # -- unit helpers ------------------------------------------------------- + + def to_seconds(self, nbytes: int) -> float: + return nbytes / self.bps + + def to_bytes(self, seconds: float) -> int: + return int(seconds * self.bps) + + def _align(self, offset: int) -> int: + """Align a byte offset down to a sample boundary. + + Replay/trim cuts at odd offsets would byte-shift every subsequent + 16-bit sample, turning replayed speech into static (and poisoning the + rest of the stream on that connection). + """ + return max(0, offset - (offset % self.block_align)) + + @property + def write_head(self) -> int: + return self.buffer.end_offset + + # -- append/commit path ------------------------------------------------- + + def record_append(self, data: bytes) -> int: + offset = self.buffer.append(data) + self._enforce_cap() + self.metrics.buffered_bytes = self.buffer.size + return offset + + def record_commit(self) -> None: + self.outstanding_commits.append(self.write_head) + + def record_clear(self) -> None: + # Server drops its buffered audio and resets offsets; mirror that. + self.buffer.clear() + self.anchor = None + self.watermark = None + self.outstanding_commits.clear() + self.conn_base = self.write_head + + # -- server event path --------------------------------------------------- + + def record_completed(self, processed_seconds: Optional[float] = None) -> None: + """Called when a `completed` frame arrives; anchors recovery at 'now'. + + `processed_seconds` is the event's `start + duration` when the server + provided both: its position on the connection's audio timeline marks + how much audio has been transcribed. The server derives it from a + consumed-speech clock (silence gaps are skipped), so it is treated as + approximate — the replay margin absorbs the drift. + """ + self.anchor = self.write_head + self.metrics.completed_events += 1 + if processed_seconds is not None: + self._advance_watermark(self.conn_base + self.to_bytes(processed_seconds)) + # Commits at or before the anchor are covered (a commit yields 0 or 1 + # completed, and completeds are FIFO — anything later will re-anchor). + self.outstanding_commits = [c for c in self.outstanding_commits if c > self.anchor] + self._trim_safe() + + def record_processed(self, processed_ms: float) -> None: + """Called on an `input_audio_buffer.processed` watermark event, if the + server ever emits one (not sent today; completed.start+duration is the + production watermark source).""" + self._advance_watermark(self.conn_base + int(processed_ms / 1000.0 * self.bps)) + self._trim_safe() + + def _advance_watermark(self, candidate: int) -> None: + new_mark = self._align(min(candidate, self.write_head)) + self.watermark = new_mark if self.watermark is None else max(self.watermark, new_mark) + + # -- trimming ------------------------------------------------------------ + + def _safe_trim_point(self) -> Optional[int]: + # The watermark rides the server's consumed-speech clock and the + # anchor includes decode latency; both are approximate, so both keep + # the replay margin as a safety buffer. + if self.watermark is not None: + point: Optional[int] = self.watermark - self.replay_margin_bytes + elif self.anchor is not None: + point = self.anchor - self.replay_margin_bytes + else: + point = None + if point is None: + return None + # Never trim past an outstanding manual commit's replay window. + if self.outstanding_commits: + point = min(point, min(self.outstanding_commits) - self.replay_margin_bytes) + return self._align(max(point, 0)) + + def _trim_safe(self) -> None: + point = self._safe_trim_point() + if point is not None: + self.buffer.trim_to(point) + self.metrics.buffered_bytes = self.buffer.size + if self.watermark is not None: + self.metrics.watermark_lag_bytes = max(0, self.write_head - self.watermark) + + def _enforce_cap(self) -> None: + excess = self.buffer.size - self.max_bytes + if excess <= 0: + return + target = self._align(self.buffer.start_offset + excess) + safe = self._safe_trim_point() + unsafe_dropped = max(0, target - safe) if safe is not None else excess + if unsafe_dropped > 0 and self.overflow == "error": + raise RealtimeBufferOverflowError( + f"realtime recovery buffer exceeded {self.to_seconds(self.max_bytes):.1f}s " + f"of retained audio and overflow policy is 'error'", + dropped_seconds=self.to_seconds(unsafe_dropped), + ) + self.buffer.trim_to(target) + if unsafe_dropped > 0: + # Un-recovered audio was lost; surfaced by the session as BufferGap. + self.pending_gap_bytes += unsafe_dropped + self.metrics.gap_bytes += unsafe_dropped + + def reclaim(self, nbytes: int) -> int: + """Pool-initiated reclamation (correlated-outage pressure). Oldest first.""" + target = self._align(min(self.buffer.start_offset + nbytes, self.buffer.end_offset)) + safe = self._safe_trim_point() + unsafe = max(0, target - safe) if safe is not None else target - self.buffer.start_offset + freed = self.buffer.trim_to(target) + if unsafe > 0: + self.pending_gap_bytes += unsafe + self.metrics.gap_bytes += unsafe + self.metrics.buffered_bytes = self.buffer.size + return freed + + def consume_pending_gap(self) -> int: + gap, self.pending_gap_bytes = self.pending_gap_bytes, 0 + return gap + + # -- reconnect ----------------------------------------------------------- + + def replay_plan(self) -> ReplayPlan: + """Compute where replay should start for a fresh connection. + + Replay starts at max(head - max_replay_seconds, last transcribed + position): never re-send transcribed audio, never replay more than + `max_replay_seconds` (default 5) — older untranscribed audio is skipped + and reported as a gap. The transcribed position comes from + completed.start + duration (watermark), falling back to the completed + arrival anchor; `replay_margin` (default 0) optionally rewinds further + for pre-roll. `max_replay_seconds=None` removes the cap. + """ + head = self.write_head + if self.watermark is not None: + desired = self.watermark - self.replay_margin_bytes + elif self.anchor is not None: + desired = self.anchor - self.replay_margin_bytes + else: + desired = self.buffer.start_offset + if self.outstanding_commits: + desired = min(desired, min(self.outstanding_commits) - self.replay_margin_bytes) + desired = max(desired, 0) + + gap = 0 + if self.max_replay_seconds is not None: + lag_floor = head - self.to_bytes(self.max_replay_seconds) + if lag_floor > desired: + gap += lag_floor - desired + desired = lag_floor + + if desired < self.buffer.start_offset: + gap += self.buffer.start_offset - desired + desired = self.buffer.start_offset + desired = self._align(desired) + + return ReplayPlan( + start_offset=desired, + gap_bytes=gap, + resend_commit=bool(self.outstanding_commits), + ) + + def begin_epoch(self, replay_start: int) -> int: + """Start a new connection whose server timeline zero is `replay_start`.""" + self.epoch += 1 + self.conn_base = replay_start + self.watermark = None # per-connection; re-established by new events + if self.epoch > 1: + self.metrics.reconnects += 1 + return self.epoch + + # -- segment identity ---------------------------------------------------- + + def segment_for(self, item_id: Optional[str], *, replayed: bool) -> SegmentInfo: + key = (self.epoch, item_id or _ANON_ITEM) + info = self._segments.get(key) + if info is None: + self._next_segment += 1 + # audio_start approximates the segment's position on the global + # appended timeline: the last recovery anchor when the segment was + # first observed (server start/duration are not timeline-reliable). + start_offset = self.anchor if self.anchor is not None else self.conn_base + info = SegmentInfo( + segment_id=f"seg_{self._next_segment}", + audio_start=self.to_seconds(start_offset), + replayed=replayed, + ) + self._segments[key] = info + return info + + def close_segment(self, item_id: Optional[str]) -> None: + self._segments.pop((self.epoch, item_id or _ANON_ITEM), None) + + +# --------------------------------------------------------------------------- +# Frame chunking +# --------------------------------------------------------------------------- + + +def iter_frames(data: bytes, frame_bytes: int) -> Iterator[bytes]: + """Split PCM into frames of at most frame_bytes (no padding).""" + view = memoryview(data) + for start in range(0, len(view), frame_bytes): + yield bytes(view[start : start + frame_bytes]) diff --git a/src/together/lib/realtime/_types.py b/src/together/lib/realtime/_types.py new file mode 100644 index 000000000..ff4d86a5c --- /dev/null +++ b/src/together/lib/realtime/_types.py @@ -0,0 +1,383 @@ +from __future__ import annotations + +import json +import base64 +from typing import Any, Dict, Union, Mapping, Optional, cast +from typing_extensions import Literal, Required, Annotated, TypeAlias, TypedDict + +from ..._utils import PropertyInfo +from ..._models import BaseModel, construct_type_unchecked + +__all__ = [ + # audio formats + "AUDIO_FORMATS", + "bytes_per_second", + # wire (server -> client) events + "RealtimeErrorInfo", + "RealtimeSessionInfo", + "SessionCreatedEvent", + "TranscriptionDeltaEvent", + "TranscriptionCompletedEvent", + "TranscriptionFailedEvent", + "TurnEvent", + "InputAudioBufferProcessedEvent", + "EchoResponseEvent", + "UnknownEvent", + "RealtimeServerEvent", + "parse_server_event", + # client -> server event builders + "append_event", + "commit_event", + "clear_event", + "session_update_event", + "echo_event", + # options + "TurnDetectionParam", + "ReconnectOptions", + "BufferOptions", + # normalized session events + "SessionStarted", + "TranscriptDelta", + "TranscriptCompleted", + "TranscriptFailed", + "Reconnecting", + "Reconnected", + "BufferGap", + "RealtimeSessionEvent", +] + +# --------------------------------------------------------------------------- +# Audio formats +# +# Bytes/second for each `input_audio_format` accepted by the server. Note the +# bare `pcm16`/`pcm_s16`/`pcm_s16le` aliases are 24 kHz on the server side — +# the SDK defaults to the explicit `pcm_s16le_16000` to avoid that surprise. +# --------------------------------------------------------------------------- + +AUDIO_FORMATS: Dict[str, int] = { + "pcm_s16le_8000": 8_000 * 2, + "pcm_s16le_16000": 16_000 * 2, + "pcm_s16le_24000": 24_000 * 2, + "pcm16": 24_000 * 2, + "pcm_s16": 24_000 * 2, + "pcm_s16le": 24_000 * 2, +} + +DEFAULT_AUDIO_FORMAT = "pcm_s16le_16000" + + +def bytes_per_second(input_audio_format: str) -> int: + try: + return AUDIO_FORMATS[input_audio_format] + except KeyError: + raise ValueError( + f"Unsupported input_audio_format {input_audio_format!r}; expected one of {sorted(AUDIO_FORMATS)}" + ) from None + + +def sample_rate_of(input_audio_format: str) -> int: + return bytes_per_second(input_audio_format) // 2 + + +# --------------------------------------------------------------------------- +# Wire events (server -> client) +# +# Field presence is deliberately loose: the server emits several shapes for +# the same event type (e.g. `completed` without item_id/start/duration on some +# backend paths), and events must parse under both pydantic v1 and v2 — hence +# construct_type_unchecked + explicit dispatch instead of strict validation. +# --------------------------------------------------------------------------- + + +class RealtimeErrorInfo(BaseModel): + message: Optional[str] = None + type: Optional[str] = None + param: Optional[str] = None + code: Optional[str] = None + + +class RealtimeSessionInfo(BaseModel): + id: Optional[str] = None + object: Optional[str] = None + model: Optional[str] = None + modalities: Optional[Any] = None + + +class SessionCreatedEvent(BaseModel): + type: Literal["session.created"] + event_id: Optional[str] = None + session: Optional[RealtimeSessionInfo] = None + + +class TranscriptionDeltaEvent(BaseModel): + type: Literal["conversation.item.input_audio_transcription.delta"] + item_id: Optional[str] = None + delta: str = "" + start: Optional[float] = None + """Server-side consumed-speech clock — NOT a position on the appended-audio + timeline (silence is skipped); do not use for buffer accounting.""" + duration: Optional[float] = None + + +class TranscriptionCompletedEvent(BaseModel): + type: Literal["conversation.item.input_audio_transcription.completed"] + item_id: Optional[str] = None + transcript: str = "" + start: Optional[float] = None + """See TranscriptionDeltaEvent.start — informational only.""" + duration: Optional[float] = None + + +class TranscriptionFailedEvent(BaseModel): + type: Literal["conversation.item.input_audio_transcription.failed"] + item_id: Optional[str] = None + event_type: Optional[str] = None + """Present on per-turn failures ("delta"/"completed"); absent on fatal + session errors (which are followed by a server-side close).""" + error: Optional[RealtimeErrorInfo] = None + duration: Optional[float] = None + + @property + def is_fatal(self) -> bool: + return self.event_type is None + + +class TurnEvent(BaseModel): + type: Literal[ + "conversation.item.input_audio_transcription.start_of_turn", + "conversation.item.input_audio_transcription.end_of_turn", + "conversation.item.input_audio_transcription.eager_end_of_turn", + "conversation.item.input_audio_transcription.turn_resumed", + ] + item_id: Optional[str] = None + transcript: Optional[str] = None + end_of_turn_confidence: Optional[float] = None + + +class InputAudioBufferProcessedEvent(BaseModel): + """Trim-safety watermark on the appended-audio timeline (newer servers only). + + Means: the server has finished all decodes covering audio at or before + `processed_ms` and will never re-read earlier bytes. It does NOT imply a + transcript exists for all of that audio. + """ + + type: Literal["input_audio_buffer.processed"] + processed_ms: float + + +class EchoResponseEvent(BaseModel): + type: Literal["echo.response"] + echo_id: Optional[Any] = None + client_sent_at: Optional[Any] = None + server_received_at: Optional[float] = None + server_sent_at: Optional[float] = None + payload: Optional[Any] = None + + +class UnknownEvent(BaseModel): + """Forward-compat catch-all for event types this SDK version doesn't know.""" + + type: str + data: Dict[str, Any] + + +# Public type alias for annotating connect()-level raw events. Parse dispatch +# uses the _EVENT_TYPES dict below, NOT this union (repo convention: no native +# pydantic discriminators, they differ across v1/v2) — don't wire it up or +# delete it; it exists purely as a typing surface for SDK consumers. +RealtimeServerEvent: TypeAlias = Annotated[ + Union[ + SessionCreatedEvent, + TranscriptionDeltaEvent, + TranscriptionCompletedEvent, + TranscriptionFailedEvent, + TurnEvent, + InputAudioBufferProcessedEvent, + EchoResponseEvent, + UnknownEvent, + ], + PropertyInfo(discriminator="type"), +] + +_EVENT_TYPES: Dict[str, type] = { + "session.created": SessionCreatedEvent, + "conversation.item.input_audio_transcription.delta": TranscriptionDeltaEvent, + "conversation.item.input_audio_transcription.completed": TranscriptionCompletedEvent, + "conversation.item.input_audio_transcription.failed": TranscriptionFailedEvent, + "conversation.item.input_audio_transcription.start_of_turn": TurnEvent, + "conversation.item.input_audio_transcription.end_of_turn": TurnEvent, + "conversation.item.input_audio_transcription.eager_end_of_turn": TurnEvent, + "conversation.item.input_audio_transcription.turn_resumed": TurnEvent, + "input_audio_buffer.processed": InputAudioBufferProcessedEvent, + "echo.response": EchoResponseEvent, +} + + +def parse_server_event(payload: Union[str, bytes, Mapping[str, Any]]) -> object: + """Parse a wire frame into a typed server event. + + Dispatches explicitly on `type` (deterministic under both pydantic majors) + and falls back to UnknownEvent for unrecognized types. + """ + raw: Any = json.loads(payload) if isinstance(payload, (str, bytes)) else dict(payload) + if not isinstance(raw, dict): + return UnknownEvent(type="", data={"value": raw}) + data = cast("Dict[str, Any]", raw) + event_type = data.get("type") + model = _EVENT_TYPES.get(event_type) if isinstance(event_type, str) else None + if model is None: + return UnknownEvent(type=str(event_type), data=data) + return cast(object, construct_type_unchecked(value=data, type_=model)) + + +# --------------------------------------------------------------------------- +# Client events (client -> server) — plain dict builders +# --------------------------------------------------------------------------- + + +def append_event(pcm: Union[bytes, bytearray, memoryview]) -> Dict[str, Any]: + return {"type": "input_audio_buffer.append", "audio": base64.b64encode(bytes(pcm)).decode("ascii")} + + +def commit_event() -> Dict[str, Any]: + return {"type": "input_audio_buffer.commit"} + + +def clear_event() -> Dict[str, Any]: + return {"type": "input_audio_buffer.clear"} + + +def session_update_event(session: Mapping[str, Any]) -> Dict[str, Any]: + # The server listens for the ".updated" variant (not ".update"). + return {"type": "transcription_session.updated", "session": dict(session)} + + +def echo_event(echo_id: Any = None, payload: Any = None) -> Dict[str, Any]: + event: Dict[str, Any] = {"type": "echo"} + if echo_id is not None: + event["echo_id"] = echo_id + if payload is not None: + event["payload"] = payload + return event + + +# --------------------------------------------------------------------------- +# Options +# --------------------------------------------------------------------------- + + +class TurnDetectionParam(TypedDict, total=False): + type: Required[Literal["server_vad", "none"]] + threshold: float + min_silence_duration_ms: int + min_speech_duration_ms: int + max_speech_duration_s: float + speech_pad_ms: int + # Deepgram backends + eot_threshold: float + eot_timeout_ms: int + eager_eot_threshold: float + + +class ReconnectOptions(TypedDict, total=False): + max_attempts: int + """Maximum consecutive reconnect attempts before giving up (default 10).""" + max_elapsed: float + """Maximum seconds spent in a single reconnect episode (default 120).""" + backoff_initial: float + """Initial backoff delay in seconds (default 0.5).""" + backoff_max: float + """Backoff delay ceiling in seconds (default 15).""" + + +class BufferOptions(TypedDict, total=False): + max_seconds: float + """Outer bound on retained audio per session (default 120).""" + overflow: Literal["drop_oldest", "error"] + """What to do when retained audio exceeds max_seconds (default drop_oldest, + which surfaces a BufferGap event).""" + replay_margin: float + """Optional pre-roll rewound before the last transcribed position when + replaying (default 0.0).""" + max_replay_seconds: Optional[float] + """Replay starts at max(head - max_replay_seconds, last transcribed + position). Default 5.0; 0 resumes live without any replay; None removes + the cap (replay the full untranscribed window).""" + + +# --------------------------------------------------------------------------- +# Normalized session events (Layer 2) +# +# segment_id and audio_start/audio_end are SDK-computed from global appended +# audio offsets: stable across reconnects, unlike server item_ids. +# --------------------------------------------------------------------------- + + +class SessionStarted(BaseModel): + type: Literal["session.started"] = "session.started" + session_id: Optional[str] = None + model: Optional[str] = None + epoch: int = 0 + + +class TranscriptDelta(BaseModel): + type: Literal["transcript.delta"] = "transcript.delta" + segment_id: str + text: str + audio_start: Optional[float] = None + audio_end: Optional[float] = None + replayed: bool = False + raw: Optional[TranscriptionDeltaEvent] = None + + +class TranscriptCompleted(BaseModel): + type: Literal["transcript.completed"] = "transcript.completed" + segment_id: str + text: str + audio_start: Optional[float] = None + audio_end: Optional[float] = None + replayed: bool = False + raw: Optional[TranscriptionCompletedEvent] = None + + +class TranscriptFailed(BaseModel): + type: Literal["transcript.failed"] = "transcript.failed" + segment_id: Optional[str] = None + message: Optional[str] = None + replayed: bool = False + raw: Optional[TranscriptionFailedEvent] = None + + +class Reconnecting(BaseModel): + type: Literal["reconnecting"] = "reconnecting" + attempt: int + reason: str + model: Optional[str] = None + """The endpoint this attempt will target (rotates through fallbacks).""" + + +class Reconnected(BaseModel): + type: Literal["reconnected"] = "reconnected" + attempt: int + replayed_seconds: float + model: Optional[str] = None + """The endpoint now serving the session (differs from the primary when a + fallback model took over).""" + + +class BufferGap(BaseModel): + type: Literal["buffer.gap"] = "buffer.gap" + dropped_seconds: float + reason: str + + +RealtimeSessionEvent: TypeAlias = Union[ + SessionStarted, + TranscriptDelta, + TranscriptCompleted, + TranscriptFailed, + Reconnecting, + Reconnected, + BufferGap, +] diff --git a/src/together/realtime.py b/src/together/realtime.py new file mode 100644 index 000000000..e16a89175 --- /dev/null +++ b/src/together/realtime.py @@ -0,0 +1,73 @@ +"""Public types for the realtime transcription API (`client.beta.realtime`). + +Import events, options, and exceptions from here: + + from together.realtime import TranscriptCompleted, RealtimeConnectionError + +This module is the stable import surface; the implementation lives in +`together.lib.realtime` and may be reorganized without notice. +""" + +from .lib.realtime import ( + BufferGap as BufferGap, + TurnEvent as TurnEvent, + Reconnected as Reconnected, + Reconnecting as Reconnecting, + UnknownEvent as UnknownEvent, + BufferOptions as BufferOptions, + RealtimeError as RealtimeError, + SessionStarted as SessionStarted, + TranscriptDelta as TranscriptDelta, + ReconnectOptions as ReconnectOptions, + TranscriptFailed as TranscriptFailed, + EchoResponseEvent as EchoResponseEvent, + RealtimeErrorInfo as RealtimeErrorInfo, + TurnDetectionParam as TurnDetectionParam, + RealtimeServerEvent as RealtimeServerEvent, + SessionCreatedEvent as SessionCreatedEvent, + TranscriptCompleted as TranscriptCompleted, + RealtimeSessionError as RealtimeSessionError, + RealtimeSessionEvent as RealtimeSessionEvent, + RealtimeConnectionError as RealtimeConnectionError, + TranscriptionDeltaEvent as TranscriptionDeltaEvent, + RealtimeIdleTimeoutError as RealtimeIdleTimeoutError, + TranscriptionFailedEvent as TranscriptionFailedEvent, + RealtimeInvalidStateError as RealtimeInvalidStateError, + RealtimeBufferOverflowError as RealtimeBufferOverflowError, + TranscriptionCompletedEvent as TranscriptionCompletedEvent, + InputAudioBufferProcessedEvent as InputAudioBufferProcessedEvent, +) + +__all__ = [ + # events (wire) + "RealtimeServerEvent", + "SessionCreatedEvent", + "TranscriptionDeltaEvent", + "TranscriptionCompletedEvent", + "TranscriptionFailedEvent", + "TurnEvent", + "InputAudioBufferProcessedEvent", + "EchoResponseEvent", + "UnknownEvent", + "RealtimeErrorInfo", + # events (normalized session) + "RealtimeSessionEvent", + "SessionStarted", + "TranscriptDelta", + "TranscriptCompleted", + "TranscriptFailed", + "Reconnecting", + "Reconnected", + "BufferGap", + # options + "TurnDetectionParam", + "ReconnectOptions", + "BufferOptions", + # exceptions + "RealtimeError", + "RealtimeConnectionError", + "RealtimeSessionError", + "RealtimeIdleTimeoutError", + "RealtimeBufferOverflowError", + "RealtimeInvalidStateError", +] diff --git a/src/together/resources/beta/beta.py b/src/together/resources/beta/beta.py index 9a6121e5d..a84c5d444 100644 --- a/src/together/resources/beta/beta.py +++ b/src/together/resources/beta/beta.py @@ -2,6 +2,8 @@ from __future__ import annotations +from typing import TYPE_CHECKING + from .jig.jig import ( JigResource, AsyncJigResource, @@ -37,6 +39,9 @@ AsyncEndpointsResourceWithStreamingResponse, ) +if TYPE_CHECKING: + from ..realtime import RealtimeResource, AsyncRealtimeResource + __all__ = ["BetaResource", "AsyncBetaResource"] @@ -53,6 +58,14 @@ def models(self) -> ModelsResource: def jig(self) -> JigResource: return JigResource(self._client) + # Handwritten (not generated): realtime transcription over WebSocket. + # Guarded by tests/unit/test_realtime_wiring.py against regen drops. + @cached_property + def realtime(self) -> RealtimeResource: + from ..realtime import RealtimeResource + + return RealtimeResource(self._client) + @cached_property def clusters(self) -> ClustersResource: return ClustersResource(self._client) @@ -90,6 +103,14 @@ def models(self) -> AsyncModelsResource: def jig(self) -> AsyncJigResource: return AsyncJigResource(self._client) + # Handwritten (not generated): realtime transcription over WebSocket. + # Guarded by tests/unit/test_realtime_wiring.py against regen drops. + @cached_property + def realtime(self) -> AsyncRealtimeResource: + from ..realtime import AsyncRealtimeResource + + return AsyncRealtimeResource(self._client) + @cached_property def clusters(self) -> AsyncClustersResource: return AsyncClustersResource(self._client) diff --git a/src/together/resources/realtime.py b/src/together/resources/realtime.py new file mode 100644 index 000000000..71ff2846d --- /dev/null +++ b/src/together/resources/realtime.py @@ -0,0 +1,215 @@ +"""Realtime transcription resource (handwritten; not generated by Stainless). + +Thin shim following the files/fine_tuning precedent: all logic lives in +`together.lib.realtime` so it survives OpenAPI regeneration. +""" + +from __future__ import annotations + +import threading +from typing import Any, Mapping, Callable, Optional + +from .._resource import SyncAPIResource, AsyncAPIResource +from ..lib.realtime._state import BufferPool +from ..lib.realtime._types import ( + DEFAULT_AUDIO_FORMAT, + BufferOptions, + ReconnectOptions, + TurnDetectionParam, + RealtimeSessionEvent, +) +from ..lib.realtime._session import ( + RealtimeTranscriptionSession, + AsyncRealtimeTranscriptionSession, +) +from ..lib.realtime._connection import ( + RealtimeConnectionManager, + AsyncRealtimeConnectionManager, +) + +__all__ = ["RealtimeResource", "AsyncRealtimeResource"] + + +class RealtimeResource(SyncAPIResource): + """`client.realtime` — realtime transcription over WebSocket (sync).""" + + _pool: Optional[BufferPool] = None + _pool_lock = threading.Lock() + + @property + def buffer_pool(self) -> BufferPool: + """Client-scoped recovery-buffer pool shared by this client's sessions.""" + if self._pool is None: + with self._pool_lock: + if self._pool is None: + self._pool = BufferPool() + return self._pool + + def connect( + self, + *, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + turn_detection: Optional[TurnDetectionParam] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ) -> RealtimeConnectionManager: + """Advanced: open a raw typed realtime connection with no retry logic. + + Most callers should use `transcription()` instead, which adds + automatic reconnection with audio replay. + """ + return RealtimeConnectionManager( + client=self._client, + model=model, + input_audio_format=input_audio_format, + turn_detection=turn_detection, + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + extra_query=extra_query, + extra_headers=extra_headers, + ) + + def transcription( + self, + *, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + sample_rate: Optional[int] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + turn_detection: Optional[TurnDetectionParam] = None, + reconnect: Optional[ReconnectOptions] = None, + buffer: Optional[BufferOptions] = None, + keepalive_silence: bool = False, + reprime_prompt: bool = False, + max_chunk_ms: float = 1000.0, + event_callback: Optional[Callable[[RealtimeSessionEvent], None]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ) -> RealtimeTranscriptionSession: + """Recommended: an auto-reconnecting realtime transcription session. + + Runs on a background event-loop thread (one per session) — intended + for low session counts; use the async client for high concurrency. + """ + return RealtimeTranscriptionSession( + client=self._client, + model=model, + input_audio_format=input_audio_format, + sample_rate=sample_rate, + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + turn_detection=turn_detection, + reconnect=reconnect, + buffer=buffer, + keepalive_silence=keepalive_silence, + reprime_prompt=reprime_prompt, + max_chunk_ms=max_chunk_ms, + pool=self.buffer_pool, + event_callback=event_callback, + extra_query=extra_query, + extra_headers=extra_headers, + ) + + +class AsyncRealtimeResource(AsyncAPIResource): + """`client.realtime` — realtime transcription over WebSocket (async).""" + + _pool: Optional[BufferPool] = None + _pool_lock = threading.Lock() + + @property + def buffer_pool(self) -> BufferPool: + if self._pool is None: + with self._pool_lock: + if self._pool is None: + self._pool = BufferPool() + return self._pool + + def connect( + self, + *, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + turn_detection: Optional[TurnDetectionParam] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ) -> AsyncRealtimeConnectionManager: + """Advanced: open a raw typed realtime connection with no retry logic.""" + return AsyncRealtimeConnectionManager( + client=self._client, + model=model, + input_audio_format=input_audio_format, + turn_detection=turn_detection, + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + extra_query=extra_query, + extra_headers=extra_headers, + ) + + def transcription( + self, + *, + model: str, + input_audio_format: str = DEFAULT_AUDIO_FORMAT, + sample_rate: Optional[int] = None, + language: Optional[str] = None, + prompt: Optional[str] = None, + rolling_prompt: Optional[bool] = None, + energy_gate_rms: Optional[float] = None, + session_params: Optional[Mapping[str, Any]] = None, + turn_detection: Optional[TurnDetectionParam] = None, + reconnect: Optional[ReconnectOptions] = None, + buffer: Optional[BufferOptions] = None, + keepalive_silence: bool = False, + reprime_prompt: bool = False, + max_chunk_ms: float = 1000.0, + event_callback: Optional[Callable[[RealtimeSessionEvent], None]] = None, + extra_query: Optional[Mapping[str, Any]] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ) -> AsyncRealtimeTranscriptionSession: + """Recommended: an auto-reconnecting realtime transcription session.""" + return AsyncRealtimeTranscriptionSession( + client=self._client, + model=model, + input_audio_format=input_audio_format, + sample_rate=sample_rate, + language=language, + prompt=prompt, + rolling_prompt=rolling_prompt, + energy_gate_rms=energy_gate_rms, + session_params=session_params, + turn_detection=turn_detection, + reconnect=reconnect, + buffer=buffer, + keepalive_silence=keepalive_silence, + reprime_prompt=reprime_prompt, + max_chunk_ms=max_chunk_ms, + pool=self.buffer_pool, + event_callback=event_callback, + extra_query=extra_query, + extra_headers=extra_headers, + ) diff --git a/tests/unit/test_realtime_events.py b/tests/unit/test_realtime_events.py new file mode 100644 index 000000000..76d0e692f --- /dev/null +++ b/tests/unit/test_realtime_events.py @@ -0,0 +1,147 @@ +from __future__ import annotations + +import json + +from together.lib.realtime._types import ( + TurnEvent, + UnknownEvent, + EchoResponseEvent, + SessionCreatedEvent, + TranscriptionDeltaEvent, + TranscriptionFailedEvent, + TranscriptionCompletedEvent, + InputAudioBufferProcessedEvent, + echo_event, + append_event, + commit_event, + parse_server_event, + session_update_event, +) + + +class TestParseServerEvent: + def test_session_created(self) -> None: + event = parse_server_event( + json.dumps( + { + "event_id": "ev1", + "type": "session.created", + "session": {"id": "s1", "object": "realtime.session", "modalities": ["audio"], "model": "m"}, + } + ) + ) + assert isinstance(event, SessionCreatedEvent) + assert event.session is not None + assert event.session.id == "s1" + + def test_delta_with_timings(self) -> None: + event = parse_server_event( + { + "type": "conversation.item.input_audio_transcription.delta", + "item_id": "msg_1", + "delta": "hi", + "start": 1.5, + "duration": 0.5, + } + ) + assert isinstance(event, TranscriptionDeltaEvent) + assert event.delta == "hi" + assert event.start == 1.5 + + def test_completed_with_all_fields_absent_except_transcript(self) -> None: + # Cartesia flush path: no item_id, no start/duration + event = parse_server_event( + {"type": "conversation.item.input_audio_transcription.completed", "transcript": "done"} + ) + assert isinstance(event, TranscriptionCompletedEvent) + assert event.item_id is None + assert event.start is None + assert event.transcript == "done" + + def test_failed_per_turn_vs_fatal(self) -> None: + per_turn = parse_server_event( + { + "type": "conversation.item.input_audio_transcription.failed", + "item_id": "msg_2", + "event_type": "completed", + "error": {"message": "decode blew up"}, + } + ) + assert isinstance(per_turn, TranscriptionFailedEvent) + assert per_turn.is_fatal is False + + fatal = parse_server_event( + { + "type": "conversation.item.input_audio_transcription.failed", + "error": { + "message": "no", + "type": "invalid_request_error", + "param": None, + "code": "model_not_available", + }, + } + ) + assert isinstance(fatal, TranscriptionFailedEvent) + assert fatal.is_fatal is True + assert fatal.error is not None + assert fatal.error.code == "model_not_available" + + def test_turn_events(self) -> None: + event = parse_server_event( + { + "type": "conversation.item.input_audio_transcription.end_of_turn", + "item_id": "msg_3", + "transcript": "so far", + "end_of_turn_confidence": 0.93, + } + ) + assert isinstance(event, TurnEvent) + assert event.end_of_turn_confidence == 0.93 + + def test_processed_watermark(self) -> None: + event = parse_server_event({"type": "input_audio_buffer.processed", "processed_ms": 1234.5}) + assert isinstance(event, InputAudioBufferProcessedEvent) + assert event.processed_ms == 1234.5 + + def test_echo_response(self) -> None: + event = parse_server_event( + {"type": "echo.response", "echo_id": 7, "server_received_at": 1.0, "server_sent_at": 2.0} + ) + assert isinstance(event, EchoResponseEvent) + assert event.echo_id == 7 + + def test_unknown_type_is_forward_compatible(self) -> None: + event = parse_server_event({"type": "some.future.event", "x": 1}) + assert isinstance(event, UnknownEvent) + assert event.type == "some.future.event" + assert event.data == {"type": "some.future.event", "x": 1} + + def test_missing_type_and_non_object(self) -> None: + assert isinstance(parse_server_event({}), UnknownEvent) + assert isinstance(parse_server_event(json.dumps([1, 2])), UnknownEvent) + + def test_extra_fields_are_tolerated(self) -> None: + event = parse_server_event( + {"type": "conversation.item.input_audio_transcription.delta", "delta": "x", "brand_new_field": {"a": 1}} + ) + assert isinstance(event, TranscriptionDeltaEvent) + + +class TestClientEventBuilders: + def test_append_base64_roundtrip(self) -> None: + import base64 + + event = append_event(b"\x00\x01\x02") + assert event["type"] == "input_audio_buffer.append" + assert base64.b64decode(event["audio"]) == b"\x00\x01\x02" + + def test_commit_and_session_update(self) -> None: + assert commit_event() == {"type": "input_audio_buffer.commit"} + update = session_update_event({"turn_detection": {"type": "none"}}) + # the server listens for the ".updated" variant + assert update["type"] == "transcription_session.updated" + assert update["session"] == {"turn_detection": {"type": "none"}} + + def test_echo(self) -> None: + assert echo_event() == {"type": "echo"} + assert echo_event(echo_id=3)["echo_id"] == 3 diff --git a/tests/unit/test_realtime_session.py b/tests/unit/test_realtime_session.py new file mode 100644 index 000000000..676372723 --- /dev/null +++ b/tests/unit/test_realtime_session.py @@ -0,0 +1,508 @@ +"""Failure-injection tests for the robust realtime transcription session. + +Runs an in-process websockets server whose per-connection behavior is +scripted, so reconnect/replay behavior is exercised against real sockets. +Servers are created inside each test (not fixtures) to avoid event-loop +leakage across tests. +""" + +from __future__ import annotations + +import json +import base64 +import asyncio +import contextlib +from typing import Any, Dict, List, Callable, Optional + +import pytest + +from together import AsyncTogether +from together.lib.realtime import ( + BufferGap, + Reconnected, + Reconnecting, + SessionStarted, + TranscriptDelta, + TranscriptCompleted, + RealtimeSessionError, + RealtimeSessionEvent, + RealtimeConnectionError, +) +from together.lib.realtime._session import AsyncRealtimeTranscriptionSession + +BPS = 32_000 # pcm_s16le_16000 + + +class ConnectionLog: + """What one server-side connection observed.""" + + def __init__(self) -> None: + self.audio = bytearray() + self.commits = 0 + self.events: List[Dict[str, Any]] = [] + self.path = "" + + +class FakeRealtimeServer: + """Scriptable stand-in for the ipop realtime endpoint.""" + + def __init__( + self, + *, + reject_statuses: Optional[List[int]] = None, + drop_after_bytes: Optional[int] = None, + completed_every_bytes: Optional[int] = None, + fatal_error: Optional[Dict[str, Any]] = None, + complete_on_commit: bool = True, + transcribed_fraction: float = 1.0, + answer_echo: bool = True, + ) -> None: + self.reject_statuses = list(reject_statuses or []) + self.drop_after_bytes = drop_after_bytes + self.completed_every_bytes = completed_every_bytes + self.fatal_error = fatal_error + self.complete_on_commit = complete_on_commit + self.transcribed_fraction = transcribed_fraction + self.answer_echo = answer_echo + self.connections: List[ConnectionLog] = [] + self._server: Any = None + + async def __aenter__(self) -> "FakeRealtimeServer": + from websockets.asyncio.server import serve + + self._server = await serve(self._handler, "127.0.0.1", 0, process_request=self._process_request) + port = self._server.sockets[0].getsockname()[1] + self.url = f"http://127.0.0.1:{port}" + return self + + async def __aexit__(self, *_exc: object) -> None: + self._server.close() + await self._server.wait_closed() + + def _process_request(self, connection: Any, _request: Any) -> Any: + if self.reject_statuses: + status = self.reject_statuses.pop(0) + return connection.respond(status, f"rejected {status}\n") + return None + + async def _handler(self, ws: Any) -> None: + conn = ConnectionLog() + conn.path = getattr(ws.request, "path", "") + self.connections.append(conn) + item = len(self.connections) * 100 + await ws.send( + json.dumps( + { + "type": "session.created", + "event_id": "e1", + "session": {"id": f"s{len(self.connections)}", "object": "realtime.session"}, + } + ) + ) + if self.fatal_error is not None: + await ws.send( + json.dumps( + { + "type": "conversation.item.input_audio_transcription.failed", + "error": self.fatal_error, + } + ) + ) + await ws.close() + return + emitted_at = 0 + try: + async for message in ws: + event = json.loads(message) + conn.events.append(event) + etype = event.get("type") + if etype == "input_audio_buffer.append": + conn.audio.extend(base64.b64decode(event["audio"])) + if self.drop_after_bytes is not None and len(conn.audio) >= self.drop_after_bytes: + self.drop_after_bytes = None # only the first connection drops + ws.transport.abort() # simulate abrupt network failure + return + if ( + self.completed_every_bytes is not None + and len(conn.audio) - emitted_at >= self.completed_every_bytes + ): + # like the real whisper handler, completed carries + # start/duration on the audio timeline + start_s = emitted_at / BPS + duration_s = (len(conn.audio) - emitted_at) / BPS * self.transcribed_fraction + emitted_at = len(conn.audio) + item += 1 + await ws.send( + json.dumps( + { + "type": "conversation.item.input_audio_transcription.delta", + "item_id": f"msg_{item}", + "delta": f"partial-{item}", + } + ) + ) + await ws.send( + json.dumps( + { + "type": "conversation.item.input_audio_transcription.completed", + "item_id": f"msg_{item}", + "transcript": f"final-{item}", + "start": start_s, + "duration": duration_s, + } + ) + ) + elif etype == "input_audio_buffer.commit": + conn.commits += 1 + if self.complete_on_commit: + item += 1 + await ws.send( + json.dumps( + { + "type": "conversation.item.input_audio_transcription.completed", + "item_id": f"msg_{item}", + "transcript": f"committed-{item}", + } + ) + ) + elif etype == "echo" and self.answer_echo: + await ws.send(json.dumps({"type": "echo.response", "echo_id": event.get("echo_id")})) + except Exception: + pass + + +def make_session(server: FakeRealtimeServer, **overrides: Any) -> AsyncRealtimeTranscriptionSession: + client = AsyncTogether(api_key="test-key", base_url=server.url) + kwargs: Dict[str, Any] = dict( + client=client, + model="openai/whisper-large-v3", + reconnect={"backoff_initial": 0.01, "backoff_max": 0.02, "max_attempts": 5, "max_elapsed": 5.0}, + ) + kwargs.update(overrides) + return AsyncRealtimeTranscriptionSession(**kwargs) + + +async def collect_until( + session: AsyncRealtimeTranscriptionSession, + predicate: Callable[[List[RealtimeSessionEvent]], bool], + timeout: float = 5.0, +) -> List[RealtimeSessionEvent]: + """Consume session events until predicate(events) is truthy.""" + events: List[RealtimeSessionEvent] = [] + + async def _consume() -> None: + async for event in session: + events.append(event) + if predicate(events): + return + + await asyncio.wait_for(_consume(), timeout) + return events + + +def seconds(n: float) -> bytes: + return b"\x01" * int(n * BPS) + + +class TestHappyPath: + async def test_vad_flow_delta_completed(self) -> None: + async with FakeRealtimeServer(completed_every_bytes=BPS) as server: + session = make_session(server) + async with session: + await session.append(seconds(1.0)) + events = await collect_until(session, lambda evs: any(isinstance(e, TranscriptCompleted) for e in evs)) + assert isinstance(events[0], SessionStarted) + deltas = [e for e in events if isinstance(e, TranscriptDelta)] + finals = [e for e in events if isinstance(e, TranscriptCompleted)] + assert deltas and finals + assert finals[0].text.startswith("final-") + assert finals[0].replayed is False + assert server.connections[0].audio == seconds(1.0) + + async def test_manual_commit_flow(self) -> None: + async with FakeRealtimeServer() as server: + session = make_session(server, turn_detection={"type": "none"}) + async with session: + await session.append(seconds(0.5)) + await session.commit() + events = await collect_until(session, lambda evs: any(isinstance(e, TranscriptCompleted) for e in evs)) + finals = [e for e in events if isinstance(e, TranscriptCompleted)] + assert finals[0].text.startswith("committed-") + # commit was sent after all audio + assert server.connections[0].commits == 1 + assert len(server.connections[0].audio) == len(seconds(0.5)) + + +class TestSessionParams: + async def test_session_level_params_sent_on_connect(self) -> None: + async with FakeRealtimeServer() as server: + session = make_session( + server, + language="en", + prompt="medical terms", + rolling_prompt=True, + energy_gate_rms=0.02, + session_params={"custom_engine_knob": "x"}, + ) + async with session: + await session.append(seconds(0.1)) + await asyncio.sleep(0.1) + updates = [e for e in server.connections[0].events if e["type"] == "transcription_session.updated"] + assert updates, "expected a transcription_session.updated event" + sent = updates[0]["session"] + assert sent["language"] == "en" + assert sent["prompt"] == "medical terms" + assert sent["rolling_prompt"] is True + assert sent["energy_gate_rms"] == 0.02 + assert sent["custom_engine_knob"] == "x" + + +class TestReconnect: + async def test_drop_mid_stream_replays_from_anchor(self) -> None: + async with FakeRealtimeServer(completed_every_bytes=BPS, drop_after_bytes=int(2.5 * BPS)) as server: + session = make_session(server, buffer={"replay_margin": 1.0}) + async with session: + await session.append(seconds(2.0)) + # wait for the completeds from the first connection (anchor advances) + await collect_until(session, lambda evs: sum(isinstance(e, TranscriptCompleted) for e in evs) >= 2) + await session.append(seconds(1.0)) # triggers the drop at 2.5s + events = await collect_until( + session, lambda evs: any(isinstance(e, Reconnected) for e in evs), timeout=10.0 + ) + assert any(isinstance(e, Reconnecting) for e in events) + assert len(server.connections) == 2 + # replay window = [anchor - margin, head]; anchor was at ~2s of + # audio appended when the second completed arrived. Reconnected + # fires before the writer drains, so wait for the replay bytes. + replayed = await self._wait_bytes_stable(server, 1, int(1.0 * BPS)) + assert replayed <= int(2.0 * BPS) # bounded: never the whole 3s buffer + assert replayed >= int(1.0 * BPS) # at least margin + post-anchor audio + + async def test_max_replay_seconds_zero_resumes_live(self) -> None: + async with FakeRealtimeServer(drop_after_bytes=BPS) as server: + session = make_session(server, buffer={"max_replay_seconds": 0.0}) + async with session: + await session.append(seconds(1.0)) # server drops at 1s + await collect_until(session, lambda evs: any(isinstance(e, Reconnected) for e in evs), timeout=10.0) + assert len(server.connections) == 2 + assert len(server.connections[1].audio) == 0 # nothing replayed + await session.append(seconds(0.25)) + await asyncio.wait_for(self._wait_bytes(server, 1, int(0.25 * BPS)), 5.0) + + @staticmethod + async def _wait_bytes(server: FakeRealtimeServer, conn: int, nbytes: int) -> None: + while len(server.connections) <= conn or len(server.connections[conn].audio) < nbytes: + await asyncio.sleep(0.01) + + @staticmethod + async def _wait_bytes_stable(server: FakeRealtimeServer, conn: int, min_bytes: int, timeout: float = 5.0) -> int: + """Wait for at least min_bytes on a connection, then for the count to go quiet.""" + await asyncio.wait_for(TestReconnect._wait_bytes(server, conn, min_bytes), timeout) + prev = -1 + while prev != len(server.connections[conn].audio): + prev = len(server.connections[conn].audio) + await asyncio.sleep(0.1) + return prev + + async def test_commit_resent_after_drop(self) -> None: + async with FakeRealtimeServer(drop_after_bytes=int(0.5 * BPS)) as server: + session = make_session(server, turn_detection={"type": "none"}) + async with session: + await session.append(seconds(0.5)) + await session.commit() # server drops before answering + events = await collect_until( + session, + lambda evs: any(isinstance(e, TranscriptCompleted) for e in evs), + timeout=10.0, + ) + assert len(server.connections) == 2 + second = server.connections[1] + assert second.commits == 1 # the outstanding commit was re-issued + assert len(second.audio) == len(seconds(0.5)) # full unacked audio replayed + finals = [e for e in events if isinstance(e, TranscriptCompleted)] + assert finals and finals[0].replayed is True + + async def test_replayed_flag_set_on_replay_events(self) -> None: + async with FakeRealtimeServer(completed_every_bytes=BPS, drop_after_bytes=int(1.5 * BPS)) as server: + # explicit pre-roll so the replay window covers the already + # transcribed first second and the server re-emits its transcript + session = make_session(server, buffer={"replay_margin": 5.0}) + async with session: + await session.append(seconds(1.5)) # 1 completed, then drop + events = await collect_until( + session, + lambda evs: sum(isinstance(e, TranscriptCompleted) for e in evs) >= 2, + timeout=10.0, + ) + finals = [e for e in events if isinstance(e, TranscriptCompleted)] + assert finals[0].replayed is False + assert finals[1].replayed is True # produced from replayed audio + # stable, distinct SDK segment ids despite server item_id reuse patterns + assert finals[0].segment_id != finals[1].segment_id + + +class TestTerminalFailureUnderLoad: + async def test_continuous_appends_during_terminal_failure_do_not_starve_loop(self) -> None: + """Regression: unsent audio accumulating while a reconnect is in flight + must not busy-spin the writer and starve the event loop (the reconnect + task would then never escalate, hanging the session forever).""" + async with FakeRealtimeServer(drop_after_bytes=int(0.3 * BPS)) as server: + session = make_session(server, reconnect={"max_attempts": 0}) + with pytest.raises(RealtimeConnectionError): + async with session: + server.reject_statuses.extend([500] * 5) + + async def feed() -> None: + while True: + await session.append(seconds(0.1)) + await asyncio.sleep(0.01) + + feeder = asyncio.create_task(feed()) + try: + await asyncio.wait_for(collect_until(session, lambda _evs: False, timeout=8.0), 9.0) + finally: + feeder.cancel() + with contextlib.suppress(asyncio.CancelledError, Exception): + await feeder + + +class TestTerminalFailureSurfacesOnCalls: + async def test_append_and_flush_raise_the_terminal_failure(self) -> None: + """A plain feed loop must be able to drive failover without a separate + consumer task: after terminal failure, append()/flush() raise the + stored RealtimeConnectionError rather than a generic state error.""" + async with FakeRealtimeServer(drop_after_bytes=int(0.2 * BPS)) as server: + session = make_session(server, reconnect={"max_attempts": 0}) + async with session: + server.reject_statuses.extend([500] * 5) + with pytest.raises(RealtimeConnectionError): + for _ in range(200): # keep feeding until failure surfaces + await session.append(seconds(0.1)) + await asyncio.sleep(0.02) + with pytest.raises(RealtimeConnectionError): + await session.flush() + + +class TestExternalFailover: + """Endpoint failover is orchestrated OUTSIDE the SDK: on terminal failure + the app starts a fresh session on an alternate endpoint, seeded with + pending_audio() and context_prompt() from the failed session.""" + + async def test_pending_audio_and_prompt_resume_on_alternate_endpoint(self) -> None: + async with FakeRealtimeServer( + completed_every_bytes=BPS, drop_after_bytes=int(2 * BPS) + ) as primary, FakeRealtimeServer() as alternate: + session = make_session( + primary, + reconnect={"backoff_initial": 0.001, "backoff_max": 0.002, "max_attempts": 2, "max_elapsed": 5.0}, + ) + failed = False + async with session: + await session.append(seconds(1.5)) + await collect_until(session, lambda evs: any(isinstance(e, TranscriptCompleted) for e in evs)) + primary.reject_statuses.extend([500] * 20) # endpoint is now down + await session.append(seconds(1.0)) # triggers the drop at 2s + try: + await collect_until(session, lambda _evs: False, timeout=10.0) + except RealtimeConnectionError: + failed = True + assert failed + # ---- orchestration layer (application code, not the SDK) ---- + pending = session.pending_audio() + prompt = session.context_prompt() + assert prompt.startswith("final-") or prompt # transcript tail available + # watermark was 1.0s (completed start+duration), head 2.5s + assert len(pending) == len(seconds(1.5)) + resumed = make_session(alternate, turn_detection={"type": "none"}, prompt=prompt) + async with resumed: + await resumed.append(pending) + await resumed.commit() + events = await collect_until(resumed, lambda evs: any(isinstance(e, TranscriptCompleted) for e in evs)) + assert len(alternate.connections) == 1 + assert bytes(alternate.connections[0].audio) == pending + updates = [e for e in alternate.connections[0].events if e["type"] == "transcription_session.updated"] + assert updates and updates[0]["session"]["prompt"] == prompt + finals = [e for e in events if isinstance(e, TranscriptCompleted)] + assert finals + + +class TestFatalErrors: + async def test_fatal_model_not_available(self) -> None: + async with FakeRealtimeServer( + fatal_error={"message": "no such model", "type": "invalid_request_error", "code": "model_not_available"} + ) as server: + session = make_session(server) + async with session: + with pytest.raises(RealtimeSessionError) as err: + await collect_until(session, lambda _evs: False, timeout=5.0) + assert err.value.code == "model_not_available" + + async def test_initial_handshake_failure_raises_typed_error_naming_target(self) -> None: + async with FakeRealtimeServer(reject_statuses=[401]) as server: + session = make_session(server) + with pytest.raises(RealtimeConnectionError, match="HTTP 401") as err: + await session.start() + assert "127.0.0.1" in str(err.value) # names the target it tried + + async def test_handshake_5xx_retries_then_succeeds(self) -> None: + async with FakeRealtimeServer( + completed_every_bytes=BPS, drop_after_bytes=int(0.5 * BPS), reject_statuses=[] + ) as server: + # first connection OK; drop; then two 500s on reconnect before success + session = make_session(server) + async with session: + await session.append(seconds(0.25)) + server.reject_statuses.extend([500, 503]) + await session.append(seconds(0.25)) # triggers drop + events = await collect_until( + session, lambda evs: any(isinstance(e, Reconnected) for e in evs), timeout=10.0 + ) + reconnected = [e for e in events if isinstance(e, Reconnected)] + assert reconnected[0].attempt >= 3 # two rejected handshakes + one success + assert len(server.connections) == 2 + + async def test_retries_exhausted_raises_connection_error(self) -> None: + async with FakeRealtimeServer(drop_after_bytes=int(0.1 * BPS)) as server: + session = make_session( + server, + reconnect={"backoff_initial": 0.001, "backoff_max": 0.002, "max_attempts": 3, "max_elapsed": 5.0}, + ) + async with session: + # reject only reconnect attempts, not the initial connect + server.reject_statuses.extend([500] * 50) + await session.append(seconds(0.2)) + with pytest.raises(RealtimeConnectionError): + await collect_until(session, lambda _evs: False, timeout=10.0) + + +class TestWatermark: + async def test_completed_start_duration_bounds_replay(self) -> None: + # the server reports only half of each segment as transcribed via + # start+duration, so the watermark (1.5s) trails the anchor (2.0s); + # replay must start from the watermark minus the margin + async with FakeRealtimeServer( + completed_every_bytes=BPS, transcribed_fraction=0.5, drop_after_bytes=int(2.5 * BPS) + ) as server: + session = make_session(server, buffer={"replay_margin": 0.5}) + async with session: + await session.append(seconds(2.0)) + await collect_until(session, lambda evs: sum(isinstance(e, TranscriptCompleted) for e in evs) >= 2) + assert session.state.watermark == session.state.to_bytes(1.5) # start 1.0 + duration 0.5 + await session.append(seconds(1.0)) + await collect_until(session, lambda evs: any(isinstance(e, Reconnected) for e in evs), timeout=10.0) + # watermark 1.5s - margin 0.5s => replay [1.0s, 3.0s] = 2.0s + replayed = await TestReconnect._wait_bytes_stable(server, 1, int(2.0 * BPS)) + assert replayed == int(2.0 * BPS) + + +class TestBufferGapEvents: + async def test_overflow_emits_gap(self) -> None: + async with FakeRealtimeServer() as server: # never completes anything + session = make_session(server, buffer={"max_seconds": 1.0}) + async with session: + await session.append(seconds(3.0)) + events = await collect_until( + session, lambda evs: any(isinstance(e, BufferGap) for e in evs), timeout=5.0 + ) + gaps = [e for e in events if isinstance(e, BufferGap)] + assert abs(gaps[0].dropped_seconds - 2.0) < 0.1 diff --git a/tests/unit/test_realtime_state.py b/tests/unit/test_realtime_state.py new file mode 100644 index 000000000..f291c89ad --- /dev/null +++ b/tests/unit/test_realtime_state.py @@ -0,0 +1,396 @@ +from __future__ import annotations + +import pytest + +from together.lib.realtime._state import ( + BufferPool, + AudioBuffer, + FailureKind, + BackoffPolicy, + RecoveryState, + iter_frames, + classify_fatal_error, + classify_handshake_status, +) +from together.lib.realtime._types import RealtimeErrorInfo +from together.lib.realtime._exceptions import RealtimeBufferOverflowError + +BPS = 32_000 # pcm_s16le_16000 +FMT = "pcm_s16le_16000" + + +def seconds(n: float) -> bytes: + return b"\x01" * int(n * BPS) + + +def make_state(**kwargs: object) -> RecoveryState: + # neutral defaults so each test states its intent explicitly; production + # defaults (margin 0, max_replay_seconds 5) are covered by + # TestDefaultReplayWindow + defaults: dict[str, object] = {"input_audio_format": FMT, "replay_margin": 5.0, "max_replay_seconds": None} + defaults.update(kwargs) + return RecoveryState(**defaults) # type: ignore[arg-type] + + +class TestAudioBuffer: + def test_append_and_offsets(self) -> None: + buf = AudioBuffer() + assert buf.append(b"abc") == 0 + assert buf.append(b"defg") == 3 + assert buf.start_offset == 0 + assert buf.end_offset == 7 + assert buf.size == 7 + + def test_trim_mid_chunk_and_read_from(self) -> None: + buf = AudioBuffer() + buf.append(b"abcd") + buf.append(b"efgh") + freed = buf.trim_to(6) + assert freed == 6 + assert buf.start_offset == 6 + assert b"".join(buf.read_from(6)) == b"gh" + # reading below start yields only retained data + assert b"".join(buf.read_from(0)) == b"gh" + + def test_trim_never_regresses_or_overshoots(self) -> None: + buf = AudioBuffer() + buf.append(b"abcd") + buf.trim_to(2) + assert buf.trim_to(1) == 0 # regressing is a no-op + buf.trim_to(100) # beyond end clamps + assert buf.size == 0 + assert buf.start_offset == buf.end_offset == 4 + + def test_pool_charge_release(self) -> None: + pool = BufferPool(max_bytes=None) + buf = AudioBuffer(pool) + buf.append(b"x" * 100) + assert pool.used_bytes == 100 + buf.trim_to(40) + assert pool.used_bytes == 60 + + +class TestBufferPool: + def test_reclaims_from_largest_holder(self) -> None: + pool = BufferPool(max_bytes=100) + state_small = make_state(pool=pool, max_seconds=1000.0) + state_big = make_state(pool=pool, max_seconds=1000.0) + pool.register(state_small, lambda: state_small.buffer.size, state_small.reclaim) + pool.register(state_big, lambda: state_big.buffer.size, state_big.reclaim) + + state_small.record_append(b"a" * 10) + state_big.record_append(b"b" * 80) + # pool at 90/100; appending 20 more must reclaim >= 10 from the largest (big) + state_small.record_append(b"c" * 20) + assert pool.used_bytes <= 100 + assert state_big.buffer.size < 80 + assert state_small.consume_pending_gap() == 0 + assert state_big.consume_pending_gap() > 0 # un-acked audio was dropped + + +class TestClassification: + def test_handshake(self) -> None: + assert classify_handshake_status(401) is FailureKind.FATAL_AUTH + assert classify_handshake_status(403) is FailureKind.FATAL_AUTH + assert classify_handshake_status(429) is FailureKind.RETRYABLE + assert classify_handshake_status(500) is FailureKind.RETRYABLE + assert classify_handshake_status(None) is FailureKind.RETRYABLE + + @pytest.mark.parametrize( + "error,expected", + [ + (None, FailureKind.RETRYABLE), + (RealtimeErrorInfo(code="model_not_available"), FailureKind.FATAL), + (RealtimeErrorInfo(code="model_not_accessible"), FailureKind.FATAL), + (RealtimeErrorInfo(code="invalid_api_key"), FailureKind.FATAL_AUTH), + (RealtimeErrorInfo(code="idle_timeout"), FailureKind.IDLE_TIMEOUT), + ( + RealtimeErrorInfo(type="service_unavailable_error", message="Service unavailable: Timeout"), + FailureKind.IDLE_TIMEOUT, + ), + ( + RealtimeErrorInfo(type="service_unavailable_error", message="Unsupported format: pcm_f32"), + FailureKind.FATAL, + ), + ( + RealtimeErrorInfo(type="service_unavailable_error", message="Unsupported input sample rate 44100"), + FailureKind.FATAL, + ), + ( + RealtimeErrorInfo(type="service_unavailable_error", message="upstream decode failed"), + FailureKind.RETRYABLE, + ), + (RealtimeErrorInfo(type="invalid_request_error", message="bad payload"), FailureKind.FATAL), + ], + ) + def test_fatal_frames(self, error: RealtimeErrorInfo, expected: FailureKind) -> None: + assert classify_fatal_error(error) is expected + + +class TestBackoff: + def test_full_jitter_bounds_and_determinism(self) -> None: + policy = BackoffPolicy(initial=0.5, maximum=15.0, rng=lambda: 1.0) + assert policy.delay(0) == 0.5 + assert policy.delay(1) == 1.0 + assert policy.delay(10) == 15.0 # capped + zero = BackoffPolicy(rng=lambda: 0.0) + assert zero.delay(5) == 0.0 + + +class TestRecoveryAnchor: + def test_no_events_replays_everything_retained(self) -> None: + state = make_state() + state.record_append(seconds(3)) + plan = state.replay_plan() + assert plan.start_offset == 0 + assert plan.gap_bytes == 0 + + def test_anchor_minus_margin(self) -> None: + state = make_state(replay_margin=5.0) + state.record_append(seconds(20)) + state.record_completed() # anchor at 20s + state.record_append(seconds(4)) # head 24s + plan = state.replay_plan() + assert plan.start_offset == state.to_bytes(15.0) # 20s - 5s margin + assert plan.gap_bytes == 0 + + def test_anchor_trims_buffer(self) -> None: + state = make_state(replay_margin=5.0) + state.record_append(seconds(20)) + state.record_completed() + # retained window is [anchor - margin, head] + assert state.buffer.start_offset == state.to_bytes(15.0) + + def test_watermark_supersedes_anchor(self) -> None: + state = make_state(replay_margin=5.0) + state.begin_epoch(0) + state.record_append(seconds(20)) + # completed arrives with start+duration covering audio through 18s: + # watermark (18s) wins over the arrival anchor (20s) + state.record_completed(processed_seconds=18.0) + plan = state.replay_plan() + assert plan.start_offset == state.to_bytes(13.0) # watermark - margin + + def test_completed_start_duration_sets_watermark(self) -> None: + # completed.start + duration is the production watermark source + state = make_state(replay_margin=1.0) + state.begin_epoch(0) + state.record_append(seconds(20)) + state.record_completed(processed_seconds=12.0) + assert state.watermark == state.to_bytes(12.0) + plan = state.replay_plan() + # watermark (12s) - margin (1s) wins over anchor (20s) - margin + assert plan.start_offset == state.to_bytes(11.0) + + def test_completed_watermark_respects_conn_base_across_reconnects(self) -> None: + state = make_state(replay_margin=0.0) + state.begin_epoch(0) + state.record_append(seconds(10)) + state.record_completed(processed_seconds=10.0) + plan = state.replay_plan() + state.begin_epoch(plan.start_offset) # reconnect at 10s global + state.record_append(seconds(5)) # head 15s; conn timeline restarts + state.record_completed(processed_seconds=3.0) # 3s on the NEW connection + assert state.watermark == state.to_bytes(13.0) # conn_base 10s + 3s + + def test_watermark_is_monotonic_and_clamped(self) -> None: + state = make_state() + state.begin_epoch(0) + state.record_append(seconds(10)) + state.record_completed(processed_seconds=8.0) + state.record_completed(processed_seconds=6.0) # regression ignored + assert state.watermark == state.to_bytes(8.0) + state.record_completed(processed_seconds=99.0) # beyond head clamps to head + assert state.watermark == state.to_bytes(10.0) + + def test_max_replay_seconds_caps_window_and_reports_gap(self) -> None: + state = make_state(max_replay_seconds=5.0) + state.record_append(seconds(60)) + plan = state.replay_plan() + assert plan.start_offset == state.to_bytes(55.0) + assert plan.gap_bytes == state.to_bytes(55.0) + + def test_zero_max_replay_seconds_is_pure_resume(self) -> None: + state = make_state(max_replay_seconds=0.0) + state.record_append(seconds(30)) + plan = state.replay_plan() + assert plan.start_offset == state.write_head + + def test_replay_clamped_to_retained_data(self) -> None: + state = make_state(max_seconds=10.0) # cap retention at 10s + state.record_append(seconds(30)) # no completeds: 20s dropped un-acked + plan = state.replay_plan() + assert plan.start_offset == state.buffer.start_offset + # the drop was surfaced at overflow time, not re-reported by the plan + assert plan.gap_bytes == 0 + assert state.consume_pending_gap() == state.to_bytes(20.0) + + +class TestDefaultReplayWindow: + """Replay start = max(head - max_replay_seconds, last transcribed position).""" + + def test_recent_completed_wins(self) -> None: + state = make_state(replay_margin=0.0, max_replay_seconds=5.0) + state.record_append(seconds(30)) + state.record_completed(processed_seconds=29.0) # transcribed through 29s + state.record_append(seconds(2)) # head 32s + plan = state.replay_plan() + # max(32-5=27, 29) = 29: only untranscribed audio replays + assert plan.start_offset == state.to_bytes(29.0) + assert plan.gap_bytes == 0 + + def test_long_untranscribed_window_capped_with_gap(self) -> None: + state = make_state(replay_margin=0.0, max_replay_seconds=5.0) + state.record_append(seconds(30)) + state.record_completed(processed_seconds=10.0) + state.record_append(seconds(30)) # head 60s; 50s untranscribed + plan = state.replay_plan() + # max(60-5=55, 10) = 55: cap wins, 45s of untranscribed audio skipped + assert plan.start_offset == state.to_bytes(55.0) + assert plan.gap_bytes == state.to_bytes(45.0) + + +class TestSampleAlignment: + """Replay/trim cuts must land on 16-bit sample boundaries: an odd byte + offset byte-shifts every subsequent sample, turning replayed speech into + static (whisper hallucinates on it).""" + + def test_watermark_from_fractional_seconds_is_sample_aligned(self) -> None: + state = make_state(replay_margin=0.0) + state.begin_epoch(0) + state.record_append(seconds(3)) + # 1.0000156s * 32000 B/s = 32000.4992 -> int() = 32001, an odd byte + state.record_completed(processed_seconds=1.0000156) + assert state.watermark is not None and state.watermark % 2 == 0 + plan = state.replay_plan() + assert plan.start_offset % 2 == 0 + + def test_overflow_trim_keeps_buffer_start_aligned(self) -> None: + state = make_state(max_seconds=1.0) + state.record_append(b"\x01" * 32001) # odd-sized append forces odd excess + state.record_append(b"\x01" * 32000) + assert state.buffer.start_offset % 2 == 0 + assert state.replay_plan().start_offset % 2 == 0 + + +class TestOverflow: + def test_drop_oldest_records_pending_gap(self) -> None: + state = make_state(max_seconds=10.0) + state.record_append(seconds(15)) + assert state.buffer.size == state.to_bytes(10.0) + assert state.consume_pending_gap() == state.to_bytes(5.0) + assert state.consume_pending_gap() == 0 + + def test_acked_trim_is_not_a_gap(self) -> None: + state = make_state(max_seconds=10.0, replay_margin=0.0) + state.record_append(seconds(8)) + state.record_completed() # everything acked; buffer trimmed to anchor + state.record_append(seconds(9)) + assert state.consume_pending_gap() == 0 + + def test_error_policy_raises(self) -> None: + state = make_state(max_seconds=10.0, overflow="error") + with pytest.raises(RealtimeBufferOverflowError): + state.record_append(seconds(15)) + + +class TestCommitLedger: + def test_commit_retired_by_completed(self) -> None: + state = make_state() + state.record_append(seconds(3)) + state.record_commit() + state.record_append(seconds(1)) + state.record_completed() + assert state.outstanding_commits == [] + assert state.replay_plan().resend_commit is False + + def test_outstanding_commit_forces_replay_and_resend(self) -> None: + state = make_state(replay_margin=1.0) + state.record_append(seconds(10)) + state.record_completed() # anchor 10s + state.record_append(seconds(5)) + state.record_commit() # boundary at 15s, never completed + state.record_append(seconds(30)) + state.record_completed() # anchor 45s — but commit at 15s still outstanding? + # completeds are FIFO: a completed after the commit covers it + assert state.outstanding_commits == [] + + def test_commit_never_completed_keeps_replay_window(self) -> None: + state = make_state(replay_margin=1.0) + state.record_append(seconds(10)) + state.record_commit() # silent tail: server will never answer + plan = state.replay_plan() + assert plan.resend_commit is True + # nothing was ever acked, so the whole retained window replays + assert plan.start_offset == 0 + + def test_outstanding_commit_lowers_anchor_trim(self) -> None: + state = make_state(replay_margin=1.0) + state.record_append(seconds(10)) + state.record_completed() # anchor 10s + state.record_append(seconds(2)) + state.record_commit() # boundary 12s + state.record_append(seconds(20)) + # a completed for OTHER audio must not let trimming pass the commit window + # (completed retires the commit here per FIFO, so use replay_plan before it) + plan = state.replay_plan() + assert plan.start_offset == state.to_bytes(9.0) # anchor 10s - 1s margin + assert plan.resend_commit is True + + def test_zero_completed_commit_does_not_block_trimming_forever(self) -> None: + # A later completed retires older commit boundaries (FIFO ordering), + # so a 0-completed commit cannot deadlock the ledger. + state = make_state(replay_margin=0.0) + state.record_append(seconds(1)) + state.record_commit() + state.record_append(seconds(20)) + state.record_completed() + assert state.outstanding_commits == [] + + +class TestEpochsAndSegments: + def test_begin_epoch_resets_watermark_and_sets_base(self) -> None: + state = make_state() + state.begin_epoch(0) + state.record_append(seconds(10)) + state.record_processed(5_000.0) + plan = state.replay_plan() + epoch = state.begin_epoch(plan.start_offset) + assert epoch == 2 + assert state.watermark is None + assert state.conn_base == plan.start_offset + + def test_segment_ids_stable_and_globally_unique(self) -> None: + state = make_state() + state.begin_epoch(0) + a = state.segment_for("msg_1", replayed=False) + assert state.segment_for("msg_1", replayed=False).segment_id == a.segment_id + state.begin_epoch(0) + b = state.segment_for("msg_1", replayed=True) # same server item_id, new epoch + assert b.segment_id != a.segment_id + assert b.replayed is True + + def test_anonymous_segments_close_and_rotate(self) -> None: + state = make_state() + first = state.segment_for(None, replayed=False) + state.close_segment(None) + second = state.segment_for(None, replayed=False) + assert first.segment_id != second.segment_id + + def test_clear_resets_recovery_state(self) -> None: + state = make_state() + state.record_append(seconds(10)) + state.record_commit() + state.record_completed() + state.record_clear() + assert state.buffer.size == 0 + assert state.anchor is None + assert state.outstanding_commits == [] + assert state.replay_plan().start_offset == state.write_head + + +class TestFrames: + def test_iter_frames_exact_and_remainder(self) -> None: + frames = list(iter_frames(b"x" * 10, 4)) + assert [len(f) for f in frames] == [4, 4, 2] + assert list(iter_frames(b"", 4)) == [] diff --git a/tests/unit/test_realtime_wiring.py b/tests/unit/test_realtime_wiring.py new file mode 100644 index 000000000..395592bb6 --- /dev/null +++ b/tests/unit/test_realtime_wiring.py @@ -0,0 +1,71 @@ +"""Regen-guard: `client.beta.realtime` is a handwritten addition to the generated +resources/beta/beta.py. If a Stainless regeneration drops the cached_property, these tests +fail loudly instead of the feature silently disappearing. +""" + +from __future__ import annotations + +from together import Together, AsyncTogether +from together.resources.realtime import RealtimeResource, AsyncRealtimeResource + + +def test_sync_client_exposes_realtime() -> None: + client = Together(api_key="test-key", base_url="http://127.0.0.1:4010") + assert isinstance(client.beta.realtime, RealtimeResource) + manager = client.beta.realtime.connect(model="openai/whisper-large-v3") + assert manager is not None + + +def test_async_client_exposes_realtime() -> None: + client = AsyncTogether(api_key="test-key", base_url="http://127.0.0.1:4010") + assert isinstance(client.beta.realtime, AsyncRealtimeResource) + + +def test_public_lib_surface_importable() -> None: + import together.lib.realtime as rt + + for name in rt.__all__: + assert getattr(rt, name) is not None + + +def test_root_realtime_module_reexports_public_surface() -> None: + """`together.realtime` is the stable public import path; it must expose + everything the lib package exports (a regen or refactor that drops the + root shim or lets it drift fails here).""" + import together.realtime as public + import together.lib.realtime as lib + + for name in lib.__all__: + assert getattr(public, name) is getattr(lib, name), name + + +def test_url_and_header_derivation() -> None: + import httpx + + from together.lib.realtime._connection import build_realtime_url, build_realtime_headers + + url = build_realtime_url( + httpx.URL("https://api.together.ai/v1/"), + model="openai/whisper-large-v3", + turn_detection={"type": "server_vad", "min_silence_duration_ms": 400}, + ) + assert url.startswith("wss://api.together.ai/v1/realtime?") + assert "model=openai%2Fwhisper-large-v3" in url or "model=openai/whisper-large-v3" in url + assert "input_audio_format=pcm_s16le_16000" in url + assert "turn_detection=server_vad" in url + assert "min_silence_duration_ms=400" in url + + headers = build_realtime_headers({"Authorization": "Bearer k"}) + assert headers["Authorization"] == "Bearer k" + assert headers["OpenAI-Beta"] == "realtime=v1" + + +def test_sync_async_resource_signatures_stay_in_sync() -> None: + """The sync/async resource surfaces are intentionally duplicated for IDE + ergonomics; this guard catches parameter drift between them.""" + import inspect + + for name in ("connect", "transcription"): + sync_params = inspect.signature(getattr(RealtimeResource, name)).parameters + async_params = inspect.signature(getattr(AsyncRealtimeResource, name)).parameters + assert sync_params == async_params, f"{name}() signatures diverged" diff --git a/uv.lock b/uv.lock index 72274f16e..89410254f 100644 --- a/uv.lock +++ b/uv.lock @@ -1599,6 +1599,9 @@ pyarrow = [ { name = "pyarrow" }, { name = "pyarrow-stubs" }, ] +realtime = [ + { name = "websockets" }, +] [package.dev-dependencies] dev = [ @@ -1615,6 +1618,7 @@ dev = [ { name = "rich" }, { name = "ruff" }, { name = "time-machine" }, + { name = "websockets" }, ] pydantic-v1 = [ { name = "pydantic", version = "1.10.26", source = { registry = "https://pypi.org/simple" } }, @@ -1650,8 +1654,9 @@ requires-dist = [ { name = "types-tabulate", specifier = ">=0.9.0.20240106" }, { name = "types-tqdm", specifier = ">=4.67.0.20250516" }, { name = "typing-extensions", specifier = ">=4.14,<5" }, + { name = "websockets", marker = "extra == 'realtime'", specifier = ">=13,<16" }, ] -provides-extras = ["aiohttp", "aiofiles", "pyarrow", "cli"] +provides-extras = ["aiohttp", "aiofiles", "pyarrow", "cli", "realtime"] [package.metadata.requires-dev] dev = [ @@ -1668,6 +1673,7 @@ dev = [ { name = "rich", specifier = ">=13.7.1" }, { name = "ruff" }, { name = "time-machine" }, + { name = "websockets", specifier = ">=13,<16" }, ] pydantic-v1 = [{ name = "pydantic", specifier = ">=1.9.0,<2" }] pydantic-v2 = [ @@ -1831,6 +1837,65 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/68/5a/199c59e0a824a3db2b89c5d2dade7ab5f9624dbf6448dc291b46d5ec94d3/wcwidth-0.6.0-py3-none-any.whl", hash = "sha256:1a3a1e510b553315f8e146c54764f4fb6264ffad731b3d78088cdb1478ffbdad", size = 94189, upload-time = "2026-02-06T19:19:39.646Z" }, ] +[[package]] +name = "websockets" +version = "15.0.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/21/e6/26d09fab466b7ca9c7737474c52be4f76a40301b08362eb2dbc19dcc16c1/websockets-15.0.1.tar.gz", hash = "sha256:82544de02076bafba038ce055ee6412d68da13ab47f0c60cab827346de828dee", size = 177016, upload-time = "2025-03-05T20:03:41.606Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1e/da/6462a9f510c0c49837bbc9345aca92d767a56c1fb2939e1579df1e1cdcf7/websockets-15.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d63efaa0cd96cf0c5fe4d581521d9fa87744540d4bc999ae6e08595a1014b45b", size = 175423, upload-time = "2025-03-05T20:01:35.363Z" }, + { url = "https://files.pythonhosted.org/packages/1c/9f/9d11c1a4eb046a9e106483b9ff69bce7ac880443f00e5ce64261b47b07e7/websockets-15.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ac60e3b188ec7574cb761b08d50fcedf9d77f1530352db4eef1707fe9dee7205", size = 173080, upload-time = "2025-03-05T20:01:37.304Z" }, + { url = "https://files.pythonhosted.org/packages/d5/4f/b462242432d93ea45f297b6179c7333dd0402b855a912a04e7fc61c0d71f/websockets-15.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5756779642579d902eed757b21b0164cd6fe338506a8083eb58af5c372e39d9a", size = 173329, upload-time = "2025-03-05T20:01:39.668Z" }, + { url = "https://files.pythonhosted.org/packages/6e/0c/6afa1f4644d7ed50284ac59cc70ef8abd44ccf7d45850d989ea7310538d0/websockets-15.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0fdfe3e2a29e4db3659dbd5bbf04560cea53dd9610273917799f1cde46aa725e", size = 182312, upload-time = "2025-03-05T20:01:41.815Z" }, + { url = "https://files.pythonhosted.org/packages/dd/d4/ffc8bd1350b229ca7a4db2a3e1c482cf87cea1baccd0ef3e72bc720caeec/websockets-15.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c2529b320eb9e35af0fa3016c187dffb84a3ecc572bcee7c3ce302bfeba52bf", size = 181319, upload-time = "2025-03-05T20:01:43.967Z" }, + { url = "https://files.pythonhosted.org/packages/97/3a/5323a6bb94917af13bbb34009fac01e55c51dfde354f63692bf2533ffbc2/websockets-15.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ac1e5c9054fe23226fb11e05a6e630837f074174c4c2f0fe442996112a6de4fb", size = 181631, upload-time = "2025-03-05T20:01:46.104Z" }, + { url = "https://files.pythonhosted.org/packages/a6/cc/1aeb0f7cee59ef065724041bb7ed667b6ab1eeffe5141696cccec2687b66/websockets-15.0.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5df592cd503496351d6dc14f7cdad49f268d8e618f80dce0cd5a36b93c3fc08d", size = 182016, upload-time = "2025-03-05T20:01:47.603Z" }, + { url = "https://files.pythonhosted.org/packages/79/f9/c86f8f7af208e4161a7f7e02774e9d0a81c632ae76db2ff22549e1718a51/websockets-15.0.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:0a34631031a8f05657e8e90903e656959234f3a04552259458aac0b0f9ae6fd9", size = 181426, upload-time = "2025-03-05T20:01:48.949Z" }, + { url = "https://files.pythonhosted.org/packages/c7/b9/828b0bc6753db905b91df6ae477c0b14a141090df64fb17f8a9d7e3516cf/websockets-15.0.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:3d00075aa65772e7ce9e990cab3ff1de702aa09be3940d1dc88d5abf1ab8a09c", size = 181360, upload-time = "2025-03-05T20:01:50.938Z" }, + { url = "https://files.pythonhosted.org/packages/89/fb/250f5533ec468ba6327055b7d98b9df056fb1ce623b8b6aaafb30b55d02e/websockets-15.0.1-cp310-cp310-win32.whl", hash = "sha256:1234d4ef35db82f5446dca8e35a7da7964d02c127b095e172e54397fb6a6c256", size = 176388, upload-time = "2025-03-05T20:01:52.213Z" }, + { url = "https://files.pythonhosted.org/packages/1c/46/aca7082012768bb98e5608f01658ff3ac8437e563eca41cf068bd5849a5e/websockets-15.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:39c1fec2c11dc8d89bba6b2bf1556af381611a173ac2b511cf7231622058af41", size = 176830, upload-time = "2025-03-05T20:01:53.922Z" }, + { url = "https://files.pythonhosted.org/packages/9f/32/18fcd5919c293a398db67443acd33fde142f283853076049824fc58e6f75/websockets-15.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:823c248b690b2fd9303ba00c4f66cd5e2d8c3ba4aa968b2779be9532a4dad431", size = 175423, upload-time = "2025-03-05T20:01:56.276Z" }, + { url = "https://files.pythonhosted.org/packages/76/70/ba1ad96b07869275ef42e2ce21f07a5b0148936688c2baf7e4a1f60d5058/websockets-15.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678999709e68425ae2593acf2e3ebcbcf2e69885a5ee78f9eb80e6e371f1bf57", size = 173082, upload-time = "2025-03-05T20:01:57.563Z" }, + { url = "https://files.pythonhosted.org/packages/86/f2/10b55821dd40eb696ce4704a87d57774696f9451108cff0d2824c97e0f97/websockets-15.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d50fd1ee42388dcfb2b3676132c78116490976f1300da28eb629272d5d93e905", size = 173330, upload-time = "2025-03-05T20:01:59.063Z" }, + { url = "https://files.pythonhosted.org/packages/a5/90/1c37ae8b8a113d3daf1065222b6af61cc44102da95388ac0018fcb7d93d9/websockets-15.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d99e5546bf73dbad5bf3547174cd6cb8ba7273062a23808ffea025ecb1cf8562", size = 182878, upload-time = "2025-03-05T20:02:00.305Z" }, + { url = "https://files.pythonhosted.org/packages/8e/8d/96e8e288b2a41dffafb78e8904ea7367ee4f891dafc2ab8d87e2124cb3d3/websockets-15.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:66dd88c918e3287efc22409d426c8f729688d89a0c587c88971a0faa2c2f3792", size = 181883, upload-time = "2025-03-05T20:02:03.148Z" }, + { url = "https://files.pythonhosted.org/packages/93/1f/5d6dbf551766308f6f50f8baf8e9860be6182911e8106da7a7f73785f4c4/websockets-15.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8dd8327c795b3e3f219760fa603dcae1dcc148172290a8ab15158cf85a953413", size = 182252, upload-time = "2025-03-05T20:02:05.29Z" }, + { url = "https://files.pythonhosted.org/packages/d4/78/2d4fed9123e6620cbf1706c0de8a1632e1a28e7774d94346d7de1bba2ca3/websockets-15.0.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:8fdc51055e6ff4adeb88d58a11042ec9a5eae317a0a53d12c062c8a8865909e8", size = 182521, upload-time = "2025-03-05T20:02:07.458Z" }, + { url = "https://files.pythonhosted.org/packages/e7/3b/66d4c1b444dd1a9823c4a81f50231b921bab54eee2f69e70319b4e21f1ca/websockets-15.0.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:693f0192126df6c2327cce3baa7c06f2a117575e32ab2308f7f8216c29d9e2e3", size = 181958, upload-time = "2025-03-05T20:02:09.842Z" }, + { url = "https://files.pythonhosted.org/packages/08/ff/e9eed2ee5fed6f76fdd6032ca5cd38c57ca9661430bb3d5fb2872dc8703c/websockets-15.0.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:54479983bd5fb469c38f2f5c7e3a24f9a4e70594cd68cd1fa6b9340dadaff7cf", size = 181918, upload-time = "2025-03-05T20:02:11.968Z" }, + { url = "https://files.pythonhosted.org/packages/d8/75/994634a49b7e12532be6a42103597b71098fd25900f7437d6055ed39930a/websockets-15.0.1-cp311-cp311-win32.whl", hash = "sha256:16b6c1b3e57799b9d38427dda63edcbe4926352c47cf88588c0be4ace18dac85", size = 176388, upload-time = "2025-03-05T20:02:13.32Z" }, + { url = "https://files.pythonhosted.org/packages/98/93/e36c73f78400a65f5e236cd376713c34182e6663f6889cd45a4a04d8f203/websockets-15.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:27ccee0071a0e75d22cb35849b1db43f2ecd3e161041ac1ee9d2352ddf72f065", size = 176828, upload-time = "2025-03-05T20:02:14.585Z" }, + { url = "https://files.pythonhosted.org/packages/51/6b/4545a0d843594f5d0771e86463606a3988b5a09ca5123136f8a76580dd63/websockets-15.0.1-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:3e90baa811a5d73f3ca0bcbf32064d663ed81318ab225ee4f427ad4e26e5aff3", size = 175437, upload-time = "2025-03-05T20:02:16.706Z" }, + { url = "https://files.pythonhosted.org/packages/f4/71/809a0f5f6a06522af902e0f2ea2757f71ead94610010cf570ab5c98e99ed/websockets-15.0.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:592f1a9fe869c778694f0aa806ba0374e97648ab57936f092fd9d87f8bc03665", size = 173096, upload-time = "2025-03-05T20:02:18.832Z" }, + { url = "https://files.pythonhosted.org/packages/3d/69/1a681dd6f02180916f116894181eab8b2e25b31e484c5d0eae637ec01f7c/websockets-15.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:0701bc3cfcb9164d04a14b149fd74be7347a530ad3bbf15ab2c678a2cd3dd9a2", size = 173332, upload-time = "2025-03-05T20:02:20.187Z" }, + { url = "https://files.pythonhosted.org/packages/a6/02/0073b3952f5bce97eafbb35757f8d0d54812b6174ed8dd952aa08429bcc3/websockets-15.0.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8b56bdcdb4505c8078cb6c7157d9811a85790f2f2b3632c7d1462ab5783d215", size = 183152, upload-time = "2025-03-05T20:02:22.286Z" }, + { url = "https://files.pythonhosted.org/packages/74/45/c205c8480eafd114b428284840da0b1be9ffd0e4f87338dc95dc6ff961a1/websockets-15.0.1-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0af68c55afbd5f07986df82831c7bff04846928ea8d1fd7f30052638788bc9b5", size = 182096, upload-time = "2025-03-05T20:02:24.368Z" }, + { url = "https://files.pythonhosted.org/packages/14/8f/aa61f528fba38578ec553c145857a181384c72b98156f858ca5c8e82d9d3/websockets-15.0.1-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:64dee438fed052b52e4f98f76c5790513235efaa1ef7f3f2192c392cd7c91b65", size = 182523, upload-time = "2025-03-05T20:02:25.669Z" }, + { url = "https://files.pythonhosted.org/packages/ec/6d/0267396610add5bc0d0d3e77f546d4cd287200804fe02323797de77dbce9/websockets-15.0.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:d5f6b181bb38171a8ad1d6aa58a67a6aa9d4b38d0f8c5f496b9e42561dfc62fe", size = 182790, upload-time = "2025-03-05T20:02:26.99Z" }, + { url = "https://files.pythonhosted.org/packages/02/05/c68c5adbf679cf610ae2f74a9b871ae84564462955d991178f95a1ddb7dd/websockets-15.0.1-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:5d54b09eba2bada6011aea5375542a157637b91029687eb4fdb2dab11059c1b4", size = 182165, upload-time = "2025-03-05T20:02:30.291Z" }, + { url = "https://files.pythonhosted.org/packages/29/93/bb672df7b2f5faac89761cb5fa34f5cec45a4026c383a4b5761c6cea5c16/websockets-15.0.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3be571a8b5afed347da347bfcf27ba12b069d9d7f42cb8c7028b5e98bbb12597", size = 182160, upload-time = "2025-03-05T20:02:31.634Z" }, + { url = "https://files.pythonhosted.org/packages/ff/83/de1f7709376dc3ca9b7eeb4b9a07b4526b14876b6d372a4dc62312bebee0/websockets-15.0.1-cp312-cp312-win32.whl", hash = "sha256:c338ffa0520bdb12fbc527265235639fb76e7bc7faafbb93f6ba80d9c06578a9", size = 176395, upload-time = "2025-03-05T20:02:33.017Z" }, + { url = "https://files.pythonhosted.org/packages/7d/71/abf2ebc3bbfa40f391ce1428c7168fb20582d0ff57019b69ea20fa698043/websockets-15.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:fcd5cf9e305d7b8338754470cf69cf81f420459dbae8a3b40cee57417f4614a7", size = 176841, upload-time = "2025-03-05T20:02:34.498Z" }, + { url = "https://files.pythonhosted.org/packages/cb/9f/51f0cf64471a9d2b4d0fc6c534f323b664e7095640c34562f5182e5a7195/websockets-15.0.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:ee443ef070bb3b6ed74514f5efaa37a252af57c90eb33b956d35c8e9c10a1931", size = 175440, upload-time = "2025-03-05T20:02:36.695Z" }, + { url = "https://files.pythonhosted.org/packages/8a/05/aa116ec9943c718905997412c5989f7ed671bc0188ee2ba89520e8765d7b/websockets-15.0.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:5a939de6b7b4e18ca683218320fc67ea886038265fd1ed30173f5ce3f8e85675", size = 173098, upload-time = "2025-03-05T20:02:37.985Z" }, + { url = "https://files.pythonhosted.org/packages/ff/0b/33cef55ff24f2d92924923c99926dcce78e7bd922d649467f0eda8368923/websockets-15.0.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:746ee8dba912cd6fc889a8147168991d50ed70447bf18bcda7039f7d2e3d9151", size = 173329, upload-time = "2025-03-05T20:02:39.298Z" }, + { url = "https://files.pythonhosted.org/packages/31/1d/063b25dcc01faa8fada1469bdf769de3768b7044eac9d41f734fd7b6ad6d/websockets-15.0.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:595b6c3969023ecf9041b2936ac3827e4623bfa3ccf007575f04c5a6aa318c22", size = 183111, upload-time = "2025-03-05T20:02:40.595Z" }, + { url = "https://files.pythonhosted.org/packages/93/53/9a87ee494a51bf63e4ec9241c1ccc4f7c2f45fff85d5bde2ff74fcb68b9e/websockets-15.0.1-cp313-cp313-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3c714d2fc58b5ca3e285461a4cc0c9a66bd0e24c5da9911e30158286c9b5be7f", size = 182054, upload-time = "2025-03-05T20:02:41.926Z" }, + { url = "https://files.pythonhosted.org/packages/ff/b2/83a6ddf56cdcbad4e3d841fcc55d6ba7d19aeb89c50f24dd7e859ec0805f/websockets-15.0.1-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f3c1e2ab208db911594ae5b4f79addeb3501604a165019dd221c0bdcabe4db8", size = 182496, upload-time = "2025-03-05T20:02:43.304Z" }, + { url = "https://files.pythonhosted.org/packages/98/41/e7038944ed0abf34c45aa4635ba28136f06052e08fc2168520bb8b25149f/websockets-15.0.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:229cf1d3ca6c1804400b0a9790dc66528e08a6a1feec0d5040e8b9eb14422375", size = 182829, upload-time = "2025-03-05T20:02:48.812Z" }, + { url = "https://files.pythonhosted.org/packages/e0/17/de15b6158680c7623c6ef0db361da965ab25d813ae54fcfeae2e5b9ef910/websockets-15.0.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:756c56e867a90fb00177d530dca4b097dd753cde348448a1012ed6c5131f8b7d", size = 182217, upload-time = "2025-03-05T20:02:50.14Z" }, + { url = "https://files.pythonhosted.org/packages/33/2b/1f168cb6041853eef0362fb9554c3824367c5560cbdaad89ac40f8c2edfc/websockets-15.0.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:558d023b3df0bffe50a04e710bc87742de35060580a293c2a984299ed83bc4e4", size = 182195, upload-time = "2025-03-05T20:02:51.561Z" }, + { url = "https://files.pythonhosted.org/packages/86/eb/20b6cdf273913d0ad05a6a14aed4b9a85591c18a987a3d47f20fa13dcc47/websockets-15.0.1-cp313-cp313-win32.whl", hash = "sha256:ba9e56e8ceeeedb2e080147ba85ffcd5cd0711b89576b83784d8605a7df455fa", size = 176393, upload-time = "2025-03-05T20:02:53.814Z" }, + { url = "https://files.pythonhosted.org/packages/1b/6c/c65773d6cab416a64d191d6ee8a8b1c68a09970ea6909d16965d26bfed1e/websockets-15.0.1-cp313-cp313-win_amd64.whl", hash = "sha256:e09473f095a819042ecb2ab9465aee615bd9c2028e4ef7d933600a8401c79561", size = 176837, upload-time = "2025-03-05T20:02:55.237Z" }, + { url = "https://files.pythonhosted.org/packages/02/9e/d40f779fa16f74d3468357197af8d6ad07e7c5a27ea1ca74ceb38986f77a/websockets-15.0.1-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:0c9e74d766f2818bb95f84c25be4dea09841ac0f734d1966f415e4edfc4ef1c3", size = 173109, upload-time = "2025-03-05T20:03:17.769Z" }, + { url = "https://files.pythonhosted.org/packages/bc/cd/5b887b8585a593073fd92f7c23ecd3985cd2c3175025a91b0d69b0551372/websockets-15.0.1-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:1009ee0c7739c08a0cd59de430d6de452a55e42d6b522de7aa15e6f67db0b8e1", size = 173343, upload-time = "2025-03-05T20:03:19.094Z" }, + { url = "https://files.pythonhosted.org/packages/fe/ae/d34f7556890341e900a95acf4886833646306269f899d58ad62f588bf410/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76d1f20b1c7a2fa82367e04982e708723ba0e7b8d43aa643d3dcd404d74f1475", size = 174599, upload-time = "2025-03-05T20:03:21.1Z" }, + { url = "https://files.pythonhosted.org/packages/71/e6/5fd43993a87db364ec60fc1d608273a1a465c0caba69176dd160e197ce42/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f29d80eb9a9263b8d109135351caf568cc3f80b9928bccde535c235de55c22d9", size = 174207, upload-time = "2025-03-05T20:03:23.221Z" }, + { url = "https://files.pythonhosted.org/packages/2b/fb/c492d6daa5ec067c2988ac80c61359ace5c4c674c532985ac5a123436cec/websockets-15.0.1-pp310-pypy310_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b359ed09954d7c18bbc1680f380c7301f92c60bf924171629c5db97febb12f04", size = 174155, upload-time = "2025-03-05T20:03:25.321Z" }, + { url = "https://files.pythonhosted.org/packages/68/a1/dcb68430b1d00b698ae7a7e0194433bce4f07ded185f0ee5fb21e2a2e91e/websockets-15.0.1-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:cad21560da69f4ce7658ca2cb83138fb4cf695a2ba3e475e0559e05991aa8122", size = 176884, upload-time = "2025-03-05T20:03:27.934Z" }, + { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, +] + [[package]] name = "yarl" version = "1.22.0"