|
| 1 | +"""Audio helpers shared by the LLM Observability integrations. |
| 2 | +
|
| 3 | +Building ``AudioPart``s, mapping provider audio formats to MIME types, and turning raw audio |
| 4 | +(PCM16, G.711) into playable WAV within the per-span-event size budget. Kept separate from the |
| 5 | +larger ``utils.py`` since the audio surface has grown (chat audio, realtime, telephony). |
| 6 | +""" |
| 7 | + |
| 8 | +import base64 |
| 9 | +import io |
| 10 | +import struct |
| 11 | +from typing import Any |
| 12 | +from typing import Optional |
| 13 | +from typing import Union |
| 14 | +import wave |
| 15 | + |
| 16 | +from ddtrace.internal.logger import get_logger |
| 17 | +from ddtrace.llmobs._utils import _get_attr |
| 18 | +from ddtrace.llmobs.types import AudioPart |
| 19 | + |
| 20 | + |
| 21 | +logger = get_logger(__name__) |
| 22 | + |
| 23 | + |
| 24 | +def format_audio_part(data: Union[bytes, str], mime_type: str) -> AudioPart: |
| 25 | + """Build an ``AudioPart`` from raw audio bytes (base64-encoded) or an existing base64 string.""" |
| 26 | + content = base64.b64encode(data).decode("utf-8") if isinstance(data, bytes) else data |
| 27 | + return AudioPart(mime_type=mime_type, content=content) |
| 28 | + |
| 29 | + |
| 30 | +# OpenAI audio ``format`` values that don't map to ``audio/<format>``. |
| 31 | +_OPENAI_AUDIO_MIME_TYPES = { |
| 32 | + "mp3": "audio/mpeg", |
| 33 | +} |
| 34 | + |
| 35 | + |
| 36 | +def audio_mime_type_from_format(fmt: str) -> str: |
| 37 | + """Map an OpenAI audio ``format`` (e.g. "wav", "mp3") to a MIME type.""" |
| 38 | + fmt = (fmt or "").strip().lower() |
| 39 | + return _OPENAI_AUDIO_MIME_TYPES.get(fmt, "audio/{}".format(fmt) if fmt else "audio/wav") |
| 40 | + |
| 41 | + |
| 42 | +# Raw audio formats the UI cannot render as a player. For these we keep the transcript as the |
| 43 | +# message content and skip the inline audio_part (raw bytes would only bloat the payload). |
| 44 | +_NON_RENDERABLE_AUDIO_MIME_TYPES = frozenset( |
| 45 | + { |
| 46 | + "audio/pcm", |
| 47 | + "audio/pcm16", |
| 48 | + "audio/l16", |
| 49 | + "audio/pcmu", |
| 50 | + "audio/pcma", |
| 51 | + "audio/g711_ulaw", |
| 52 | + "audio/g711_alaw", |
| 53 | + "audio/basic", |
| 54 | + } |
| 55 | +) |
| 56 | + |
| 57 | +# Budget for inline audio measured on the *base64-encoded* size (what actually rides the span |
| 58 | +# event), so the guard reflects the real payload after encoding. Kept below the 5 MB per-span-event |
| 59 | +# limit with headroom for the rest of the event (oversize events have their whole I/O dropped |
| 60 | +# backend-side). 4 MiB encoded ≈ 3 MiB of raw audio. |
| 61 | +LLMOBS_AUDIO_INLINE_MAX_BYTES = 4 * 1024 * 1024 |
| 62 | + |
| 63 | + |
| 64 | +def _base64_encoded_len(num_bytes: int) -> int: |
| 65 | + """Length of ``num_bytes`` after standard base64 encoding (4 chars per 3 bytes, padded).""" |
| 66 | + return ((num_bytes + 2) // 3) * 4 |
| 67 | + |
| 68 | + |
| 69 | +# OpenAI Realtime audio ``format`` values (legacy string form) that don't map to ``audio/<format>``. |
| 70 | +_REALTIME_AUDIO_FORMAT_MIME_TYPES = { |
| 71 | + "pcm16": "audio/pcm", |
| 72 | + "pcm": "audio/pcm", |
| 73 | + "g711_ulaw": "audio/pcmu", |
| 74 | + "g711_alaw": "audio/pcma", |
| 75 | +} |
| 76 | + |
| 77 | + |
| 78 | +def realtime_audio_format_to_mime(fmt: Any) -> str: |
| 79 | + """Map an OpenAI Realtime audio format to a MIME type. |
| 80 | +
|
| 81 | + Handles both the legacy string form (e.g. "pcm16", "g711_ulaw") and the newer |
| 82 | + discriminated-union object whose ``type`` is already a MIME type (e.g. "audio/pcm"). |
| 83 | + """ |
| 84 | + if fmt is None: |
| 85 | + return "" |
| 86 | + type_attr = _get_attr(fmt, "type", None) |
| 87 | + if type_attr: |
| 88 | + fmt = type_attr |
| 89 | + normalized = str(fmt).strip().lower() |
| 90 | + if not normalized: |
| 91 | + return "" |
| 92 | + if normalized.startswith("audio/"): |
| 93 | + return normalized |
| 94 | + return _REALTIME_AUDIO_FORMAT_MIME_TYPES.get(normalized, "audio/{}".format(normalized)) |
| 95 | + |
| 96 | + |
| 97 | +def is_renderable_audio_mime(mime_type: str) -> bool: |
| 98 | + """Whether a MIME type can be rendered as an audio player in the UI (raw PCM cannot).""" |
| 99 | + return bool(mime_type) and mime_type.strip().lower() not in _NON_RENDERABLE_AUDIO_MIME_TYPES |
| 100 | + |
| 101 | + |
| 102 | +def concat_base64_audio(b64_chunks: "list[str]") -> bytes: |
| 103 | + """Decode and concatenate a list of base64 audio chunks into raw bytes. |
| 104 | +
|
| 105 | + Chunks must be decoded before concatenation: directly joining base64 strings is invalid |
| 106 | + unless each chunk is aligned on a 3-byte boundary. |
| 107 | + """ |
| 108 | + buf = bytearray() |
| 109 | + for chunk in b64_chunks: |
| 110 | + if not chunk: |
| 111 | + continue |
| 112 | + try: |
| 113 | + buf.extend(base64.b64decode(chunk)) |
| 114 | + except (ValueError, TypeError): |
| 115 | + continue |
| 116 | + return bytes(buf) |
| 117 | + |
| 118 | + |
| 119 | +# Raw little-endian PCM16 mime types. These aren't renderable on their own, but can be losslessly |
| 120 | +# wrapped in a WAV container (just a header) to produce a playable ``audio/wav`` part. |
| 121 | +_PCM16_MIME_TYPES = frozenset({"audio/pcm", "audio/pcm16", "audio/l16"}) |
| 122 | + |
| 123 | + |
| 124 | +def is_pcm16_audio_mime(mime_type: str) -> bool: |
| 125 | + return bool(mime_type) and mime_type.strip().lower() in _PCM16_MIME_TYPES |
| 126 | + |
| 127 | + |
| 128 | +def pcm16_to_wav(pcm_bytes: bytes, sample_rate: int = 24000, channels: int = 1) -> bytes: |
| 129 | + """Wrap raw little-endian PCM16 audio in a WAV container. |
| 130 | +
|
| 131 | + This is lossless and cheap (it only prepends a header), and turns raw PCM — which the UI can't |
| 132 | + render — into a playable ``audio/wav`` payload. |
| 133 | + """ |
| 134 | + buf = io.BytesIO() |
| 135 | + with wave.open(buf, "wb") as wav_file: |
| 136 | + wav_file.setnchannels(channels) |
| 137 | + wav_file.setsampwidth(2) # PCM16 = 2 bytes/sample |
| 138 | + wav_file.setframerate(sample_rate) |
| 139 | + wav_file.writeframes(pcm_bytes) |
| 140 | + return buf.getvalue() |
| 141 | + |
| 142 | + |
| 143 | +# G.711 telephony audio (8kHz, 8-bit companded). Realtime uses these for phone-call integrations |
| 144 | +# (Twilio/SIP). We decode to PCM16 so it can be WAV-wrapped and played in the UI; the stdlib `wave` |
| 145 | +# module can't emit G.711-tagged WAV, and `audioop` (ulaw2lin/alaw2lin) was removed in Python 3.13. |
| 146 | +_G711_MIME_TO_VARIANT = { |
| 147 | + "audio/pcmu": "ulaw", |
| 148 | + "audio/g711_ulaw": "ulaw", |
| 149 | + "audio/pcma": "alaw", |
| 150 | + "audio/g711_alaw": "alaw", |
| 151 | +} |
| 152 | +G711_SAMPLE_RATE = 8000 # G.711 is always 8kHz mono. |
| 153 | + |
| 154 | + |
| 155 | +def g711_variant(mime_type: str) -> Optional[str]: |
| 156 | + """Return "ulaw"/"alaw" for a G.711 MIME type, else ``None``.""" |
| 157 | + return _G711_MIME_TO_VARIANT.get((mime_type or "").strip().lower()) |
| 158 | + |
| 159 | + |
| 160 | +def _ulaw_decode_sample(byte: int) -> int: |
| 161 | + """Decode one G.711 μ-law byte to a signed 16-bit linear PCM sample (CCITT G.711).""" |
| 162 | + byte = ~byte & 0xFF |
| 163 | + sample = ((byte & 0x0F) << 3) + 0x84 |
| 164 | + sample <<= (byte & 0x70) >> 4 |
| 165 | + return (0x84 - sample) if (byte & 0x80) else (sample - 0x84) |
| 166 | + |
| 167 | + |
| 168 | +def _alaw_decode_sample(byte: int) -> int: |
| 169 | + """Decode one G.711 A-law byte to a signed 16-bit linear PCM sample (CCITT G.711).""" |
| 170 | + byte ^= 0x55 |
| 171 | + sample = (byte & 0x0F) << 4 |
| 172 | + seg = (byte & 0x70) >> 4 |
| 173 | + if seg == 0: |
| 174 | + sample += 8 |
| 175 | + elif seg == 1: |
| 176 | + sample += 0x108 |
| 177 | + else: |
| 178 | + sample = (sample + 0x108) << (seg - 1) |
| 179 | + return sample if (byte & 0x80) else -sample |
| 180 | + |
| 181 | + |
| 182 | +# Precompute 256-entry decode tables (the input domain is a single byte). |
| 183 | +_ULAW_TABLE = [_ulaw_decode_sample(i) for i in range(256)] |
| 184 | +_ALAW_TABLE = [_alaw_decode_sample(i) for i in range(256)] |
| 185 | + |
| 186 | + |
| 187 | +def g711_to_pcm16(data: bytes, variant: str) -> bytes: |
| 188 | + """Decode G.711 ("ulaw"/"alaw") bytes to raw little-endian PCM16.""" |
| 189 | + table = _ALAW_TABLE if variant == "alaw" else _ULAW_TABLE |
| 190 | + return b"".join(struct.pack("<h", table[b]) for b in data) |
| 191 | + |
| 192 | + |
| 193 | +def format_audio_part_with_guard( |
| 194 | + audio_bytes: bytes, mime_type: str, max_bytes: int = LLMOBS_AUDIO_INLINE_MAX_BYTES |
| 195 | +) -> Optional[AudioPart]: |
| 196 | + """Build a playable ``AudioPart`` only for renderable formats within the size budget. |
| 197 | +
|
| 198 | + Returns ``None`` for non-renderable formats (e.g. raw PCM) or oversize audio; callers should |
| 199 | + fall back to the transcript as the message content in that case. |
| 200 | + """ |
| 201 | + if not audio_bytes or not is_renderable_audio_mime(mime_type): |
| 202 | + return None |
| 203 | + # Compare the *encoded* size: format_audio_part base64-encodes the bytes (~4/3 expansion), and |
| 204 | + # that encoded content is what counts against the per-span-event limit. |
| 205 | + encoded_len = _base64_encoded_len(len(audio_bytes)) |
| 206 | + if encoded_len > max_bytes: |
| 207 | + logger.debug( |
| 208 | + "Audio (%d encoded bytes) exceeds inline budget %d; omitting inline audio content", encoded_len, max_bytes |
| 209 | + ) |
| 210 | + return None |
| 211 | + return format_audio_part(audio_bytes, mime_type) |
![gh-worker-dd-mergequeue-cf854d[bot]](https://avatars.githubusercontent.com/u/244837012?v=4&size=40){kind=avatar}
0 commit comments