From c1e4763af01140a0cb0e242655d9882d6c44b1d7 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 11:39:35 -0700 Subject: [PATCH 01/31] add support for packet trailer features --- examples/demo/demo.ts | 33 ++++ examples/demo/index.html | 4 + examples/demo/styles.css | 16 ++ package.json | 10 +- rollup.config.pt-worker.js | 25 +++ src/ConnectionSession.ts | 36 ++++ src/e2ee/E2eeManager.ts | 6 +- src/e2ee/createBuiltinE2EEWorker.ts | 27 +++ src/e2ee/packetTrailer.test.ts | 24 +++ src/e2ee/packetTrailer.ts | 177 +++++++++++++++++ src/index.ts | 2 + src/options.ts | 8 + src/packetTrailer/PacketTrailerManager.ts | 179 ++++++++++++++++++ src/packetTrailer/types.ts | 47 +++++ .../worker/packetTrailer.worker.ts | 85 +++++++++ src/packetTrailer/worker/tsconfig.json | 14 ++ src/room/RTCEngine.ts | 4 +- src/room/Room.ts | 26 +++ src/room/track/PacketTrailerExtractor.ts | 124 ++++++++++++ src/room/track/RemoteVideoTrack.ts | 23 +++ src/room/track/utils.ts | 5 +- src/version.ts | 2 +- 22 files changed, 871 insertions(+), 6 deletions(-) create mode 100644 rollup.config.pt-worker.js create mode 100644 src/ConnectionSession.ts create mode 100644 src/e2ee/createBuiltinE2EEWorker.ts create mode 100644 src/e2ee/packetTrailer.test.ts create mode 100644 src/e2ee/packetTrailer.ts create mode 100644 src/packetTrailer/PacketTrailerManager.ts create mode 100644 src/packetTrailer/types.ts create mode 100644 src/packetTrailer/worker/packetTrailer.worker.ts create mode 100644 src/packetTrailer/worker/tsconfig.json create mode 100644 src/room/track/PacketTrailerExtractor.ts diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 8b625152a7..3ec2a7938e 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -1,5 +1,7 @@ //@ts-ignore import E2EEWorker from '../../src/e2ee/worker/e2ee.worker?worker'; +//@ts-ignore +import PTWorker from '../../src/packetTrailer/worker/packetTrailer.worker?worker'; import type { ChatMessage, LocalDataTrack, @@ -23,6 +25,7 @@ import { ParticipantEvent, RemoteParticipant, RemoteTrackPublication, + RemoteVideoTrack, Room, RoomEvent, ScreenSharePresets, @@ -40,6 +43,7 @@ import { supportsAV1, supportsVP9, } from '../../src/index'; +import { TrackEvent } from '../../src/room/events'; import type { DataTrackFrame } from '../../src/room/data-track/frame'; import { isSVCCodec, sleep, supportsH265 } from '../../src/room/utils'; @@ -106,6 +110,7 @@ const appActions = { const cryptoKey = ($('crypto-key')).value; const autoSubscribe = ($('auto-subscribe')).checked; const e2eeEnabled = ($('e2ee')).checked; + const packetTrailerEnabled = ($('packet-trailer')).checked; const audioOutputId = ($('audio-output')).value; let backupCodecPolicy: BackupCodecPolicy | undefined; if (($('multicodec-simulcast')).checked) { @@ -137,6 +142,7 @@ const appActions = { encryption: e2eeEnabled ? { keyProvider: state.e2eeKeyProvider, worker: new E2EEWorker() } : undefined, + packetTrailer: packetTrailerEnabled ? { worker: new PTWorker() } : undefined, }; if ( roomOpts.publishDefaults?.videoCodec === 'av1' || @@ -243,6 +249,32 @@ const appActions = { appendLog('subscribed to track', pub.trackSid, participant.identity); renderParticipant(participant); renderScreenShare(room); + if (track instanceof RemoteVideoTrack) { + let lastLatencyUpdate = 0; + let latencyDisplay = ''; + track.on(TrackEvent.TimeSyncUpdate, ({ rtpTimestamp }) => { + const meta = track.lookupFrameMetadata({ rtpTimestamp }); + const overlayElm = document.getElementById(`pt-overlay-${participant.identity}`); + if (overlayElm && meta) { + const now = Date.now(); + const receiveTime = new Date(now); + const publishTime = new Date(meta.userTimestampUs / 1000); + if (now - lastLatencyUpdate >= 500) { + lastLatencyUpdate = now; + latencyDisplay = `${(receiveTime.getTime() - publishTime.getTime()).toFixed(1)}ms`; + } + const fmt = (d: Date) => { + const pad = (n: number, w = 2) => String(n).padStart(w, '0'); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}:${pad(d.getMilliseconds(), 4)}`; + }; + overlayElm.textContent = + `Frame ID: ${meta.frameId}\n` + + `Publish: ${fmt(publishTime)}\n` + + `Receive: ${fmt(receiveTime)}\n` + + `Latency: ${latencyDisplay}`; + } + }); + } }) .on(RoomEvent.TrackUnsubscribed, (_, pub, participant) => { appendLog('unsubscribed from track', pub.trackSid); @@ -850,6 +882,7 @@ function renderParticipant(participant: Participant, remove: boolean = false) { div.innerHTML = ` +
diff --git a/examples/demo/index.html b/examples/demo/index.html index 6fac900a81..4a41fce1ed 100644 --- a/examples/demo/index.html +++ b/examples/demo/index.html @@ -70,6 +70,10 @@

Livekit Sample App

+
+ + +
diff --git a/examples/demo/styles.css b/examples/demo/styles.css index 059a0815e5..cd2ea7f551 100644 --- a/examples/demo/styles.css +++ b/examples/demo/styles.css @@ -164,3 +164,19 @@ position: absolute; z-index: 4; } + +.participant .pt-overlay { + position: absolute; + top: 4px; + left: 4px; + padding: 2px 6px; + background-color: rgba(0, 0, 0, 0.6); + color: #fff; + font-family: monospace; + font-size: 11px; + line-height: 1.4; + border-radius: 3px; + z-index: 6; + pointer-events: none; + white-space: pre; +} diff --git a/package.json b/package.json index 02c97a227f..fedd071b0e 100644 --- a/package.json +++ b/package.json @@ -15,6 +15,11 @@ "types": "./dist/src/e2ee/worker/e2ee.worker.d.ts", "import": "./dist/livekit-client.e2ee.worker.mjs", "require": "./dist/livekit-client.e2ee.worker.js" + }, + "./packet-trailer-worker": { + "types": "./dist/src/packetTrailer/worker/packetTrailer.worker.d.ts", + "import": "./dist/livekit-client.pt.worker.mjs", + "require": "./dist/livekit-client.pt.worker.js" } }, "files": [ @@ -29,6 +34,9 @@ ], "./dist/src/e2ee/worker/e2ee.worker.d.ts": [ "./dist/ts4.2/e2ee/worker/e2ee.worker.d.ts" + ], + "./dist/src/packetTrailer/worker/packetTrailer.worker.d.ts": [ + "./dist/ts4.2/packetTrailer/worker/packetTrailer.worker.d.ts" ] } }, @@ -36,7 +44,7 @@ "author": "LiveKit ", "license": "Apache-2.0", "scripts": { - "build": "rollup --config --bundleConfigAsCjs && rollup --config rollup.config.worker.js --bundleConfigAsCjs && pnpm downlevel-dts", + "build": "rollup --config --bundleConfigAsCjs && rollup --config rollup.config.worker.js --bundleConfigAsCjs && rollup --config rollup.config.pt-worker.js --bundleConfigAsCjs && pnpm downlevel-dts", "build:clean": "rm -rf ./dist && pnpm build", "build:watch": "rollup --watch --config --bundleConfigAsCjs", "build:worker:watch": "rollup --watch --config rollup.config.worker.js --bundleConfigAsCjs", diff --git a/rollup.config.pt-worker.js b/rollup.config.pt-worker.js new file mode 100644 index 0000000000..c6f00d04f8 --- /dev/null +++ b/rollup.config.pt-worker.js @@ -0,0 +1,25 @@ +import terser from '@rollup/plugin-terser'; +import typescript from 'rollup-plugin-typescript2'; +import packageJson from './package.json'; +import { commonPlugins, kebabCaseToPascalCase } from './rollup.config'; + +export default { + input: 'src/packetTrailer/worker/packetTrailer.worker.ts', + output: [ + { + file: `dist/${packageJson.name}.pt.worker.mjs`, + format: 'es', + strict: true, + sourcemap: true, + }, + { + file: `dist/${packageJson.name}.pt.worker.js`, + format: 'umd', + strict: true, + sourcemap: true, + name: kebabCaseToPascalCase(packageJson.name) + '.pt.worker', + plugins: [terser()], + }, + ], + plugins: [typescript({ tsconfig: './src/packetTrailer/worker/tsconfig.json' }), ...commonPlugins], +}; diff --git a/src/ConnectionSession.ts b/src/ConnectionSession.ts new file mode 100644 index 0000000000..0c33193b45 --- /dev/null +++ b/src/ConnectionSession.ts @@ -0,0 +1,36 @@ +import { type JoinRequest } from '@livekit/protocol'; +import { + SignalClient, + type JoinedRoomMembership, + type SignalOptions, +} from './api/SignalClient'; +import type { LoggerOptions } from './room/types'; + +export class ConnectionSession { + readonly signalClient: SignalClient; + + constructor(useJSON: boolean = false, loggerOptions: LoggerOptions = {}) { + this.signalClient = new SignalClient(useJSON, loggerOptions); + } + + async connect( + url: string, + token: string, + options: SignalOptions, + abortSignal?: AbortSignal, + ): Promise { + const joinResponse = await this.signalClient.join(url, token, options, abortSignal); + return { + joinResponse, + membershipId: this.signalClient.defaultMembershipId ?? '', + }; + } + + async joinRoom(token: string, joinRequest?: JoinRequest): Promise { + return this.signalClient.joinRoom(token, joinRequest); + } + + async close(reason?: string) { + await this.signalClient.close(true, reason); + } +} diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index 7bc0d4d71d..ec3af0c091 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -479,11 +479,15 @@ export class E2EEManager return; } + // @ts-ignore -- readableStream is set by PacketTrailerManager when it has pre-processed the receiver + const ptPreProcessed = !!receiver.readableStream && !!receiver.writableStream; + if ( isScriptTransformSupported() && // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141. // Disabling it for Chrome based browsers until the API has stabilized - !isChromiumBased() + !isChromiumBased() && + !ptPreProcessed ) { const options: ScriptTransformOptions = { kind: 'decode', diff --git a/src/e2ee/createBuiltinE2EEWorker.ts b/src/e2ee/createBuiltinE2EEWorker.ts new file mode 100644 index 0000000000..a071656391 --- /dev/null +++ b/src/e2ee/createBuiltinE2EEWorker.ts @@ -0,0 +1,27 @@ +import log from '../logger'; + +export function createBuiltinE2EEWorker() { + if (typeof Worker === 'undefined') { + return undefined; + } + + try { + return new Worker(new URL('./livekit-client.e2ee.worker.mjs', import.meta.url), { + type: 'module', + name: 'livekit-client-e2ee', + }); + } catch (moduleError) { + log.debug('failed to initialize module e2ee worker, falling back to classic worker', { + error: moduleError, + }); + } + + try { + return new Worker(new URL('./livekit-client.e2ee.worker.js', import.meta.url), { + name: 'livekit-client-e2ee', + }); + } catch (workerError) { + log.warn('failed to initialize built-in e2ee worker', { error: workerError }); + return undefined; + } +} diff --git a/src/e2ee/packetTrailer.test.ts b/src/e2ee/packetTrailer.test.ts new file mode 100644 index 0000000000..ba1b5dc9ae --- /dev/null +++ b/src/e2ee/packetTrailer.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from 'vitest'; +import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; + +describe('packetTrailer', () => { + it('extracts user timestamp and frame id using the Rust wire format', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456, 42); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestampUs: 1_744_249_600_123_456, + frameId: 42, + }); + }); + + it('passes frames through when there is no valid trailer', () => { + const payload = Uint8Array.from([1, 2, 3, 4, 5]); + const extracted = extractPacketTrailer(payload); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toBeUndefined(); + }); +}); diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts new file mode 100644 index 0000000000..5d6f863a68 --- /dev/null +++ b/src/e2ee/packetTrailer.ts @@ -0,0 +1,177 @@ +export const PACKET_TRAILER_MAGIC = Uint8Array.from([ + 'L'.charCodeAt(0), + 'K'.charCodeAt(0), + 'T'.charCodeAt(0), + 'S'.charCodeAt(0), +]); + +export const PACKET_TRAILER_TIMESTAMP_TAG = 0x01; +export const PACKET_TRAILER_FRAME_ID_TAG = 0x02; +export const PACKET_TRAILER_ENVELOPE_SIZE = 5; + +const TIMESTAMP_TLV_SIZE = 10; +const FRAME_ID_TLV_SIZE = 6; + +export interface PacketTrailerMetadata { + userTimestampUs: number; + frameId: number; +} + +export interface ExtractPacketTrailerResult { + data: Uint8Array; + metadata?: PacketTrailerMetadata; +} + +export function appendPacketTrailer( + data: Uint8Array, + userTimestampUs: number, + frameId: number, +): Uint8Array { + const hasFrameId = frameId !== 0; + const trailerLength = + TIMESTAMP_TLV_SIZE + (hasFrameId ? FRAME_ID_TLV_SIZE : 0) + PACKET_TRAILER_ENVELOPE_SIZE; + const result = new Uint8Array(data.length + trailerLength); + let offset = 0; + + result.set(data, offset); + offset += data.length; + + result[offset++] = PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff; + result[offset++] = 8 ^ 0xff; + writeUint64Xor(result, offset, userTimestampUs); + offset += 8; + + if (hasFrameId) { + result[offset++] = PACKET_TRAILER_FRAME_ID_TAG ^ 0xff; + result[offset++] = 4 ^ 0xff; + writeUint32Xor(result, offset, frameId); + offset += 4; + } + + result[offset++] = trailerLength ^ 0xff; + result.set(PACKET_TRAILER_MAGIC, offset); + + return result; +} + +export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPacketTrailerResult { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + if (bytes.length < PACKET_TRAILER_ENVELOPE_SIZE) { + return { data: bytes }; + } + + const magicOffset = bytes.length - PACKET_TRAILER_MAGIC.length; + if (!matchesMagic(bytes, magicOffset)) { + return { data: bytes }; + } + + const trailerLength = bytes[bytes.length - PACKET_TRAILER_ENVELOPE_SIZE] ^ 0xff; + if (trailerLength < PACKET_TRAILER_ENVELOPE_SIZE || trailerLength > bytes.length) { + return { data: bytes }; + } + + const trailerStart = bytes.length - trailerLength; + const trailerEnd = bytes.length - PACKET_TRAILER_ENVELOPE_SIZE; + const strippedData = bytes.subarray(0, trailerStart); + let offset = trailerStart; + let foundAny = false; + const metadata: PacketTrailerMetadata = { + userTimestampUs: 0, + frameId: 0, + }; + + while (offset + 2 <= trailerEnd) { + const tag = bytes[offset++] ^ 0xff; + const length = bytes[offset++] ^ 0xff; + + if (offset + length > trailerEnd) { + break; + } + + if (tag === PACKET_TRAILER_TIMESTAMP_TAG && length === 8) { + metadata.userTimestampUs = readUint64Xor(bytes, offset, length); + foundAny = true; + } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { + metadata.frameId = readUint32Xor(bytes, offset, length); + foundAny = true; + } + + offset += length; + } + + if (!foundAny) { + return { data: bytes }; + } + + return { data: strippedData, metadata }; +} + +function matchesMagic(data: Uint8Array, offset: number) { + for (let index = 0; index < PACKET_TRAILER_MAGIC.length; index += 1) { + if (data[offset + index] !== PACKET_TRAILER_MAGIC[index]) { + return false; + } + } + return true; +} + +function readUint64Xor(data: Uint8Array, offset: number, length: number) { + let value = 0; + for (let index = 0; index < length; index += 1) { + value = value * 256 + (data[offset + index] ^ 0xff); + } + return value; +} + +function readUint32Xor(data: Uint8Array, offset: number, length: number) { + let value = 0; + for (let index = 0; index < length; index += 1) { + value = (value << 8) | (data[offset + index] ^ 0xff); + } + return value >>> 0; +} + +function writeUint64Xor(target: Uint8Array, offset: number, value: number) { + let remaining = value; + for (let index = 7; index >= 0; index -= 1) { + const shift = 256 ** index; + const currentByte = Math.floor(remaining / shift); + target[offset + (7 - index)] = currentByte ^ 0xff; + remaining -= currentByte * shift; + } +} + +function writeUint32Xor(target: Uint8Array, offset: number, value: number) { + for (let index = 3; index >= 0; index -= 1) { + target[offset + (3 - index)] = ((value >> (index * 8)) & 0xff) ^ 0xff; + } +} + +export function getFrameRtpTimestamp( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, +): number | undefined { + try { + const metadata = frame.getMetadata() as Record; + if (typeof metadata.rtpTimestamp === 'number') { + return metadata.rtpTimestamp; + } + if (typeof metadata.timestamp === 'number') { + return metadata.timestamp; + } + } catch { + // getMetadata() might not be available + } + return undefined; +} + +export function getFrameSsrc( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, +): number { + try { + const metadata = frame.getMetadata() as Record; + if (typeof metadata.synchronizationSource === 'number') { + return metadata.synchronizationSource; + } + } catch {} + return 0; +} diff --git a/src/index.ts b/src/index.ts index 73abc26cf7..db7b484692 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,6 +63,8 @@ import { import { getBrowser } from './utils/browserParser'; export { RpcError, type RpcInvocationData, type PerformRpcParams } from './room/rpc'; +export type { PacketTrailerMetadata } from './e2ee/packetTrailer'; +export { PacketTrailerManager, type PacketTrailerOptions } from './packetTrailer/PacketTrailerManager'; export * from './connectionHelper/ConnectionCheck'; export * from './connectionHelper/checks/Checker'; diff --git a/src/options.ts b/src/options.ts index f562672220..9b0ef36e77 100644 --- a/src/options.ts +++ b/src/options.ts @@ -1,4 +1,5 @@ import type { E2EEOptions } from './e2ee/types'; +import type { PacketTrailerOptions } from './packetTrailer/PacketTrailerManager'; import type { ReconnectPolicy } from './room/ReconnectPolicy'; import type { AudioCaptureOptions, @@ -100,6 +101,13 @@ export interface InternalRoomOptions { loggerName?: string; + /** + * @experimental + * Options for enabling packet trailer extraction on received video tracks. + * Packet trailers carry frame-level metadata such as user timestamps and frame IDs. + */ + packetTrailer?: PacketTrailerOptions; + /** * will attempt to connect via single peer connection mode. * falls back to dual peer connection mode if not available. diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts new file mode 100644 index 0000000000..dfe6140051 --- /dev/null +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -0,0 +1,179 @@ +import type { TrackInfo } from '@livekit/protocol'; +import log from '../logger'; +import type Room from '../room/Room'; +import { RoomEvent } from '../room/events'; +import type RemoteTrack from '../room/track/RemoteTrack'; +import type RemoteVideoTrack from '../room/track/RemoteVideoTrack'; +import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; +import type { PTDecodeMessage, PTRemoveTransformMessage, PTWorkerMessage } from './types'; + +const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; + +export interface PacketTrailerOptions { + worker: Worker; +} + +/** + * Manages packet trailer extraction for received video tracks. + * + * When a track's TrackInfo indicates packet trailer features, the manager + * wires up an Insertable Streams pipeline (via a dedicated worker) to strip + * the trailer from encoded frames and cache the metadata for lookup. + * + * Works independently of E2EE. When both are active, the PT pipeline runs + * first and provides pre-processed streams that E2EE can consume. + * + * @experimental + */ +export class PacketTrailerManager { + private worker: Worker; + + private room?: Room; + + private extractors = new Map(); + + private preProcessedStreams = new Map< + RTCRtpReceiver, + { readable: ReadableStream; writable: WritableStream } + >(); + + constructor(options: PacketTrailerOptions) { + this.worker = options.worker; + } + + /** @internal */ + setup(room: Room) { + if (room === this.room) { + return; + } + this.room = room; + + this.worker.onmessage = this.onWorkerMessage; + this.worker.onerror = this.onWorkerError; + this.worker.postMessage({ kind: 'init' }); + + room + .on(RoomEvent.TrackSubscribed, (track, pub, _participant) => { + if (track.kind !== 'video') { + return; + } + log.info('PacketTrailerManager: subscribed video track', { + trackSid: pub.trackSid, + packetTrailerFeatures: pub.trackInfo?.packetTrailerFeatures, + trackInfo: pub.trackInfo, + }); + this.setupReceiver(track as unknown as RemoteVideoTrack, pub.trackInfo); + }) + .on(RoomEvent.TrackUnsubscribed, (track) => { + this.teardownTrack(track); + }); + } + + private setupReceiver(track: RemoteVideoTrack, _trackInfo?: TrackInfo) { + if (!track.receiver) { + return; + } + + if (PACKET_TRAILER_FLAG in track.receiver) { + return; + } + + const extractor = new PacketTrailerExtractor(); + const trackId = track.mediaStreamID; + + this.extractors.set(trackId, extractor); + track.packetTrailerExtractor = extractor; + + const receiver = track.receiver; + + if (!('createEncodedStreams' in receiver)) { + log.warn( + 'createEncodedStreams not supported, packet trailer extraction unavailable', + ); + return; + } + + // @ts-ignore -- createEncodedStreams is not in standard typings + const streams = receiver.createEncodedStreams(); + const hasE2EE = !!this.room?.hasE2EESetup; + + if (hasE2EE) { + const bridge = new TransformStream(); + const msg: PTDecodeMessage = { + kind: 'decode', + data: { + readableStream: streams.readable, + writableStream: bridge.writable, + trackId, + }, + }; + this.worker.postMessage(msg, [streams.readable, bridge.writable]); + + // @ts-ignore + receiver.readableStream = bridge.readable; + // @ts-ignore + receiver.writableStream = streams.writable; + + this.preProcessedStreams.set(receiver, { + readable: bridge.readable, + writable: streams.writable, + }); + } else { + const msg: PTDecodeMessage = { + kind: 'decode', + data: { + readableStream: streams.readable, + writableStream: streams.writable, + trackId, + }, + }; + this.worker.postMessage(msg, [streams.readable, streams.writable]); + } + + // @ts-ignore + receiver[PACKET_TRAILER_FLAG] = true; + } + + private teardownTrack(track: RemoteTrack) { + const trackId = track.mediaStreamID; + const extractor = this.extractors.get(trackId); + if (extractor) { + extractor.dispose(); + this.extractors.delete(trackId); + } + if (track.receiver) { + this.preProcessedStreams.delete(track.receiver); + } + + const msg: PTRemoveTransformMessage = { + kind: 'removeTransform', + data: { trackId }, + }; + this.worker.postMessage(msg); + } + + private onWorkerMessage = (ev: MessageEvent) => { + const msg = ev.data; + if (msg.kind === 'metadata') { + const extractor = this.extractors.get(msg.data.trackId); + if (extractor) { + extractor.storeMetadata(msg.data.rtpTimestamp, msg.data.ssrc, msg.data.metadata); + } + } + }; + + private onWorkerError = (ev: ErrorEvent) => { + log.error('packet trailer worker encountered an error:', { error: ev.error }); + }; + + /** + * Returns pre-processed streams for a receiver if the PT pipeline has already + * consumed the receiver's encoded streams. Used by E2EE to chain after PT. + * @internal + */ + getPreProcessedStreams( + receiver: RTCRtpReceiver, + ): { readable: ReadableStream; writable: WritableStream } | undefined { + return this.preProcessedStreams.get(receiver); + } +} diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts new file mode 100644 index 0000000000..1b7936c662 --- /dev/null +++ b/src/packetTrailer/types.ts @@ -0,0 +1,47 @@ +import type { PacketTrailerMetadata } from '../e2ee/packetTrailer'; + +export interface PTBaseMessage { + kind: string; + data?: unknown; +} + +export interface PTInitMessage extends PTBaseMessage { + kind: 'init'; +} + +export interface PTInitAck extends PTBaseMessage { + kind: 'initAck'; +} + +export interface PTDecodeMessage extends PTBaseMessage { + kind: 'decode'; + data: { + readableStream: ReadableStream; + writableStream: WritableStream; + trackId: string; + }; +} + +export interface PTRemoveTransformMessage extends PTBaseMessage { + kind: 'removeTransform'; + data: { + trackId: string; + }; +} + +export interface PTMetadataMessage extends PTBaseMessage { + kind: 'metadata'; + data: { + trackId: string; + rtpTimestamp: number; + ssrc: number; + metadata: PacketTrailerMetadata; + }; +} + +export type PTWorkerMessage = + | PTInitMessage + | PTInitAck + | PTDecodeMessage + | PTRemoveTransformMessage + | PTMetadataMessage; diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts new file mode 100644 index 0000000000..810d4c7ab4 --- /dev/null +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -0,0 +1,85 @@ +import { + extractPacketTrailer, + getFrameRtpTimestamp, + getFrameSsrc, +} from '../../e2ee/packetTrailer'; +import type { PTMetadataMessage, PTWorkerMessage } from '../types'; + +const activeTransforms = new Map(); + +onmessage = (ev: MessageEvent) => { + const msg = ev.data; + + switch (msg.kind) { + case 'init': + postMessage({ kind: 'initAck' }); + break; + + case 'decode': + setupDecodeTransform(msg.data.readableStream, msg.data.writableStream, msg.data.trackId); + break; + + case 'removeTransform': + teardownTransform(msg.data.trackId); + break; + + default: + break; + } +}; + +function setupDecodeTransform( + readable: ReadableStream, + writable: WritableStream, + trackId: string, +) { + teardownTransform(trackId); + + const abortController = new AbortController(); + activeTransforms.set(trackId, abortController); + + const transform = new TransformStream({ + transform( + frame: RTCEncodedVideoFrame, + controller: TransformStreamDefaultController, + ) { + const result = extractPacketTrailer(frame.data); + if (result.metadata) { + const rtpTimestamp = getFrameRtpTimestamp(frame); + const ssrc = getFrameSsrc(frame); + if (rtpTimestamp !== undefined) { + const msg: PTMetadataMessage = { + kind: 'metadata', + data: { + trackId, + rtpTimestamp, + ssrc, + metadata: result.metadata, + }, + }; + postMessage(msg); + } + frame.data = result.data.buffer.slice( + result.data.byteOffset, + result.data.byteOffset + result.data.byteLength, + ); + } + controller.enqueue(frame); + }, + }); + + readable + .pipeThrough(transform) + .pipeTo(writable, { signal: abortController.signal }) + .catch(() => { + // pipe aborted via teardown -- expected + }); +} + +function teardownTransform(trackId: string) { + const existing = activeTransforms.get(trackId); + if (existing) { + existing.abort(); + activeTransforms.delete(trackId); + } +} diff --git a/src/packetTrailer/worker/tsconfig.json b/src/packetTrailer/worker/tsconfig.json new file mode 100644 index 0000000000..5e10a3dfc2 --- /dev/null +++ b/src/packetTrailer/worker/tsconfig.json @@ -0,0 +1,14 @@ +{ + "extends": "../../../tsconfig.json", + "compilerOptions": { + "lib": [ + "DOM", + "DOM.Iterable", + "ES2017", + "ES2018.Promise", + "WebWorker", + "ES2021.WeakRef", + "DOM.AsyncIterable" + ] + } +} diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index aca0209cef..36a6c26b36 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -745,8 +745,8 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit ): RTCConfiguration { const rtcConfig = { ...this.rtcConfig }; - if (this.signalOpts?.e2eeEnabled) { - this.log.debug('E2EE - setting up transports with insertable streams', this.logContext); + if (this.signalOpts?.e2eeEnabled || this.options.packetTrailer) { + this.log.debug('setting up transports with insertable streams', this.logContext); // this makes sure that no data is sent before the transforms are ready // @ts-ignore rtcConfig.encodedInsertableStreams = true; diff --git a/src/room/Room.ts b/src/room/Room.ts index f8a36e0c58..3f71898282 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -37,6 +37,7 @@ import { ensureTrailingSlash } from '../api/utils'; import { EncryptionEvent } from '../e2ee'; import { type BaseE2EEManager, E2EEManager } from '../e2ee/E2eeManager'; import log, { LoggerNames, getLogger } from '../logger'; +import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager'; import type { InternalRoomConnectOptions, InternalRoomOptions, @@ -187,6 +188,9 @@ class Room extends (EventEmitter as new () => TypedEmitter) private e2eeManager: BaseE2EEManager | undefined; + /** @internal */ + packetTrailerManager: PacketTrailerManager | undefined; + private e2eeStateMutex: Mutex = new Mutex(); private connectionReconcileInterval?: ReturnType; @@ -305,6 +309,10 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.outgoingDataTrackManager, ); + if (this.options.packetTrailer) { + this.setupPacketTrailer(); + } + if (this.options.e2ee || this.options.encryption) { this.setupE2EE(); } @@ -463,6 +471,13 @@ class Room extends (EventEmitter as new () => TypedEmitter) } } + private setupPacketTrailer() { + if (this.options.packetTrailer) { + this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer); + this.packetTrailerManager.setup(this); + } + } + private get logContext() { return { room: this.name, @@ -2319,6 +2334,17 @@ class Room extends (EventEmitter as new () => TypedEmitter) track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed); track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted); } + if ( + !this.packetTrailerManager && + publication.trackInfo?.packetTrailerFeatures && + publication.trackInfo.packetTrailerFeatures.length > 0 + ) { + this.log.warn( + 'Track has packet trailer features but no packet trailer worker is configured. ' + + 'Pass packetTrailer: { worker } in RoomOptions to enable frame metadata extraction.', + { ...this.logContext, trackSid: publication.trackSid }, + ); + } this.emitWhenConnected(RoomEvent.TrackSubscribed, track, publication, participant); }, ) diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts new file mode 100644 index 0000000000..763f09af1a --- /dev/null +++ b/src/room/track/PacketTrailerExtractor.ts @@ -0,0 +1,124 @@ +import log from '../../logger'; +import { + extractPacketTrailer, + getFrameRtpTimestamp, + getFrameSsrc, + type PacketTrailerMetadata, +} from '../../e2ee/packetTrailer'; + +const MAX_ENTRIES = 300; +const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; + +/** + * Extracts and caches packet trailer metadata from received video frames. + * + * In the non-E2EE path, this sets up an Insertable Streams pipeline on the + * receiver to strip trailers from encoded frames on the main thread. + * + * In the E2EE path, metadata is injected externally after the worker decrypts + * and strips the trailer. + * + * Metadata is stored in an LRU map keyed by RTP timestamp so it can be + * looked up when the frame is displayed. + * + * @experimental + */ +export class PacketTrailerExtractor { + private metadataMap = new Map(); + + private insertionOrder: number[] = []; + + private activeSsrc: number = 0; + + storeMetadata(rtpTimestamp: number, ssrc: number, metadata: PacketTrailerMetadata) { + // Simulcast layer switch: SSRC changed, flush stale entries from old layer. + if (this.activeSsrc !== 0 && this.activeSsrc !== ssrc) { + const keep: number[] = []; + for (const ts of this.insertionOrder) { + const m = this.metadataMap.get(ts); + if (!m || (m as PacketTrailerMetadataInternal).ssrc !== ssrc) { + this.metadataMap.delete(ts); + } else { + keep.push(ts); + } + } + this.insertionOrder = keep; + } + this.activeSsrc = ssrc; + + const collision = this.metadataMap.has(rtpTimestamp); + + while (this.metadataMap.size >= MAX_ENTRIES && this.insertionOrder.length > 0) { + const evicted = this.insertionOrder.shift()!; + this.metadataMap.delete(evicted); + } + + if (!collision) { + this.insertionOrder.push(rtpTimestamp); + } + (metadata as PacketTrailerMetadataInternal).ssrc = ssrc; + this.metadataMap.set(rtpTimestamp, metadata); + } + + lookupMetadata(rtpTimestamp: number): PacketTrailerMetadata | undefined { + return this.metadataMap.get(rtpTimestamp); + } + + /** + * Sets up an Insertable Streams pipeline on the receiver to extract + * packet trailers from encoded video frames on the main thread. + * Only used when E2EE is NOT active. + */ + setupReceiver(receiver: RTCRtpReceiver): boolean { + if (PACKET_TRAILER_FLAG in receiver) { + return true; + } + + if (!('createEncodedStreams' in receiver)) { + log.debug('createEncodedStreams not supported, packet trailer extraction unavailable'); + return false; + } + + // @ts-ignore — createEncodedStreams is not in standard typings + const streams = receiver.createEncodedStreams(); + const extractor = this; + + const transform = new TransformStream({ + transform( + frame: RTCEncodedVideoFrame, + controller: TransformStreamDefaultController, + ) { + const result = extractPacketTrailer(frame.data); + if (result.metadata) { + const rtpTimestamp = getFrameRtpTimestamp(frame); + const ssrc = getFrameSsrc(frame); + if (rtpTimestamp !== undefined) { + extractor.storeMetadata(rtpTimestamp, ssrc, result.metadata); + } + frame.data = result.data.buffer.slice( + result.data.byteOffset, + result.data.byteOffset + result.data.byteLength, + ); + } + controller.enqueue(frame); + }, + }); + + streams.readable.pipeThrough(transform).pipeTo(streams.writable); + + // @ts-ignore + receiver[PACKET_TRAILER_FLAG] = true; + return true; + } + + dispose() { + this.metadataMap.clear(); + this.insertionOrder.length = 0; + this.activeSsrc = 0; + } +} + +/** @internal */ +interface PacketTrailerMetadataInternal extends PacketTrailerMetadata { + ssrc?: number; +} diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index b3d585db04..16c72e2e7f 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -1,3 +1,4 @@ +import type { PacketTrailerMetadata } from '../../e2ee/packetTrailer'; import { debounce } from '../debounce'; import { TrackEvent } from '../events'; import type { VideoReceiverStats } from '../stats'; @@ -6,6 +7,7 @@ import CriticalTimers from '../timers'; import type { LoggerOptions } from '../types'; import type { ObservableMediaElement } from '../utils'; import { getDevicePixelRatio, getIntersectionObserver, getResizeObserver, isWeb } from '../utils'; +import type { PacketTrailerExtractor } from './PacketTrailerExtractor'; import RemoteTrack from './RemoteTrack'; import { Track, attachToElement, detachTrack } from './Track'; import type { AdaptiveStreamSettings } from './types'; @@ -23,6 +25,9 @@ export default class RemoteVideoTrack extends RemoteTrack { private lastDimensions?: Track.Dimensions; + /** @internal */ + packetTrailerExtractor?: PacketTrailerExtractor; + constructor( mediaTrack: MediaStreamTrack, sid: string, @@ -38,6 +43,24 @@ export default class RemoteVideoTrack extends RemoteTrack { return this.adaptiveStreamSettings !== undefined; } + /** + * Look up frame-level metadata (user timestamp, frame ID) for a given RTP timestamp. + * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames + * with their capture-time metadata. + * + * Requires the room to be configured with `packetTrailer: { worker }` and the + * publishing track to have packet trailer features enabled. + * + * @experimental + */ + lookupFrameMetadata({ + rtpTimestamp, + }: { + rtpTimestamp: number; + }): PacketTrailerMetadata | undefined { + return this.packetTrailerExtractor?.lookupMetadata(rtpTimestamp); + } + override setStreamState(value: Track.StreamState) { super.setStreamState(value); this.log.debug('setStreamState', value); diff --git a/src/room/track/utils.ts b/src/room/track/utils.ts index e4974e76eb..19e99354e4 100644 --- a/src/room/track/utils.ts +++ b/src/room/track/utils.ts @@ -280,7 +280,10 @@ export function getLogContextFromTrack(track: Track | TrackPublication): Record< } export function supportsSynchronizationSources(): boolean { - return typeof RTCRtpReceiver !== 'undefined' && 'getSynchronizationSources' in RTCRtpReceiver; + return ( + typeof RTCRtpReceiver !== 'undefined' && + typeof RTCRtpReceiver.prototype.getSynchronizationSources === 'function' + ); } export function diffAttributes( diff --git a/src/version.ts b/src/version.ts index e0d96ee71b..87eaf89839 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,4 +1,4 @@ import { version as v } from '../package.json'; export const version = v; -export const protocolVersion = 16; +export const protocolVersion = 17; From 3d00ac00877f8017cb874813439460589fcb694c Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 11:45:59 -0700 Subject: [PATCH 02/31] update frame metadata rendering in demo --- examples/demo/demo.ts | 33 ++++++++++++++++++--------------- 1 file changed, 18 insertions(+), 15 deletions(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 3ec2a7938e..f0e761dda6 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -256,22 +256,25 @@ const appActions = { const meta = track.lookupFrameMetadata({ rtpTimestamp }); const overlayElm = document.getElementById(`pt-overlay-${participant.identity}`); if (overlayElm && meta) { - const now = Date.now(); - const receiveTime = new Date(now); - const publishTime = new Date(meta.userTimestampUs / 1000); - if (now - lastLatencyUpdate >= 500) { - lastLatencyUpdate = now; - latencyDisplay = `${(receiveTime.getTime() - publishTime.getTime()).toFixed(1)}ms`; + let text = `Frame ID: ${meta.frameId}`; + if (meta.userTimestampUs) { + const now = Date.now(); + const receiveTime = new Date(now); + const publishTime = new Date(meta.userTimestampUs / 1000); + if (now - lastLatencyUpdate >= 500) { + lastLatencyUpdate = now; + latencyDisplay = `${(receiveTime.getTime() - publishTime.getTime()).toFixed(1)}ms`; + } + const fmt = (d: Date) => { + const pad = (n: number, w = 2) => String(n).padStart(w, '0'); + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}:${pad(d.getMilliseconds(), 4)}`; + }; + text += + `\nPublish: ${fmt(publishTime)}` + + `\nReceive: ${fmt(receiveTime)}` + + `\nLatency: ${latencyDisplay}`; } - const fmt = (d: Date) => { - const pad = (n: number, w = 2) => String(n).padStart(w, '0'); - return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}:${pad(d.getMilliseconds(), 4)}`; - }; - overlayElm.textContent = - `Frame ID: ${meta.frameId}\n` + - `Publish: ${fmt(publishTime)}\n` + - `Receive: ${fmt(receiveTime)}\n` + - `Latency: ${latencyDisplay}`; + overlayElm.textContent = text; } }); } From 0e61562cae562e8fa1496732bdff7dc4c2f909e6 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 12:47:40 -0700 Subject: [PATCH 03/31] when e2ee is enabled, handle the packet trailer parsing within FrameCryptor for performance --- src/e2ee/E2eeManager.ts | 31 ++++++++ src/e2ee/types.ts | 16 +++- src/e2ee/worker/FrameCryptor.ts | 37 +++++++++- src/packetTrailer/PacketTrailerManager.ts | 89 ++++++++--------------- 4 files changed, 112 insertions(+), 61 deletions(-) diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index ec3af0c091..72694b5253 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -8,6 +8,7 @@ import { ConnectionState } from '../room/Room'; import { DeviceUnsupportedError } from '../room/errors'; import { EngineEvent, ParticipantEvent, RoomEvent } from '../room/events'; import type RemoteTrack from '../room/track/RemoteTrack'; +import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { Track } from '../room/track/Track'; import type { VideoCodec } from '../room/track/options'; import { mimeTypeToVideoCodecString } from '../room/track/utils'; @@ -221,6 +222,9 @@ export class E2EEManager encryptFuture.resolve(data as EncryptDataResponseMessage['data']); } break; + case 'packetTrailerMetadata': + this.handlePacketTrailerMetadata(data.trackId, data.rtpTimestamp, data.ssrc, data.metadata); + break; default: break; } @@ -231,6 +235,33 @@ export class E2EEManager this.emit(EncryptionEvent.EncryptionError, ev.error, undefined); }; + private handlePacketTrailerMetadata( + trackId: string, + rtpTimestamp: number, + ssrc: number, + metadata: { userTimestampUs: number; frameId: number }, + ) { + if (!this.room) { + return; + } + for (const participant of [ + this.room.localParticipant, + ...this.room.remoteParticipants.values(), + ]) { + for (const pub of participant.trackPublications.values()) { + if ( + pub.track && + pub.track.mediaStreamID === trackId && + pub.track instanceof RemoteVideoTrack && + pub.track.packetTrailerExtractor + ) { + pub.track.packetTrailerExtractor.storeMetadata(rtpTimestamp, ssrc, metadata); + return; + } + } + } + } + public setupEngine(engine: RTCEngine) { engine.on(EngineEvent.RTPVideoMapUpdate, (rtpMap) => { this.postRTPMap(rtpMap); diff --git a/src/e2ee/types.ts b/src/e2ee/types.ts index d7caa48e3e..f2849d11ea 100644 --- a/src/e2ee/types.ts +++ b/src/e2ee/types.ts @@ -150,6 +150,19 @@ export interface EncryptDataResponseMessage extends BaseMessage { }; } +export interface PTMetadataFromE2EEMessage extends BaseMessage { + kind: 'packetTrailerMetadata'; + data: { + trackId: string; + rtpTimestamp: number; + ssrc: number; + metadata: { + userTimestampUs: number; + frameId: number; + }; + }; +} + export type E2EEWorkerMessage = | InitMessage | SetKeyMessage @@ -166,7 +179,8 @@ export type E2EEWorkerMessage = | DecryptDataRequestMessage | DecryptDataResponseMessage | EncryptDataRequestMessage - | EncryptDataResponseMessage; + | EncryptDataResponseMessage + | PTMetadataFromE2EEMessage; export type KeySet = { material: CryptoKey; encryptionKey: CryptoKey }; diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 1958f2bec1..6de6ed73f8 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -7,7 +7,14 @@ import type { VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { type CryptorCallbacks, CryptorEvent } from '../events'; -import type { DecodeRatchetOptions, KeyProviderOptions, KeySet, RatchetResult } from '../types'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../packetTrailer'; +import type { + DecodeRatchetOptions, + KeyProviderOptions, + KeySet, + PTMetadataFromE2EEMessage, + RatchetResult, +} from '../types'; import { deriveKeys, isVideoFrame, needsRbspUnescaping, parseRbsp, writeRbsp } from '../utils'; import type { ParticipantKeyHandler } from './ParticipantKeyHandler'; import { processNALUsForEncryption } from './naluUtils'; @@ -454,6 +461,34 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, controller: TransformStreamDefaultController, ) { + if (isVideoFrame(encodedFrame) && encodedFrame.data.byteLength > 0) { + try { + const ptResult = extractPacketTrailer(encodedFrame.data); + if (ptResult.metadata) { + const rtpTimestamp = getFrameRtpTimestamp(encodedFrame); + const ssrc = getFrameSsrc(encodedFrame); + if (rtpTimestamp !== undefined && this.trackId && this.participantIdentity) { + const msg: PTMetadataFromE2EEMessage = { + kind: 'packetTrailerMetadata', + data: { + trackId: this.trackId, + rtpTimestamp, + ssrc, + metadata: ptResult.metadata, + }, + }; + postMessage(msg); + } + encodedFrame.data = (ptResult.data.buffer as ArrayBuffer).slice( + ptResult.data.byteOffset, + ptResult.data.byteOffset + ptResult.data.byteLength, + ); + } + } catch { + // best-effort: never break the media pipeline if trailer parsing fails + } + } + if ( !this.isEnabled() || // skip for decryption for empty dtx frames diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index dfe6140051..216d1d10ba 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -20,8 +20,10 @@ export interface PacketTrailerOptions { * wires up an Insertable Streams pipeline (via a dedicated worker) to strip * the trailer from encoded frames and cache the metadata for lookup. * - * Works independently of E2EE. When both are active, the PT pipeline runs - * first and provides pre-processed streams that E2EE can consume. + * When E2EE is active, the E2EE FrameCryptor worker handles trailer + * extraction directly (before decryption), so this manager only creates + * the extractor/metadata cache — no separate insertable-streams pipeline + * is needed. * * @experimental */ @@ -32,11 +34,6 @@ export class PacketTrailerManager { private extractors = new Map(); - private preProcessedStreams = new Map< - RTCRtpReceiver, - { readable: ReadableStream; writable: WritableStream } - >(); - constructor(options: PacketTrailerOptions) { this.worker = options.worker; } @@ -84,6 +81,16 @@ export class PacketTrailerManager { this.extractors.set(trackId, extractor); track.packetTrailerExtractor = extractor; + const hasE2EE = !!this.room?.hasE2EESetup; + + if (hasE2EE) { + // When E2EE is active, the FrameCryptor worker strips the trailer + // inside its decodeFunction before decryption and sends metadata + // back via the E2EEManager. No separate pipeline is needed here; + // we only create the extractor/cache above. + return; + } + const receiver = track.receiver; if (!('createEncodedStreams' in receiver)) { @@ -95,40 +102,16 @@ export class PacketTrailerManager { // @ts-ignore -- createEncodedStreams is not in standard typings const streams = receiver.createEncodedStreams(); - const hasE2EE = !!this.room?.hasE2EESetup; - - if (hasE2EE) { - const bridge = new TransformStream(); - const msg: PTDecodeMessage = { - kind: 'decode', - data: { - readableStream: streams.readable, - writableStream: bridge.writable, - trackId, - }, - }; - this.worker.postMessage(msg, [streams.readable, bridge.writable]); - // @ts-ignore - receiver.readableStream = bridge.readable; - // @ts-ignore - receiver.writableStream = streams.writable; - - this.preProcessedStreams.set(receiver, { - readable: bridge.readable, - writable: streams.writable, - }); - } else { - const msg: PTDecodeMessage = { - kind: 'decode', - data: { - readableStream: streams.readable, - writableStream: streams.writable, - trackId, - }, - }; - this.worker.postMessage(msg, [streams.readable, streams.writable]); - } + const msg: PTDecodeMessage = { + kind: 'decode', + data: { + readableStream: streams.readable, + writableStream: streams.writable, + trackId, + }, + }; + this.worker.postMessage(msg, [streams.readable, streams.writable]); // @ts-ignore receiver[PACKET_TRAILER_FLAG] = true; @@ -141,15 +124,14 @@ export class PacketTrailerManager { extractor.dispose(); this.extractors.delete(trackId); } - if (track.receiver) { - this.preProcessedStreams.delete(track.receiver); - } - const msg: PTRemoveTransformMessage = { - kind: 'removeTransform', - data: { trackId }, - }; - this.worker.postMessage(msg); + if (!this.room?.hasE2EESetup) { + const msg: PTRemoveTransformMessage = { + kind: 'removeTransform', + data: { trackId }, + }; + this.worker.postMessage(msg); + } } private onWorkerMessage = (ev: MessageEvent) => { @@ -165,15 +147,4 @@ export class PacketTrailerManager { private onWorkerError = (ev: ErrorEvent) => { log.error('packet trailer worker encountered an error:', { error: ev.error }); }; - - /** - * Returns pre-processed streams for a receiver if the PT pipeline has already - * consumed the receiver's encoded streams. Used by E2EE to chain after PT. - * @internal - */ - getPreProcessedStreams( - receiver: RTCRtpReceiver, - ): { readable: ReadableStream; writable: WritableStream } | undefined { - return this.preProcessedStreams.get(receiver); - } } From f11025ae784015cf03df558464690b3558380483 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 13:06:19 -0700 Subject: [PATCH 04/31] remove orphaned file --- src/ConnectionSession.ts | 36 ------------------------------------ 1 file changed, 36 deletions(-) delete mode 100644 src/ConnectionSession.ts diff --git a/src/ConnectionSession.ts b/src/ConnectionSession.ts deleted file mode 100644 index 0c33193b45..0000000000 --- a/src/ConnectionSession.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { type JoinRequest } from '@livekit/protocol'; -import { - SignalClient, - type JoinedRoomMembership, - type SignalOptions, -} from './api/SignalClient'; -import type { LoggerOptions } from './room/types'; - -export class ConnectionSession { - readonly signalClient: SignalClient; - - constructor(useJSON: boolean = false, loggerOptions: LoggerOptions = {}) { - this.signalClient = new SignalClient(useJSON, loggerOptions); - } - - async connect( - url: string, - token: string, - options: SignalOptions, - abortSignal?: AbortSignal, - ): Promise { - const joinResponse = await this.signalClient.join(url, token, options, abortSignal); - return { - joinResponse, - membershipId: this.signalClient.defaultMembershipId ?? '', - }; - } - - async joinRoom(token: string, joinRequest?: JoinRequest): Promise { - return this.signalClient.joinRoom(token, joinRequest); - } - - async close(reason?: string) { - await this.signalClient.close(true, reason); - } -} From e00d1dd01f3876248067630f8a17726fac7c005d Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 13:09:16 -0700 Subject: [PATCH 05/31] remove orphaned file --- src/e2ee/createBuiltinE2EEWorker.ts | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 src/e2ee/createBuiltinE2EEWorker.ts diff --git a/src/e2ee/createBuiltinE2EEWorker.ts b/src/e2ee/createBuiltinE2EEWorker.ts deleted file mode 100644 index a071656391..0000000000 --- a/src/e2ee/createBuiltinE2EEWorker.ts +++ /dev/null @@ -1,27 +0,0 @@ -import log from '../logger'; - -export function createBuiltinE2EEWorker() { - if (typeof Worker === 'undefined') { - return undefined; - } - - try { - return new Worker(new URL('./livekit-client.e2ee.worker.mjs', import.meta.url), { - type: 'module', - name: 'livekit-client-e2ee', - }); - } catch (moduleError) { - log.debug('failed to initialize module e2ee worker, falling back to classic worker', { - error: moduleError, - }); - } - - try { - return new Worker(new URL('./livekit-client.e2ee.worker.js', import.meta.url), { - name: 'livekit-client-e2ee', - }); - } catch (workerError) { - log.warn('failed to initialize built-in e2ee worker', { error: workerError }); - return undefined; - } -} From 09e2b1f96feb91bdce213a04ed02fdf1980e0129 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:11:12 -0700 Subject: [PATCH 06/31] cleanup for consistency --- src/e2ee/packetTrailer.test.ts | 2 +- src/room/Room.ts | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/e2ee/packetTrailer.test.ts b/src/e2ee/packetTrailer.test.ts index ba1b5dc9ae..7a26696d88 100644 --- a/src/e2ee/packetTrailer.test.ts +++ b/src/e2ee/packetTrailer.test.ts @@ -2,7 +2,7 @@ import { describe, expect, it } from 'vitest'; import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; describe('packetTrailer', () => { - it('extracts user timestamp and frame id using the Rust wire format', () => { + it('extracts user timestamp and frame id from packet trailer', () => { const payload = Uint8Array.from([1, 2, 3, 4]); const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456, 42); const extracted = extractPacketTrailer(trailer); diff --git a/src/room/Room.ts b/src/room/Room.ts index 3f71898282..05e25d568e 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -188,8 +188,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private e2eeManager: BaseE2EEManager | undefined; - /** @internal */ - packetTrailerManager: PacketTrailerManager | undefined; + private packetTrailerManager: PacketTrailerManager | undefined; private e2eeStateMutex: Mutex = new Mutex(); From c9250dd7aa772c5d9164faa401df05a296dec68f Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:15:47 -0700 Subject: [PATCH 07/31] dont display frame id if not set --- examples/demo/demo.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index f0e761dda6..15ba2670b1 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -256,7 +256,7 @@ const appActions = { const meta = track.lookupFrameMetadata({ rtpTimestamp }); const overlayElm = document.getElementById(`pt-overlay-${participant.identity}`); if (overlayElm && meta) { - let text = `Frame ID: ${meta.frameId}`; + let text = meta.frameId ? `Frame ID: ${meta.frameId}` : ''; if (meta.userTimestampUs) { const now = Date.now(); const receiveTime = new Date(now); From 5d9d1bccd48499cbed7138c86ed8e76aebce6859 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:24:18 -0700 Subject: [PATCH 08/31] clean up --- examples/demo/demo.ts | 4 ++-- src/e2ee/E2eeManager.ts | 2 +- src/e2ee/packetTrailer.test.ts | 33 ++++++++++++++++++++++++++++++++- src/e2ee/packetTrailer.ts | 28 +++++++++++++++++++--------- src/e2ee/types.ts | 2 +- 5 files changed, 55 insertions(+), 14 deletions(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 15ba2670b1..74bc7ee38c 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -257,10 +257,10 @@ const appActions = { const overlayElm = document.getElementById(`pt-overlay-${participant.identity}`); if (overlayElm && meta) { let text = meta.frameId ? `Frame ID: ${meta.frameId}` : ''; - if (meta.userTimestampUs) { + if (meta.userTimestamp) { const now = Date.now(); const receiveTime = new Date(now); - const publishTime = new Date(meta.userTimestampUs / 1000); + const publishTime = new Date(meta.userTimestamp / 1000); if (now - lastLatencyUpdate >= 500) { lastLatencyUpdate = now; latencyDisplay = `${(receiveTime.getTime() - publishTime.getTime()).toFixed(1)}ms`; diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index 72694b5253..c09bdc26e4 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -239,7 +239,7 @@ export class E2EEManager trackId: string, rtpTimestamp: number, ssrc: number, - metadata: { userTimestampUs: number; frameId: number }, + metadata: { userTimestamp: number; frameId: number }, ) { if (!this.room) { return; diff --git a/src/e2ee/packetTrailer.test.ts b/src/e2ee/packetTrailer.test.ts index 7a26696d88..8683ebbbda 100644 --- a/src/e2ee/packetTrailer.test.ts +++ b/src/e2ee/packetTrailer.test.ts @@ -9,11 +9,42 @@ describe('packetTrailer', () => { expect(Array.from(extracted.data)).toEqual(Array.from(payload)); expect(extracted.metadata).toEqual({ - userTimestampUs: 1_744_249_600_123_456, + userTimestamp: 1_744_249_600_123_456, frameId: 42, }); }); + it('extracts timestamp-only trailer when frameId is 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456, 0); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestamp: 1_744_249_600_123_456, + frameId: 0, + }); + }); + + it('extracts frameId-only trailer when timestamp is 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 0, 42); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestamp: 0, + frameId: 42, + }); + }); + + it('returns data unchanged when both timestamp and frameId are 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const result = appendPacketTrailer(payload, 0, 0); + + expect(Array.from(result)).toEqual(Array.from(payload)); + }); + it('passes frames through when there is no valid trailer', () => { const payload = Uint8Array.from([1, 2, 3, 4, 5]); const extracted = extractPacketTrailer(payload); diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts index 5d6f863a68..4c788b71d5 100644 --- a/src/e2ee/packetTrailer.ts +++ b/src/e2ee/packetTrailer.ts @@ -13,7 +13,7 @@ const TIMESTAMP_TLV_SIZE = 10; const FRAME_ID_TLV_SIZE = 6; export interface PacketTrailerMetadata { - userTimestampUs: number; + userTimestamp: number; frameId: number; } @@ -24,22 +24,32 @@ export interface ExtractPacketTrailerResult { export function appendPacketTrailer( data: Uint8Array, - userTimestampUs: number, + userTimestamp: number, frameId: number, ): Uint8Array { + const hasTimestamp = userTimestamp !== 0; const hasFrameId = frameId !== 0; + + if (!hasTimestamp && !hasFrameId) { + return data; + } + const trailerLength = - TIMESTAMP_TLV_SIZE + (hasFrameId ? FRAME_ID_TLV_SIZE : 0) + PACKET_TRAILER_ENVELOPE_SIZE; + (hasTimestamp ? TIMESTAMP_TLV_SIZE : 0) + + (hasFrameId ? FRAME_ID_TLV_SIZE : 0) + + PACKET_TRAILER_ENVELOPE_SIZE; const result = new Uint8Array(data.length + trailerLength); let offset = 0; result.set(data, offset); offset += data.length; - result[offset++] = PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff; - result[offset++] = 8 ^ 0xff; - writeUint64Xor(result, offset, userTimestampUs); - offset += 8; + if (hasTimestamp) { + result[offset++] = PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff; + result[offset++] = 8 ^ 0xff; + writeUint64Xor(result, offset, userTimestamp); + offset += 8; + } if (hasFrameId) { result[offset++] = PACKET_TRAILER_FRAME_ID_TAG ^ 0xff; @@ -76,7 +86,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac let offset = trailerStart; let foundAny = false; const metadata: PacketTrailerMetadata = { - userTimestampUs: 0, + userTimestamp: 0, frameId: 0, }; @@ -89,7 +99,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac } if (tag === PACKET_TRAILER_TIMESTAMP_TAG && length === 8) { - metadata.userTimestampUs = readUint64Xor(bytes, offset, length); + metadata.userTimestamp = readUint64Xor(bytes, offset, length); foundAny = true; } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { metadata.frameId = readUint32Xor(bytes, offset, length); diff --git a/src/e2ee/types.ts b/src/e2ee/types.ts index f2849d11ea..ca717ceca9 100644 --- a/src/e2ee/types.ts +++ b/src/e2ee/types.ts @@ -157,7 +157,7 @@ export interface PTMetadataFromE2EEMessage extends BaseMessage { rtpTimestamp: number; ssrc: number; metadata: { - userTimestampUs: number; + userTimestamp: number; frameId: number; }; }; From df3dc78055c3adcacdd0e3e29b7f3d1c3f5ff4cb Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:33:42 -0700 Subject: [PATCH 09/31] improve parser --- src/e2ee/packetTrailer.ts | 45 +++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 14 deletions(-) diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts index 4c788b71d5..c434e6a2c6 100644 --- a/src/e2ee/packetTrailer.ts +++ b/src/e2ee/packetTrailer.ts @@ -99,7 +99,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac } if (tag === PACKET_TRAILER_TIMESTAMP_TAG && length === 8) { - metadata.userTimestamp = readUint64Xor(bytes, offset, length); + metadata.userTimestamp = readUint64Xor(bytes, offset); foundAny = true; } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { metadata.frameId = readUint32Xor(bytes, offset, length); @@ -125,12 +125,23 @@ function matchesMagic(data: Uint8Array, offset: number) { return true; } -function readUint64Xor(data: Uint8Array, offset: number, length: number) { - let value = 0; - for (let index = 0; index < length; index += 1) { - value = value * 256 + (data[offset + index] ^ 0xff); - } - return value; +// Reads a big-endian XOR-masked 64-bit value by combining two 32-bit halves. +// Uses bitwise shifts for each half, then a single multiply to join them. +// Safe for values up to Number.MAX_SAFE_INTEGER (2^53 - 1). +function readUint64Xor(data: Uint8Array, offset: number) { + const hi = ( + ((data[offset] ^ 0xff) << 24) | + ((data[offset + 1] ^ 0xff) << 16) | + ((data[offset + 2] ^ 0xff) << 8) | + (data[offset + 3] ^ 0xff) + ) >>> 0; + const lo = ( + ((data[offset + 4] ^ 0xff) << 24) | + ((data[offset + 5] ^ 0xff) << 16) | + ((data[offset + 6] ^ 0xff) << 8) | + (data[offset + 7] ^ 0xff) + ) >>> 0; + return hi * 0x100000000 + lo; } function readUint32Xor(data: Uint8Array, offset: number, length: number) { @@ -141,14 +152,20 @@ function readUint32Xor(data: Uint8Array, offset: number, length: number) { return value >>> 0; } +// Writes a 64-bit value as 8 big-endian XOR-masked bytes. +// Splits into high/low 32-bit halves: one integer division to get the +// upper 32 bits, then pure bitwise shifts for individual bytes. function writeUint64Xor(target: Uint8Array, offset: number, value: number) { - let remaining = value; - for (let index = 7; index >= 0; index -= 1) { - const shift = 256 ** index; - const currentByte = Math.floor(remaining / shift); - target[offset + (7 - index)] = currentByte ^ 0xff; - remaining -= currentByte * shift; - } + const hi = (value / 0x100000000) >>> 0; + const lo = value >>> 0; + target[offset] = (hi >>> 24) ^ 0xff; + target[offset + 1] = ((hi >>> 16) & 0xff) ^ 0xff; + target[offset + 2] = ((hi >>> 8) & 0xff) ^ 0xff; + target[offset + 3] = (hi & 0xff) ^ 0xff; + target[offset + 4] = (lo >>> 24) ^ 0xff; + target[offset + 5] = ((lo >>> 16) & 0xff) ^ 0xff; + target[offset + 6] = ((lo >>> 8) & 0xff) ^ 0xff; + target[offset + 7] = (lo & 0xff) ^ 0xff; } function writeUint32Xor(target: Uint8Array, offset: number, value: number) { From de281af735db49964ba831d936bb8fe806128822 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:41:19 -0700 Subject: [PATCH 10/31] clean up --- src/room/track/RemoteVideoTrack.ts | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 16c72e2e7f..81bed9e36d 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -25,8 +25,7 @@ export default class RemoteVideoTrack extends RemoteTrack { private lastDimensions?: Track.Dimensions; - /** @internal */ - packetTrailerExtractor?: PacketTrailerExtractor; + private packetTrailerExtractor?: PacketTrailerExtractor; constructor( mediaTrack: MediaStreamTrack, @@ -44,14 +43,13 @@ export default class RemoteVideoTrack extends RemoteTrack { } /** - * Look up frame-level metadata (user timestamp, frame ID) for a given RTP timestamp. + * Look up frame-level metadata for a given RTP timestamp. * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames * with their capture-time metadata. * * Requires the room to be configured with `packetTrailer: { worker }` and the * publishing track to have packet trailer features enabled. * - * @experimental */ lookupFrameMetadata({ rtpTimestamp, From a47dee73341c7e719041b74ffe372126b6dba58e Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:47:16 -0700 Subject: [PATCH 11/31] ensure user_timestamp is bigint --- examples/demo/demo.ts | 2 +- src/e2ee/E2eeManager.ts | 2 +- src/e2ee/packetTrailer.test.ts | 14 +++++----- src/e2ee/packetTrailer.ts | 50 ++++++++++++++++------------------ src/e2ee/types.ts | 2 +- 5 files changed, 33 insertions(+), 37 deletions(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 74bc7ee38c..2654930a03 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -260,7 +260,7 @@ const appActions = { if (meta.userTimestamp) { const now = Date.now(); const receiveTime = new Date(now); - const publishTime = new Date(meta.userTimestamp / 1000); + const publishTime = new Date(Number(meta.userTimestamp / 1000n)); if (now - lastLatencyUpdate >= 500) { lastLatencyUpdate = now; latencyDisplay = `${(receiveTime.getTime() - publishTime.getTime()).toFixed(1)}ms`; diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index c09bdc26e4..a1f026492e 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -239,7 +239,7 @@ export class E2EEManager trackId: string, rtpTimestamp: number, ssrc: number, - metadata: { userTimestamp: number; frameId: number }, + metadata: { userTimestamp: bigint; frameId: number }, ) { if (!this.room) { return; diff --git a/src/e2ee/packetTrailer.test.ts b/src/e2ee/packetTrailer.test.ts index 8683ebbbda..29ee031a98 100644 --- a/src/e2ee/packetTrailer.test.ts +++ b/src/e2ee/packetTrailer.test.ts @@ -4,43 +4,43 @@ import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; describe('packetTrailer', () => { it('extracts user timestamp and frame id from packet trailer', () => { const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456, 42); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42); const extracted = extractPacketTrailer(trailer); expect(Array.from(extracted.data)).toEqual(Array.from(payload)); expect(extracted.metadata).toEqual({ - userTimestamp: 1_744_249_600_123_456, + userTimestamp: 1_744_249_600_123_456n, frameId: 42, }); }); it('extracts timestamp-only trailer when frameId is 0', () => { const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456, 0); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 0); const extracted = extractPacketTrailer(trailer); expect(Array.from(extracted.data)).toEqual(Array.from(payload)); expect(extracted.metadata).toEqual({ - userTimestamp: 1_744_249_600_123_456, + userTimestamp: 1_744_249_600_123_456n, frameId: 0, }); }); it('extracts frameId-only trailer when timestamp is 0', () => { const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 0, 42); + const trailer = appendPacketTrailer(payload, 0n, 42); const extracted = extractPacketTrailer(trailer); expect(Array.from(extracted.data)).toEqual(Array.from(payload)); expect(extracted.metadata).toEqual({ - userTimestamp: 0, + userTimestamp: 0n, frameId: 42, }); }); it('returns data unchanged when both timestamp and frameId are 0', () => { const payload = Uint8Array.from([1, 2, 3, 4]); - const result = appendPacketTrailer(payload, 0, 0); + const result = appendPacketTrailer(payload, 0n, 0); expect(Array.from(result)).toEqual(Array.from(payload)); }); diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts index c434e6a2c6..35a2d450ef 100644 --- a/src/e2ee/packetTrailer.ts +++ b/src/e2ee/packetTrailer.ts @@ -13,7 +13,7 @@ const TIMESTAMP_TLV_SIZE = 10; const FRAME_ID_TLV_SIZE = 6; export interface PacketTrailerMetadata { - userTimestamp: number; + userTimestamp: bigint; frameId: number; } @@ -24,10 +24,10 @@ export interface ExtractPacketTrailerResult { export function appendPacketTrailer( data: Uint8Array, - userTimestamp: number, + userTimestamp: bigint, frameId: number, ): Uint8Array { - const hasTimestamp = userTimestamp !== 0; + const hasTimestamp = userTimestamp !== 0n; const hasFrameId = frameId !== 0; if (!hasTimestamp && !hasFrameId) { @@ -86,7 +86,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac let offset = trailerStart; let foundAny = false; const metadata: PacketTrailerMetadata = { - userTimestamp: 0, + userTimestamp: 0n, frameId: 0, }; @@ -125,23 +125,22 @@ function matchesMagic(data: Uint8Array, offset: number) { return true; } -// Reads a big-endian XOR-masked 64-bit value by combining two 32-bit halves. -// Uses bitwise shifts for each half, then a single multiply to join them. -// Safe for values up to Number.MAX_SAFE_INTEGER (2^53 - 1). -function readUint64Xor(data: Uint8Array, offset: number) { - const hi = ( - ((data[offset] ^ 0xff) << 24) | - ((data[offset + 1] ^ 0xff) << 16) | - ((data[offset + 2] ^ 0xff) << 8) | - (data[offset + 3] ^ 0xff) - ) >>> 0; - const lo = ( - ((data[offset + 4] ^ 0xff) << 24) | - ((data[offset + 5] ^ 0xff) << 16) | - ((data[offset + 6] ^ 0xff) << 8) | - (data[offset + 7] ^ 0xff) - ) >>> 0; - return hi * 0x100000000 + lo; +function readUint64Xor(data: Uint8Array, offset: number): bigint { + const hi = BigInt( + (((data[offset] ^ 0xff) << 24) | + ((data[offset + 1] ^ 0xff) << 16) | + ((data[offset + 2] ^ 0xff) << 8) | + (data[offset + 3] ^ 0xff)) >>> + 0, + ); + const lo = BigInt( + (((data[offset + 4] ^ 0xff) << 24) | + ((data[offset + 5] ^ 0xff) << 16) | + ((data[offset + 6] ^ 0xff) << 8) | + (data[offset + 7] ^ 0xff)) >>> + 0, + ); + return (hi << 32n) | lo; } function readUint32Xor(data: Uint8Array, offset: number, length: number) { @@ -152,12 +151,9 @@ function readUint32Xor(data: Uint8Array, offset: number, length: number) { return value >>> 0; } -// Writes a 64-bit value as 8 big-endian XOR-masked bytes. -// Splits into high/low 32-bit halves: one integer division to get the -// upper 32 bits, then pure bitwise shifts for individual bytes. -function writeUint64Xor(target: Uint8Array, offset: number, value: number) { - const hi = (value / 0x100000000) >>> 0; - const lo = value >>> 0; +function writeUint64Xor(target: Uint8Array, offset: number, value: bigint) { + const hi = Number((value >> 32n) & 0xffffffffn); + const lo = Number(value & 0xffffffffn); target[offset] = (hi >>> 24) ^ 0xff; target[offset + 1] = ((hi >>> 16) & 0xff) ^ 0xff; target[offset + 2] = ((hi >>> 8) & 0xff) ^ 0xff; diff --git a/src/e2ee/types.ts b/src/e2ee/types.ts index ca717ceca9..b36727ff3c 100644 --- a/src/e2ee/types.ts +++ b/src/e2ee/types.ts @@ -157,7 +157,7 @@ export interface PTMetadataFromE2EEMessage extends BaseMessage { rtpTimestamp: number; ssrc: number; metadata: { - userTimestamp: number; + userTimestamp: bigint; frameId: number; }; }; From 2028be149805e32dfd2c20a379ec0370b6d90445 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 16:58:03 -0700 Subject: [PATCH 12/31] fix bigint literals --- src/e2ee/packetTrailer.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts index 35a2d450ef..04bafa90c8 100644 --- a/src/e2ee/packetTrailer.ts +++ b/src/e2ee/packetTrailer.ts @@ -27,7 +27,7 @@ export function appendPacketTrailer( userTimestamp: bigint, frameId: number, ): Uint8Array { - const hasTimestamp = userTimestamp !== 0n; + const hasTimestamp = userTimestamp !== BigInt(0); const hasFrameId = frameId !== 0; if (!hasTimestamp && !hasFrameId) { @@ -86,7 +86,7 @@ export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPac let offset = trailerStart; let foundAny = false; const metadata: PacketTrailerMetadata = { - userTimestamp: 0n, + userTimestamp: BigInt(0), frameId: 0, }; @@ -140,7 +140,7 @@ function readUint64Xor(data: Uint8Array, offset: number): bigint { (data[offset + 7] ^ 0xff)) >>> 0, ); - return (hi << 32n) | lo; + return (hi << BigInt(32)) | lo; } function readUint32Xor(data: Uint8Array, offset: number, length: number) { @@ -152,8 +152,8 @@ function readUint32Xor(data: Uint8Array, offset: number, length: number) { } function writeUint64Xor(target: Uint8Array, offset: number, value: bigint) { - const hi = Number((value >> 32n) & 0xffffffffn); - const lo = Number(value & 0xffffffffn); + const hi = Number((value >> BigInt(32)) & BigInt(0xffffffff)); + const lo = Number(value & BigInt(0xffffffff)); target[offset] = (hi >>> 24) ^ 0xff; target[offset + 1] = ((hi >>> 16) & 0xff) ^ 0xff; target[offset + 2] = ((hi >>> 8) & 0xff) ^ 0xff; From cbda15ee820954995e68bf666f171f1b33eabf18 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 17:00:59 -0700 Subject: [PATCH 13/31] lint --- src/api/WebSocketStream.test.ts | 1 - src/e2ee/packetTrailer.ts | 4 +--- src/e2ee/worker/FrameCryptor.ts | 1 - src/index.ts | 5 ++++- src/packetTrailer/PacketTrailerManager.ts | 6 ++---- src/packetTrailer/worker/packetTrailer.worker.ts | 12 ++---------- src/room/BackOffStrategy.test.ts | 2 +- src/room/Room.ts | 2 +- src/room/data-track/depacketizer.test.ts | 1 - src/room/data-track/handle.test.ts | 1 - .../incoming/IncomingDataTrackManager.test.ts | 1 - .../outgoing/OutgoingDataTrackManager.test.ts | 1 - src/room/data-track/packet/index.test.ts | 1 - src/room/data-track/packetizer.test.ts | 1 - src/room/data-track/utils.test.ts | 1 - src/room/token-source/types.ts | 2 +- src/room/track/PacketTrailerExtractor.ts | 4 ++-- src/test/MockMediaStreamTrack.ts | 1 - 18 files changed, 14 insertions(+), 33 deletions(-) diff --git a/src/api/WebSocketStream.test.ts b/src/api/WebSocketStream.test.ts index 3445348042..08de033427 100644 --- a/src/api/WebSocketStream.test.ts +++ b/src/api/WebSocketStream.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { WebSocketStream } from './WebSocketStream'; diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts index 04bafa90c8..fce18a0196 100644 --- a/src/e2ee/packetTrailer.ts +++ b/src/e2ee/packetTrailer.ts @@ -187,9 +187,7 @@ export function getFrameRtpTimestamp( return undefined; } -export function getFrameSsrc( - frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, -): number { +export function getFrameSsrc(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): number { try { const metadata = frame.getMetadata() as Record; if (typeof metadata.synchronizationSource === 'number') { diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 6de6ed73f8..ce1664bf4e 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ // TODO code inspired by https://github.com/webrtc/samples/blob/gh-pages/src/content/insertable-streams/endtoend-encryption/js/worker.js import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; diff --git a/src/index.ts b/src/index.ts index db7b484692..7ffa39fa4e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -64,7 +64,10 @@ import { getBrowser } from './utils/browserParser'; export { RpcError, type RpcInvocationData, type PerformRpcParams } from './room/rpc'; export type { PacketTrailerMetadata } from './e2ee/packetTrailer'; -export { PacketTrailerManager, type PacketTrailerOptions } from './packetTrailer/PacketTrailerManager'; +export { + PacketTrailerManager, + type PacketTrailerOptions, +} from './packetTrailer/PacketTrailerManager'; export * from './connectionHelper/ConnectionCheck'; export * from './connectionHelper/checks/Checker'; diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 216d1d10ba..22115972b0 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -2,9 +2,9 @@ import type { TrackInfo } from '@livekit/protocol'; import log from '../logger'; import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; +import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; import type RemoteVideoTrack from '../room/track/RemoteVideoTrack'; -import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type { PTDecodeMessage, PTRemoveTransformMessage, PTWorkerMessage } from './types'; const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; @@ -94,9 +94,7 @@ export class PacketTrailerManager { const receiver = track.receiver; if (!('createEncodedStreams' in receiver)) { - log.warn( - 'createEncodedStreams not supported, packet trailer extraction unavailable', - ); + log.warn('createEncodedStreams not supported, packet trailer extraction unavailable'); return; } diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 810d4c7ab4..448eff4e90 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -1,8 +1,4 @@ -import { - extractPacketTrailer, - getFrameRtpTimestamp, - getFrameSsrc, -} from '../../e2ee/packetTrailer'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../../e2ee/packetTrailer'; import type { PTMetadataMessage, PTWorkerMessage } from '../types'; const activeTransforms = new Map(); @@ -28,11 +24,7 @@ onmessage = (ev: MessageEvent) => { } }; -function setupDecodeTransform( - readable: ReadableStream, - writable: WritableStream, - trackId: string, -) { +function setupDecodeTransform(readable: ReadableStream, writable: WritableStream, trackId: string) { teardownTransform(trackId); const abortController = new AbortController(); diff --git a/src/room/BackOffStrategy.test.ts b/src/room/BackOffStrategy.test.ts index 270face22d..9bd9fab454 100644 --- a/src/room/BackOffStrategy.test.ts +++ b/src/room/BackOffStrategy.test.ts @@ -6,7 +6,7 @@ vi.mock('./utils', async () => { const actual = await vi.importActual('./utils'); return { ...actual, - // eslint-disable-next-line @typescript-eslint/no-unused-vars + sleep: vi.fn((ms: number) => Promise.resolve()), extractProjectFromUrl: vi.fn((url: URL) => { // @ts-ignore diff --git a/src/room/Room.ts b/src/room/Room.ts index 05e25d568e..99dc66b9c1 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -37,13 +37,13 @@ import { ensureTrailingSlash } from '../api/utils'; import { EncryptionEvent } from '../e2ee'; import { type BaseE2EEManager, E2EEManager } from '../e2ee/E2eeManager'; import log, { LoggerNames, getLogger } from '../logger'; -import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager'; import type { InternalRoomConnectOptions, InternalRoomOptions, RoomConnectOptions, RoomOptions, } from '../options'; +import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; import { BackOffStrategy } from './BackOffStrategy'; diff --git a/src/room/data-track/depacketizer.test.ts b/src/room/data-track/depacketizer.test.ts index 9a35ec9069..7d1de0b5fd 100644 --- a/src/room/data-track/depacketizer.test.ts +++ b/src/room/data-track/depacketizer.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import DataTrackDepacketizer from './depacketizer'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/handle.test.ts b/src/room/data-track/handle.test.ts index c486c4a8c0..34e46408ee 100644 --- a/src/room/data-track/handle.test.ts +++ b/src/room/data-track/handle.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts index 9d69c17614..045a4d4c68 100644 --- a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts +++ b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { subscribeToEvents } from '../../../utils/subscribeToEvents'; import { type DataTrackFrame } from '../frame'; diff --git a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts index 4ef1bb8b08..a2e8378607 100644 --- a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts +++ b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type DecryptDataResponseMessage, diff --git a/src/room/data-track/packet/index.test.ts b/src/room/data-track/packet/index.test.ts index f01610a8cd..8bc19b56ef 100644 --- a/src/room/data-track/packet/index.test.ts +++ b/src/room/data-track/packet/index.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackPacket, DataTrackPacketHeader, FrameMarker } from '.'; import { DataTrackHandle } from '../handle'; diff --git a/src/room/data-track/packetizer.test.ts b/src/room/data-track/packetizer.test.ts index 452f765bc2..7e6e333a32 100644 --- a/src/room/data-track/packetizer.test.ts +++ b/src/room/data-track/packetizer.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackFrameInternal } from './frame'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/utils.test.ts b/src/room/data-track/utils.test.ts index dedf0e41bd..ce951a4cc8 100644 --- a/src/room/data-track/utils.test.ts +++ b/src/room/data-track/utils.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { U16_MAX_SIZE, WrapAroundUnsignedInt } from './utils'; diff --git a/src/room/token-source/types.ts b/src/room/token-source/types.ts index b60e864af0..7ca2bacd26 100644 --- a/src/room/token-source/types.ts +++ b/src/room/token-source/types.ts @@ -3,7 +3,7 @@ import type { JWTPayload } from 'jose'; import type { ValueToSnakeCase } from '../../utils/camelToSnakeCase'; // The below imports are being linked in tsdoc comments, so they have to be imported even if they // aren't being used. -// eslint-disable-next-line @typescript-eslint/no-unused-vars + import type { TokenSourceCustom, TokenSourceEndpoint, TokenSourceLiteral } from './TokenSource'; export type TokenSourceRequestObject = Required< diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index 763f09af1a..ae973da03f 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -1,10 +1,10 @@ -import log from '../../logger'; import { + type PacketTrailerMetadata, extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc, - type PacketTrailerMetadata, } from '../../e2ee/packetTrailer'; +import log from '../../logger'; const MAX_ENTRIES = 300; const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; diff --git a/src/test/MockMediaStreamTrack.ts b/src/test/MockMediaStreamTrack.ts index 593143d3ec..3ccbb809fb 100644 --- a/src/test/MockMediaStreamTrack.ts +++ b/src/test/MockMediaStreamTrack.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ // @ts-ignore export default class MockMediaStreamTrack implements MediaStreamTrack { contentHint: string = ''; From c9721ffa40246a68d88fd32a14300de752b92152 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 17:03:08 -0700 Subject: [PATCH 14/31] lint --- src/room/track/PacketTrailerExtractor.ts | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index ae973da03f..85d38aa9e8 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -81,19 +81,17 @@ export class PacketTrailerExtractor { // @ts-ignore — createEncodedStreams is not in standard typings const streams = receiver.createEncodedStreams(); - const extractor = this; - const transform = new TransformStream({ - transform( + transform: ( frame: RTCEncodedVideoFrame, controller: TransformStreamDefaultController, - ) { + ) => { const result = extractPacketTrailer(frame.data); if (result.metadata) { const rtpTimestamp = getFrameRtpTimestamp(frame); const ssrc = getFrameSsrc(frame); if (rtpTimestamp !== undefined) { - extractor.storeMetadata(rtpTimestamp, ssrc, result.metadata); + this.storeMetadata(rtpTimestamp, ssrc, result.metadata); } frame.data = result.data.buffer.slice( result.data.byteOffset, From 96b1cecf232bfba45f85729bbee8921a1f2354a2 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 17:08:04 -0700 Subject: [PATCH 15/31] bump @livekit/protocol to 1.45.3 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index fedd071b0e..0bc2ebd95a 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ }, "dependencies": { "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.44.0", + "@livekit/protocol": "1.45.3", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", From 02ec08746863f0ca0dcabcf479f861189aebb783 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 17:10:02 -0700 Subject: [PATCH 16/31] update pnpm lock file --- pnpm-lock.yaml | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4a968f6b2f..74ec405a1d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: 1.44.0 - version: 1.44.0 + specifier: 1.45.3 + version: 1.45.3 '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -1128,8 +1128,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.44.0': - resolution: {integrity: sha512-/vfhDUGcUKO8Q43r6i+5FrDhl5oZjm/X3U4x2Iciqvgn5C8qbj+57YPcWSJ1kyIZm5Cm6AV2nAPjMm3ETD/iyg==} + '@livekit/protocol@1.45.3': + resolution: {integrity: sha512-WmMxBTsy4dRBqcrswFwUUlgq3Z0nnhOqKR6tX749Rb/PcB1yBMUtrHxZvcsS6qi3/5+86zHeVG+exmu1sZqfJg==} '@livekit/throws-transformer@0.1.3': resolution: {integrity: sha512-PBttE6W6g/2ALGu6kWOunZ5qdrXwP9Ge1An2/62OfE6Rhc0Abd4yp6ex2pWhwUfGxDsSZvFgoB1Ia/5mWAMuKQ==} @@ -1264,66 +1264,79 @@ packages: resolution: {integrity: sha512-t4ONHboXi/3E0rT6OZl1pKbl2Vgxf9vJfWgmUoCEVQVxhW6Cw/c8I6hbbu7DAvgp82RKiH7TpLwxnJeKv2pbsw==} cpu: [arm] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm-musleabihf@4.59.0': resolution: {integrity: sha512-CikFT7aYPA2ufMD086cVORBYGHffBo4K8MQ4uPS/ZnY54GKj36i196u8U+aDVT2LX4eSMbyHtyOh7D7Zvk2VvA==} cpu: [arm] os: [linux] + libc: [musl] '@rollup/rollup-linux-arm64-gnu@4.59.0': resolution: {integrity: sha512-jYgUGk5aLd1nUb1CtQ8E+t5JhLc9x5WdBKew9ZgAXg7DBk0ZHErLHdXM24rfX+bKrFe+Xp5YuJo54I5HFjGDAA==} cpu: [arm64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-arm64-musl@4.59.0': resolution: {integrity: sha512-peZRVEdnFWZ5Bh2KeumKG9ty7aCXzzEsHShOZEFiCQlDEepP1dpUl/SrUNXNg13UmZl+gzVDPsiCwnV1uI0RUA==} cpu: [arm64] os: [linux] + libc: [musl] '@rollup/rollup-linux-loong64-gnu@4.59.0': resolution: {integrity: sha512-gbUSW/97f7+r4gHy3Jlup8zDG190AuodsWnNiXErp9mT90iCy9NKKU0Xwx5k8VlRAIV2uU9CsMnEFg/xXaOfXg==} cpu: [loong64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-loong64-musl@4.59.0': resolution: {integrity: sha512-yTRONe79E+o0FWFijasoTjtzG9EBedFXJMl888NBEDCDV9I2wGbFFfJQQe63OijbFCUZqxpHz1GzpbtSFikJ4Q==} cpu: [loong64] os: [linux] + libc: [musl] '@rollup/rollup-linux-ppc64-gnu@4.59.0': resolution: {integrity: sha512-sw1o3tfyk12k3OEpRddF68a1unZ5VCN7zoTNtSn2KndUE+ea3m3ROOKRCZxEpmT9nsGnogpFP9x6mnLTCaoLkA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-ppc64-musl@4.59.0': resolution: {integrity: sha512-+2kLtQ4xT3AiIxkzFVFXfsmlZiG5FXYW7ZyIIvGA7Bdeuh9Z0aN4hVyXS/G1E9bTP/vqszNIN/pUKCk/BTHsKA==} cpu: [ppc64] os: [linux] + libc: [musl] '@rollup/rollup-linux-riscv64-gnu@4.59.0': resolution: {integrity: sha512-NDYMpsXYJJaj+I7UdwIuHHNxXZ/b/N2hR15NyH3m2qAtb/hHPA4g4SuuvrdxetTdndfj9b1WOmy73kcPRoERUg==} cpu: [riscv64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-riscv64-musl@4.59.0': resolution: {integrity: sha512-nLckB8WOqHIf1bhymk+oHxvM9D3tyPndZH8i8+35p/1YiVoVswPid2yLzgX7ZJP0KQvnkhM4H6QZ5m0LzbyIAg==} cpu: [riscv64] os: [linux] + libc: [musl] '@rollup/rollup-linux-s390x-gnu@4.59.0': resolution: {integrity: sha512-oF87Ie3uAIvORFBpwnCvUzdeYUqi2wY6jRFWJAy1qus/udHFYIkplYRW+wo+GRUP4sKzYdmE1Y3+rY5Gc4ZO+w==} cpu: [s390x] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-gnu@4.59.0': resolution: {integrity: sha512-3AHmtQq/ppNuUspKAlvA8HtLybkDflkMuLK4DPo77DfthRb71V84/c4MlWJXixZz4uruIH4uaa07IqoAkG64fg==} cpu: [x64] os: [linux] + libc: [glibc] '@rollup/rollup-linux-x64-musl@4.59.0': resolution: {integrity: sha512-2UdiwS/9cTAx7qIUZB/fWtToJwvt0Vbo0zmnYt7ED35KPg13Q0ym1g442THLC7VyI6JfYTP4PiSOWyoMdV2/xg==} cpu: [x64] os: [linux] + libc: [musl] '@rollup/rollup-openbsd-x64@4.59.0': resolution: {integrity: sha512-M3bLRAVk6GOwFlPTIxVBSYKUaqfLrn8l0psKinkCFxl4lQvOSz8ZrKDz2gxcBwHFpci0B6rttydI4IpS4IS/jQ==} @@ -1662,41 +1675,49 @@ packages: resolution: {integrity: sha512-34gw7PjDGB9JgePJEmhEqBhWvCiiWCuXsL9hYphDF7crW7UgI05gyBAi6MF58uGcMOiOqSJ2ybEeCvHcq0BCmQ==} cpu: [arm64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-arm64-musl@1.11.1': resolution: {integrity: sha512-RyMIx6Uf53hhOtJDIamSbTskA99sPHS96wxVE/bJtePJJtpdKGXO1wY90oRdXuYOGOTuqjT8ACccMc4K6QmT3w==} cpu: [arm64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-ppc64-gnu@1.11.1': resolution: {integrity: sha512-D8Vae74A4/a+mZH0FbOkFJL9DSK2R6TFPC9M+jCWYia/q2einCubX10pecpDiTmkJVUH+y8K3BZClycD8nCShA==} cpu: [ppc64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-gnu@1.11.1': resolution: {integrity: sha512-frxL4OrzOWVVsOc96+V3aqTIQl1O2TjgExV4EKgRY09AJ9leZpEg8Ak9phadbuX0BA4k8U5qtvMSQQGGmaJqcQ==} cpu: [riscv64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-riscv64-musl@1.11.1': resolution: {integrity: sha512-mJ5vuDaIZ+l/acv01sHoXfpnyrNKOk/3aDoEdLO/Xtn9HuZlDD6jKxHlkN8ZhWyLJsRBxfv9GYM2utQ1SChKew==} cpu: [riscv64] os: [linux] + libc: [musl] '@unrs/resolver-binding-linux-s390x-gnu@1.11.1': resolution: {integrity: sha512-kELo8ebBVtb9sA7rMe1Cph4QHreByhaZ2QEADd9NzIQsYNQpt9UkM9iqr2lhGr5afh885d/cB5QeTXSbZHTYPg==} cpu: [s390x] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-gnu@1.11.1': resolution: {integrity: sha512-C3ZAHugKgovV5YvAMsxhq0gtXuwESUKc5MhEtjBpLoHPLYM+iuwSj3lflFwK3DPm68660rZ7G8BMcwSro7hD5w==} cpu: [x64] os: [linux] + libc: [glibc] '@unrs/resolver-binding-linux-x64-musl@1.11.1': resolution: {integrity: sha512-rV0YSoyhK2nZ4vEswT/QwqzqQXw5I6CjoaYMOX0TqBlWhojUf8P94mvI7nuJTeaCkkds3QE4+zS8Ko+GdXuZtA==} cpu: [x64] os: [linux] + libc: [musl] '@unrs/resolver-binding-wasm32-wasi@1.11.1': resolution: {integrity: sha512-5u4RkfxJm+Ng7IWgkzi3qrFOvLvQYnPBmjmZQ8+szTK/b31fQCnleNl1GgEt7nIsZRIf5PLhPwT0WM+q45x/UQ==} @@ -3824,8 +3845,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - typescript@6.0.0-dev.20260324: - resolution: {integrity: sha512-+Vhswp/n1W3i/33j4ar/yeVz4xuI+7gmHkKKZw5eM2zN5PDB4umd36EpDDLjRO0sgH31G1k4bsu7kr84MfTtug==} + typescript@6.0.0-dev.20260401: + resolution: {integrity: sha512-trGd1r4vZ89ABUR8VnF2Lsl/Tb9ObfQNH3r+kF8NFVSLCq3b3x96McL6pXMnH/aa4tutrziJZvb7zm3WsAEfCA==} engines: {node: '>=14.17'} hasBin: true @@ -5279,7 +5300,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.44.0': + '@livekit/protocol@1.45.3': dependencies: '@bufbuild/protobuf': 1.10.1 @@ -6379,7 +6400,7 @@ snapshots: dependencies: semver: 7.6.0 shelljs: 0.8.5 - typescript: 6.0.0-dev.20260324 + typescript: 6.0.0-dev.20260401 dunder-proto@1.0.1: dependencies: @@ -8241,7 +8262,7 @@ snapshots: typescript@5.8.3: {} - typescript@6.0.0-dev.20260324: {} + typescript@6.0.0-dev.20260401: {} uc.micro@2.1.0: {} From ed291a7a853e734d19e5063ec7bd2b759e8b8b98 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 23:17:23 -0700 Subject: [PATCH 17/31] formatting --- examples/demo/demo.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index 2654930a03..e6d0b0b6ba 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -1,7 +1,5 @@ //@ts-ignore import E2EEWorker from '../../src/e2ee/worker/e2ee.worker?worker'; -//@ts-ignore -import PTWorker from '../../src/packetTrailer/worker/packetTrailer.worker?worker'; import type { ChatMessage, LocalDataTrack, @@ -43,8 +41,10 @@ import { supportsAV1, supportsVP9, } from '../../src/index'; -import { TrackEvent } from '../../src/room/events'; +//@ts-ignore +import PTWorker from '../../src/packetTrailer/worker/packetTrailer.worker?worker'; import type { DataTrackFrame } from '../../src/room/data-track/frame'; +import { TrackEvent } from '../../src/room/events'; import { isSVCCodec, sleep, supportsH265 } from '../../src/room/utils'; setLogLevel(LogLevel.debug); From b3d39e3f4884cf98c1a221881fe46d044bbba185 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 14 Apr 2026 23:20:47 -0700 Subject: [PATCH 18/31] fix access to packetTrailerExtractor --- src/room/track/RemoteVideoTrack.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 81bed9e36d..6b620a5045 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -25,7 +25,8 @@ export default class RemoteVideoTrack extends RemoteTrack { private lastDimensions?: Track.Dimensions; - private packetTrailerExtractor?: PacketTrailerExtractor; + /** @internal */ + packetTrailerExtractor?: PacketTrailerExtractor; constructor( mediaTrack: MediaStreamTrack, From 4bd5adad3daa2304e885827aa7c321d05c72a7f5 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 10:11:47 -0700 Subject: [PATCH 19/31] fix reconnect logic & dont attach transformer to published tracks from js --- src/packetTrailer/PacketTrailerManager.ts | 97 +++++++++++++------ src/packetTrailer/types.ts | 17 ++-- .../worker/packetTrailer.worker.ts | 38 +++++--- src/room/RTCEngine.ts | 60 +++++++++--- 4 files changed, 144 insertions(+), 68 deletions(-) diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 22115972b0..6dea05bf20 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -4,10 +4,8 @@ import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; -import type RemoteVideoTrack from '../room/track/RemoteVideoTrack'; -import type { PTDecodeMessage, PTRemoveTransformMessage, PTWorkerMessage } from './types'; - -const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; +import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; +import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; export interface PacketTrailerOptions { worker: Worker; @@ -34,6 +32,15 @@ export class PacketTrailerManager { private extractors = new Map(); + /** + * Maps each receiver that has had its encoded streams transferred to + * the worker to the trackId currently associated with that pipeline. + * Used to detect receiver reuse (transceiver recycling) so we can + * update the worker's trackId instead of trying to re-transfer the + * already-consumed streams. + */ + private receiverPipelines = new Map(); + constructor(options: PacketTrailerOptions) { this.worker = options.worker; } @@ -63,6 +70,9 @@ export class PacketTrailerManager { }) .on(RoomEvent.TrackUnsubscribed, (track) => { this.teardownTrack(track); + }) + .on(RoomEvent.Disconnected, () => { + this.cleanup(); }); } @@ -71,48 +81,60 @@ export class PacketTrailerManager { return; } - if (PACKET_TRAILER_FLAG in track.receiver) { - return; - } - const extractor = new PacketTrailerExtractor(); - const trackId = track.mediaStreamID; + const newTrackId = track.mediaStreamID; - this.extractors.set(trackId, extractor); + this.extractors.set(newTrackId, extractor); track.packetTrailerExtractor = extractor; const hasE2EE = !!this.room?.hasE2EESetup; if (hasE2EE) { - // When E2EE is active, the FrameCryptor worker strips the trailer - // inside its decodeFunction before decryption and sends metadata - // back via the E2EEManager. No separate pipeline is needed here; - // we only create the extractor/cache above. return; } const receiver = track.receiver; - - if (!('createEncodedStreams' in receiver)) { - log.warn('createEncodedStreams not supported, packet trailer extraction unavailable'); + const existingTrackId = this.receiverPipelines.get(receiver); + + if (existingTrackId) { + // Receiver is reused (transceiver recycled). The worker already + // owns the encoded streams — just remap the trackId so metadata + // is keyed correctly and re-activate processing. + log.info('PacketTrailerManager: reusing pipeline for receiver', { + oldTrackId: existingTrackId, + newTrackId, + }); + const msg: PTUpdateTrackIdMessage = { + kind: 'updateTrackId', + data: { oldTrackId: existingTrackId, newTrackId }, + }; + this.worker.postMessage(msg); + this.receiverPipelines.set(receiver, newTrackId); return; } - // @ts-ignore -- createEncodedStreams is not in standard typings - const streams = receiver.createEncodedStreams(); + // @ts-ignore + const readable: ReadableStream | undefined = receiver.readableStream; + // @ts-ignore + const writable: WritableStream | undefined = receiver.writableStream; + + if (!readable || !writable) { + log.warn( + 'encoded streams not available on receiver — ensure encodedInsertableStreams is enabled', + ); + return; + } const msg: PTDecodeMessage = { kind: 'decode', data: { - readableStream: streams.readable, - writableStream: streams.writable, - trackId, + readableStream: readable, + writableStream: writable, + trackId: newTrackId, }, }; - this.worker.postMessage(msg, [streams.readable, streams.writable]); - - // @ts-ignore - receiver[PACKET_TRAILER_FLAG] = true; + this.worker.postMessage(msg, [readable, writable]); + this.receiverPipelines.set(receiver, newTrackId); } private teardownTrack(track: RemoteTrack) { @@ -123,13 +145,24 @@ export class PacketTrailerManager { this.extractors.delete(trackId); } - if (!this.room?.hasE2EESetup) { - const msg: PTRemoveTransformMessage = { - kind: 'removeTransform', - data: { trackId }, - }; - this.worker.postMessage(msg); + if (track instanceof RemoteVideoTrack) { + track.packetTrailerExtractor = undefined; + } + + // The worker pipeline is intentionally left running on the receiver. If the + // receiver is reused for a new track, `setupReceiver` will send an + // `updateTrackId` to remap it. If the room disconnects, `cleanup` terminates + // the worker entirely. Any metadata posted in the meantime is dropped because + // the extractor lookup above has already been removed. + } + + private cleanup() { + for (const extractor of this.extractors.values()) { + extractor.dispose(); } + this.extractors.clear(); + this.receiverPipelines.clear(); + this.worker.terminate(); } private onWorkerMessage = (ev: MessageEvent) => { diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts index 1b7936c662..63fd5910e3 100644 --- a/src/packetTrailer/types.ts +++ b/src/packetTrailer/types.ts @@ -22,13 +22,6 @@ export interface PTDecodeMessage extends PTBaseMessage { }; } -export interface PTRemoveTransformMessage extends PTBaseMessage { - kind: 'removeTransform'; - data: { - trackId: string; - }; -} - export interface PTMetadataMessage extends PTBaseMessage { kind: 'metadata'; data: { @@ -39,9 +32,17 @@ export interface PTMetadataMessage extends PTBaseMessage { }; } +export interface PTUpdateTrackIdMessage extends PTBaseMessage { + kind: 'updateTrackId'; + data: { + oldTrackId: string; + newTrackId: string; + }; +} + export type PTWorkerMessage = | PTInitMessage | PTInitAck | PTDecodeMessage - | PTRemoveTransformMessage + | PTUpdateTrackIdMessage | PTMetadataMessage; diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 448eff4e90..51b3f30134 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -1,7 +1,16 @@ import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../../e2ee/packetTrailer'; import type { PTMetadataMessage, PTWorkerMessage } from '../types'; -const activeTransforms = new Map(); +/** + * Holds the trackId currently associated with a pipeline. A mutable + * wrapper is used so the transform closure always reads the latest + * trackId after a receiver gets re-bound to a new track. + */ +interface PipelineState { + trackId: string; +} + +const pipelines = new Map(); onmessage = (ev: MessageEvent) => { const msg = ev.data; @@ -15,8 +24,8 @@ onmessage = (ev: MessageEvent) => { setupDecodeTransform(msg.data.readableStream, msg.data.writableStream, msg.data.trackId); break; - case 'removeTransform': - teardownTransform(msg.data.trackId); + case 'updateTrackId': + updateTrackId(msg.data.oldTrackId, msg.data.newTrackId); break; default: @@ -25,10 +34,8 @@ onmessage = (ev: MessageEvent) => { }; function setupDecodeTransform(readable: ReadableStream, writable: WritableStream, trackId: string) { - teardownTransform(trackId); - - const abortController = new AbortController(); - activeTransforms.set(trackId, abortController); + const state: PipelineState = { trackId }; + pipelines.set(trackId, state); const transform = new TransformStream({ transform( @@ -43,7 +50,7 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream const msg: PTMetadataMessage = { kind: 'metadata', data: { - trackId, + trackId: state.trackId, rtpTimestamp, ssrc, metadata: result.metadata, @@ -62,16 +69,17 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream readable .pipeThrough(transform) - .pipeTo(writable, { signal: abortController.signal }) + .pipeTo(writable) .catch(() => { - // pipe aborted via teardown -- expected + pipelines.delete(state.trackId); }); } -function teardownTransform(trackId: string) { - const existing = activeTransforms.get(trackId); - if (existing) { - existing.abort(); - activeTransforms.delete(trackId); +function updateTrackId(oldTrackId: string, newTrackId: string) { + const state = pipelines.get(oldTrackId); + if (state) { + state.trackId = newTrackId; + pipelines.delete(oldTrackId); + pipelines.set(newTrackId, state); } } diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 36a6c26b36..f4c6700b35 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -601,6 +601,25 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit // this fires after the underlying transceiver is stopped and potentially // peer connection closed, so do not bubble up if there are no streams if (ev.streams.length === 0) return; + if ( + this.options.packetTrailer && + !this.signalOpts?.e2eeEnabled && + ev.track.kind === 'video' && + 'createEncodedStreams' in ev.receiver + ) { + try { + // @ts-ignore + const streams = ev.receiver.createEncodedStreams(); + // @ts-ignore + ev.receiver.readableStream = streams.readable; + // @ts-ignore + ev.receiver.writableStream = streams.writable; + } catch { + // createEncodedStreams() can only be called once per receiver. + // When a receiver is reused for a new track the existing worker + // pipeline continues to process frames automatically. + } + } this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver); }; } @@ -990,16 +1009,17 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit opts: TrackPublishOptions, encodings?: RTCRtpEncodingParameters[], ) { + let sender: RTCRtpSender; if (supportsTransceiver()) { - const sender = await this.createTransceiverRTCRtpSender(track, opts, encodings); - return sender; - } - if (supportsAddTrack()) { + sender = await this.createTransceiverRTCRtpSender(track, opts, encodings); + } else if (supportsAddTrack()) { this.log.warn('using add-track fallback', this.logContext); - const sender = await this.createRTCRtpSender(track.mediaStreamTrack); - return sender; + sender = await this.createRTCRtpSender(track.mediaStreamTrack); + } else { + throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device'); } - throw new UnexpectedConnectionState('Required webRTC APIs not supported on this device'); + this.setupSenderPassthrough(sender); + return sender; } async createSimulcastSender( @@ -1008,16 +1028,30 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit opts: TrackPublishOptions, encodings?: RTCRtpEncodingParameters[], ) { - // store RTCRtpSender + let sender: RTCRtpSender | undefined; if (supportsTransceiver()) { - return this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings); - } - if (supportsAddTrack()) { + sender = await this.createSimulcastTransceiverSender(track, simulcastTrack, opts, encodings); + } else if (supportsAddTrack()) { this.log.debug('using add-track fallback', this.logContext); - return this.createRTCRtpSender(track.mediaStreamTrack); + sender = await this.createRTCRtpSender(track.mediaStreamTrack); + } else { + throw new UnexpectedConnectionState('Cannot stream on this device'); + } + if (sender) { + this.setupSenderPassthrough(sender); } + return sender; + } - throw new UnexpectedConnectionState('Cannot stream on this device'); + private setupSenderPassthrough(sender: RTCRtpSender) { + if (!this.options.packetTrailer || this.signalOpts?.e2eeEnabled) { + return; + } + if ('createEncodedStreams' in sender) { + // @ts-ignore + const { readable, writable } = sender.createEncodedStreams(); + readable.pipeTo(writable); + } } private async createTransceiverRTCRtpSender( From 9f91c94a788d01c9ac731c64fa0b431bad6e26c7 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 13:20:28 -0700 Subject: [PATCH 20/31] backup transformer can be added in main thread if worker is not created --- src/e2ee/E2eeManager.ts | 6 +- src/packetTrailer/PacketTrailerManager.ts | 245 +++++++++++++----- .../worker/packetTrailer.worker.ts | 43 +-- src/room/RTCEngine.ts | 28 +- src/room/Room.ts | 26 +- src/room/track/PacketTrailerExtractor.ts | 68 +---- src/room/track/RemoteVideoTrack.ts | 5 +- 7 files changed, 236 insertions(+), 185 deletions(-) diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index a1f026492e..d4db0ec455 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -510,15 +510,11 @@ export class E2EEManager return; } - // @ts-ignore -- readableStream is set by PacketTrailerManager when it has pre-processed the receiver - const ptPreProcessed = !!receiver.readableStream && !!receiver.writableStream; - if ( isScriptTransformSupported() && // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141. // Disabling it for Chrome based browsers until the API has stabilized - !isChromiumBased() && - !ptPreProcessed + !isChromiumBased() ) { const options: ScriptTransformOptions = { kind: 'decode', diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 6dea05bf20..3f37e91552 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -1,4 +1,5 @@ import type { TrackInfo } from '@livekit/protocol'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../e2ee/packetTrailer'; import log from '../logger'; import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; @@ -8,40 +9,76 @@ import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; export interface PacketTrailerOptions { - worker: Worker; + /** + * Optional dedicated worker for extracting packet trailers off the main thread. + * + * When provided, encoded video streams are transferred to the worker for + * processing, which avoids any per-frame work on the main thread and is the + * recommended configuration for production. + * + * When omitted, the manager falls back to an inline `TransformStream` on the + * main thread. This is a safety net so video still decodes correctly if the + * worker wasn't wired up but a publishing track advertises packet trailer + * features — at a small CPU cost on the subscriber. + */ + worker?: Worker; +} + +/** @internal */ +const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; + +/** Mutable pipeline state so the transform closure can be remapped when a receiver is reused. */ +interface MainThreadPipelineState { + extractor: PacketTrailerExtractor; } /** * Manages packet trailer extraction for received video tracks. * * When a track's TrackInfo indicates packet trailer features, the manager - * wires up an Insertable Streams pipeline (via a dedicated worker) to strip - * the trailer from encoded frames and cache the metadata for lookup. + * wires up an Insertable Streams pipeline to strip the trailer from encoded + * frames and cache the metadata for lookup. + * + * Two processing modes are supported: + * - **Worker** (preferred): encoded streams are transferred to a dedicated + * worker. No per-frame work happens on the main thread. + * - **Main-thread fallback**: if no worker was supplied, the transform runs + * inline. Slightly higher main-thread cost, but ensures video still + * decodes correctly when the worker was not wired up. * * When E2EE is active, the E2EE FrameCryptor worker handles trailer * extraction directly (before decryption), so this manager only creates - * the extractor/metadata cache — no separate insertable-streams pipeline - * is needed. + * the extractor/metadata cache — no separate pipeline is installed. * * @experimental */ export class PacketTrailerManager { - private worker: Worker; + private worker?: Worker; private room?: Room; private extractors = new Map(); /** - * Maps each receiver that has had its encoded streams transferred to - * the worker to the trackId currently associated with that pipeline. - * Used to detect receiver reuse (transceiver recycling) so we can - * update the worker's trackId instead of trying to re-transfer the - * already-consumed streams. + * Tracks the trackId associated with each receiver that has had its + * encoded streams handed off to the worker. Used to detect receiver + * reuse (transceiver recycling) so we can remap trackIds instead of + * re-transferring already-consumed streams. */ - private receiverPipelines = new Map(); + private workerPipelines = new Map(); - constructor(options: PacketTrailerOptions) { + /** + * Tracks the active main-thread pipeline state for each receiver. When a + * receiver is reused for a new track, the state's extractor is swapped + * in-place so the existing `TransformStream` keeps feeding the correct + * extractor without re-calling `createEncodedStreams` (which would throw). + */ + private mainThreadPipelines = new Map(); + + /** Ensures the "no worker, using main-thread fallback" warning only fires once per room. */ + private mainThreadFallbackWarned = false; + + constructor(options: PacketTrailerOptions = {}) { this.worker = options.worker; } @@ -52,20 +89,17 @@ export class PacketTrailerManager { } this.room = room; - this.worker.onmessage = this.onWorkerMessage; - this.worker.onerror = this.onWorkerError; - this.worker.postMessage({ kind: 'init' }); + if (this.worker) { + this.worker.onmessage = this.onWorkerMessage; + this.worker.onerror = this.onWorkerError; + this.worker.postMessage({ kind: 'init' }); + } room .on(RoomEvent.TrackSubscribed, (track, pub, _participant) => { if (track.kind !== 'video') { return; } - log.info('PacketTrailerManager: subscribed video track', { - trackSid: pub.trackSid, - packetTrailerFeatures: pub.trackInfo?.packetTrailerFeatures, - trackInfo: pub.trackInfo, - }); this.setupReceiver(track as unknown as RemoteVideoTrack, pub.trackInfo); }) .on(RoomEvent.TrackUnsubscribed, (track) => { @@ -76,65 +110,159 @@ export class PacketTrailerManager { }); } - private setupReceiver(track: RemoteVideoTrack, _trackInfo?: TrackInfo) { - if (!track.receiver) { + private setupReceiver(track: RemoteVideoTrack, trackInfo?: TrackInfo) { + const receiver = track.receiver; + if (!receiver) { + return; + } + + // Only install a pipeline for tracks that actually advertise packet + // trailer features. This keeps us out of the way for tracks published by + // clients on older protocols or that don't opt into the feature. + const hasFeatures = + !!trackInfo?.packetTrailerFeatures && trackInfo.packetTrailerFeatures.length > 0; + if (!hasFeatures) { return; } const extractor = new PacketTrailerExtractor(); - const newTrackId = track.mediaStreamID; + const trackId = track.mediaStreamID; - this.extractors.set(newTrackId, extractor); + this.extractors.set(trackId, extractor); track.packetTrailerExtractor = extractor; - const hasE2EE = !!this.room?.hasE2EESetup; - - if (hasE2EE) { + if (this.room?.hasE2EESetup) { + // E2EE worker strips the trailer and injects metadata directly into + // the extractor via E2eeManager; no pipeline is needed here. return; } - const receiver = track.receiver; - const existingTrackId = this.receiverPipelines.get(receiver); + log.debug('PacketTrailerManager: installing pipeline', { + trackSid: track.sid, + mode: this.worker ? 'worker' : 'main-thread', + }); + + if (this.worker) { + this.setupWorkerReceiver(receiver, trackId); + } else { + if (!this.mainThreadFallbackWarned) { + log.warn( + 'subscribed to a track with packet trailer features but no packet trailer worker ' + + 'is configured — falling back to main-thread frame processing. For best performance ' + + 'pass `packetTrailer: { worker }` in RoomOptions.', + ); + this.mainThreadFallbackWarned = true; + } + this.setupMainThreadReceiver(receiver, extractor); + } + } + + private setupWorkerReceiver(receiver: RTCRtpReceiver, newTrackId: string) { + const worker = this.worker!; + const existingTrackId = this.workerPipelines.get(receiver); if (existingTrackId) { - // Receiver is reused (transceiver recycled). The worker already - // owns the encoded streams — just remap the trackId so metadata - // is keyed correctly and re-activate processing. - log.info('PacketTrailerManager: reusing pipeline for receiver', { - oldTrackId: existingTrackId, - newTrackId, - }); + // Receiver is reused (transceiver recycled). The worker already owns + // the encoded streams — just remap the trackId so metadata is keyed + // correctly and re-activate processing. const msg: PTUpdateTrackIdMessage = { kind: 'updateTrackId', data: { oldTrackId: existingTrackId, newTrackId }, }; - this.worker.postMessage(msg); - this.receiverPipelines.set(receiver, newTrackId); + worker.postMessage(msg); + this.workerPipelines.set(receiver, newTrackId); return; } - // @ts-ignore - const readable: ReadableStream | undefined = receiver.readableStream; - // @ts-ignore - const writable: WritableStream | undefined = receiver.writableStream; + if (!('createEncodedStreams' in receiver)) { + log.warn('createEncodedStreams not supported, packet trailer extraction unavailable'); + return; + } - if (!readable || !writable) { - log.warn( - 'encoded streams not available on receiver — ensure encodedInsertableStreams is enabled', - ); + let streams: { readable: ReadableStream; writable: WritableStream }; + try { + // @ts-ignore — createEncodedStreams is not in standard typings + streams = receiver.createEncodedStreams(); + } catch (err) { + log.warn('failed to create encoded streams for packet trailer extraction', { error: err }); return; } const msg: PTDecodeMessage = { kind: 'decode', data: { - readableStream: readable, - writableStream: writable, + readableStream: streams.readable, + writableStream: streams.writable, trackId: newTrackId, }, }; - this.worker.postMessage(msg, [readable, writable]); - this.receiverPipelines.set(receiver, newTrackId); + worker.postMessage(msg, [streams.readable, streams.writable]); + this.workerPipelines.set(receiver, newTrackId); + } + + private setupMainThreadReceiver(receiver: RTCRtpReceiver, extractor: PacketTrailerExtractor) { + const existingState = this.mainThreadPipelines.get(receiver); + if (existingState) { + // Receiver was reused for a new track. The pipeline is still running + // on the already-transferred encoded streams — just swap the extractor + // the transform feeds into. + existingState.extractor = extractor; + return; + } + + if (PACKET_TRAILER_FLAG in receiver) { + return; + } + if (!('createEncodedStreams' in receiver)) { + log.debug('createEncodedStreams not supported, packet trailer extraction unavailable'); + return; + } + + let streams: { readable: ReadableStream; writable: WritableStream }; + try { + // @ts-ignore — createEncodedStreams is not in standard typings + streams = receiver.createEncodedStreams(); + } catch (err) { + log.warn('failed to create encoded streams for packet trailer extraction', { error: err }); + return; + } + + const state: MainThreadPipelineState = { extractor }; + this.mainThreadPipelines.set(receiver, state); + + const transform = new TransformStream({ + transform: (frame, controller) => { + try { + const result = extractPacketTrailer(frame.data); + if (result.metadata) { + const rtpTimestamp = getFrameRtpTimestamp(frame); + const ssrc = getFrameSsrc(frame); + if (rtpTimestamp !== undefined) { + state.extractor.storeMetadata(rtpTimestamp, ssrc, result.metadata); + } + frame.data = result.data.buffer.slice( + result.data.byteOffset, + result.data.byteOffset + result.data.byteLength, + ) as ArrayBuffer; + } + } catch (err) { + // Never drop frames on trailer-extraction failure — pass through so + // video keeps decoding even if metadata is lost for this frame. + log.debug('packet trailer extraction failed, passing frame through', { error: err }); + } + controller.enqueue(frame); + }, + }); + + streams.readable + .pipeThrough(transform) + .pipeTo(streams.writable) + .catch((err) => { + log.debug('packet trailer pipeline ended', { error: err }); + }); + + // @ts-ignore + receiver[PACKET_TRAILER_FLAG] = true; } private teardownTrack(track: RemoteTrack) { @@ -149,11 +277,11 @@ export class PacketTrailerManager { track.packetTrailerExtractor = undefined; } - // The worker pipeline is intentionally left running on the receiver. If the - // receiver is reused for a new track, `setupReceiver` will send an - // `updateTrackId` to remap it. If the room disconnects, `cleanup` terminates - // the worker entirely. Any metadata posted in the meantime is dropped because - // the extractor lookup above has already been removed. + // The receiver pipeline (worker or main-thread) is intentionally left + // running. If the receiver is reused for a new track, `setupReceiver` will + // remap it. If the room disconnects, `cleanup` drops all state. Any + // metadata produced in the meantime is harmless — the extractor above has + // already been disposed and is no longer reachable from any track. } private cleanup() { @@ -161,8 +289,9 @@ export class PacketTrailerManager { extractor.dispose(); } this.extractors.clear(); - this.receiverPipelines.clear(); - this.worker.terminate(); + this.workerPipelines.clear(); + this.mainThreadPipelines.clear(); + this.worker?.terminate(); } private onWorkerMessage = (ev: MessageEvent) => { diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 51b3f30134..aee1771840 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -42,26 +42,31 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream frame: RTCEncodedVideoFrame, controller: TransformStreamDefaultController, ) { - const result = extractPacketTrailer(frame.data); - if (result.metadata) { - const rtpTimestamp = getFrameRtpTimestamp(frame); - const ssrc = getFrameSsrc(frame); - if (rtpTimestamp !== undefined) { - const msg: PTMetadataMessage = { - kind: 'metadata', - data: { - trackId: state.trackId, - rtpTimestamp, - ssrc, - metadata: result.metadata, - }, - }; - postMessage(msg); + try { + const result = extractPacketTrailer(frame.data); + if (result.metadata) { + const rtpTimestamp = getFrameRtpTimestamp(frame); + const ssrc = getFrameSsrc(frame); + if (rtpTimestamp !== undefined) { + const msg: PTMetadataMessage = { + kind: 'metadata', + data: { + trackId: state.trackId, + rtpTimestamp, + ssrc, + metadata: result.metadata, + }, + }; + postMessage(msg); + } + frame.data = result.data.buffer.slice( + result.data.byteOffset, + result.data.byteOffset + result.data.byteLength, + ); } - frame.data = result.data.buffer.slice( - result.data.byteOffset, - result.data.byteOffset + result.data.byteLength, - ); + } catch { + // Never drop frames on trailer-extraction failure — pass through so + // video keeps decoding even if metadata is lost for this frame. } controller.enqueue(frame); }, diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index f4c6700b35..dbc72874a1 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -53,7 +53,7 @@ import { toProtoSessionDescription, } from '../api/SignalClient'; import type { BaseE2EEManager } from '../e2ee/E2eeManager'; -import { asEncryptablePacket } from '../e2ee/utils'; +import { asEncryptablePacket, isInsertableStreamSupported } from '../e2ee/utils'; import log, { LoggerNames, getLogger } from '../logger'; import type { InternalRoomOptions } from '../options'; import TypedPromise from '../utils/TypedPromise'; @@ -601,25 +601,6 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit // this fires after the underlying transceiver is stopped and potentially // peer connection closed, so do not bubble up if there are no streams if (ev.streams.length === 0) return; - if ( - this.options.packetTrailer && - !this.signalOpts?.e2eeEnabled && - ev.track.kind === 'video' && - 'createEncodedStreams' in ev.receiver - ) { - try { - // @ts-ignore - const streams = ev.receiver.createEncodedStreams(); - // @ts-ignore - ev.receiver.readableStream = streams.readable; - // @ts-ignore - ev.receiver.writableStream = streams.writable; - } catch { - // createEncodedStreams() can only be called once per receiver. - // When a receiver is reused for a new track the existing worker - // pipeline continues to process frames automatically. - } - } this.emit(EngineEvent.MediaTrackAdded, ev.track, ev.streams[0], ev.receiver); }; } @@ -764,7 +745,12 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit ): RTCConfiguration { const rtcConfig = { ...this.rtcConfig }; - if (this.signalOpts?.e2eeEnabled || this.options.packetTrailer) { + // Always enable encoded insertable streams when supported. E2EE and packet + // trailer both rely on it, and enabling the flag is a no-op when nothing + // calls `createEncodedStreams()`. Having it always on means subscribers + // can automatically handle publishers that advertise packet trailer + // features without any explicit RoomOptions configuration. + if (isInsertableStreamSupported()) { this.log.debug('setting up transports with insertable streams', this.logContext); // this makes sure that no data is sent before the transforms are ready // @ts-ignore diff --git a/src/room/Room.ts b/src/room/Room.ts index 99dc66b9c1..9c0ab9a649 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -308,9 +308,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) this.outgoingDataTrackManager, ); - if (this.options.packetTrailer) { - this.setupPacketTrailer(); - } + this.setupPacketTrailer(); if (this.options.e2ee || this.options.encryption) { this.setupE2EE(); @@ -471,10 +469,13 @@ class Room extends (EventEmitter as new () => TypedEmitter) } private setupPacketTrailer() { - if (this.options.packetTrailer) { - this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer); - this.packetTrailerManager.setup(this); - } + // The manager is always created so tracks that advertise packet trailer + // features always have their trailers stripped — even if the app didn't + // pass `packetTrailer` in RoomOptions. A worker is used when provided for + // best performance; otherwise the manager falls back to a main-thread + // TransformStream so video still decodes correctly. + this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer); + this.packetTrailerManager.setup(this); } private get logContext() { @@ -2333,17 +2334,6 @@ class Room extends (EventEmitter as new () => TypedEmitter) track.on(TrackEvent.VideoPlaybackFailed, this.handleVideoPlaybackFailed); track.on(TrackEvent.VideoPlaybackStarted, this.handleVideoPlaybackStarted); } - if ( - !this.packetTrailerManager && - publication.trackInfo?.packetTrailerFeatures && - publication.trackInfo.packetTrailerFeatures.length > 0 - ) { - this.log.warn( - 'Track has packet trailer features but no packet trailer worker is configured. ' + - 'Pass packetTrailer: { worker } in RoomOptions to enable frame metadata extraction.', - { ...this.logContext, trackSid: publication.trackSid }, - ); - } this.emitWhenConnected(RoomEvent.TrackSubscribed, track, publication, participant); }, ) diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index 85d38aa9e8..a24944c0f0 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -1,25 +1,14 @@ -import { - type PacketTrailerMetadata, - extractPacketTrailer, - getFrameRtpTimestamp, - getFrameSsrc, -} from '../../e2ee/packetTrailer'; -import log from '../../logger'; +import type { PacketTrailerMetadata } from '../../e2ee/packetTrailer'; const MAX_ENTRIES = 300; -const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; /** - * Extracts and caches packet trailer metadata from received video frames. + * Caches packet trailer metadata extracted from received video frames, + * keyed by RTP timestamp so it can be looked up when the frame is displayed. * - * In the non-E2EE path, this sets up an Insertable Streams pipeline on the - * receiver to strip trailers from encoded frames on the main thread. - * - * In the E2EE path, metadata is injected externally after the worker decrypts - * and strips the trailer. - * - * Metadata is stored in an LRU map keyed by RTP timestamp so it can be - * looked up when the frame is displayed. + * Metadata is populated either by the main-thread pipeline installed by + * `PacketTrailerManager` (non-E2EE) or by the E2EE FrameCryptor worker + * after decryption (E2EE). * * @experimental */ @@ -64,51 +53,6 @@ export class PacketTrailerExtractor { return this.metadataMap.get(rtpTimestamp); } - /** - * Sets up an Insertable Streams pipeline on the receiver to extract - * packet trailers from encoded video frames on the main thread. - * Only used when E2EE is NOT active. - */ - setupReceiver(receiver: RTCRtpReceiver): boolean { - if (PACKET_TRAILER_FLAG in receiver) { - return true; - } - - if (!('createEncodedStreams' in receiver)) { - log.debug('createEncodedStreams not supported, packet trailer extraction unavailable'); - return false; - } - - // @ts-ignore — createEncodedStreams is not in standard typings - const streams = receiver.createEncodedStreams(); - const transform = new TransformStream({ - transform: ( - frame: RTCEncodedVideoFrame, - controller: TransformStreamDefaultController, - ) => { - const result = extractPacketTrailer(frame.data); - if (result.metadata) { - const rtpTimestamp = getFrameRtpTimestamp(frame); - const ssrc = getFrameSsrc(frame); - if (rtpTimestamp !== undefined) { - this.storeMetadata(rtpTimestamp, ssrc, result.metadata); - } - frame.data = result.data.buffer.slice( - result.data.byteOffset, - result.data.byteOffset + result.data.byteLength, - ); - } - controller.enqueue(frame); - }, - }); - - streams.readable.pipeThrough(transform).pipeTo(streams.writable); - - // @ts-ignore - receiver[PACKET_TRAILER_FLAG] = true; - return true; - } - dispose() { this.metadataMap.clear(); this.insertionOrder.length = 0; diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 6b620a5045..acfa70a216 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -48,8 +48,9 @@ export default class RemoteVideoTrack extends RemoteTrack { * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames * with their capture-time metadata. * - * Requires the room to be configured with `packetTrailer: { worker }` and the - * publishing track to have packet trailer features enabled. + * Requires the room to be configured with the `packetTrailer` option + * (ideally with a dedicated `worker` for performance) and the publishing + * track to have packet trailer features enabled. * */ lookupFrameMetadata({ From f120282a937598e4a09e803012f1389986949b2e Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 14:52:14 -0700 Subject: [PATCH 21/31] move packetTrailer files --- src/e2ee/packetTrailer.test.ts | 55 ----- src/e2ee/packetTrailer.ts | 198 ------------------ src/e2ee/worker/FrameCryptor.ts | 6 +- src/index.ts | 2 +- src/packetTrailer/PacketTrailerManager.ts | 2 +- src/packetTrailer/types.ts | 5 +- .../worker/packetTrailer.worker.ts | 2 +- src/room/track/PacketTrailerExtractor.ts | 2 +- src/room/track/RemoteVideoTrack.ts | 2 +- 9 files changed, 14 insertions(+), 260 deletions(-) delete mode 100644 src/e2ee/packetTrailer.test.ts delete mode 100644 src/e2ee/packetTrailer.ts diff --git a/src/e2ee/packetTrailer.test.ts b/src/e2ee/packetTrailer.test.ts deleted file mode 100644 index 29ee031a98..0000000000 --- a/src/e2ee/packetTrailer.test.ts +++ /dev/null @@ -1,55 +0,0 @@ -import { describe, expect, it } from 'vitest'; -import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; - -describe('packetTrailer', () => { - it('extracts user timestamp and frame id from packet trailer', () => { - const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42); - const extracted = extractPacketTrailer(trailer); - - expect(Array.from(extracted.data)).toEqual(Array.from(payload)); - expect(extracted.metadata).toEqual({ - userTimestamp: 1_744_249_600_123_456n, - frameId: 42, - }); - }); - - it('extracts timestamp-only trailer when frameId is 0', () => { - const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 0); - const extracted = extractPacketTrailer(trailer); - - expect(Array.from(extracted.data)).toEqual(Array.from(payload)); - expect(extracted.metadata).toEqual({ - userTimestamp: 1_744_249_600_123_456n, - frameId: 0, - }); - }); - - it('extracts frameId-only trailer when timestamp is 0', () => { - const payload = Uint8Array.from([1, 2, 3, 4]); - const trailer = appendPacketTrailer(payload, 0n, 42); - const extracted = extractPacketTrailer(trailer); - - expect(Array.from(extracted.data)).toEqual(Array.from(payload)); - expect(extracted.metadata).toEqual({ - userTimestamp: 0n, - frameId: 42, - }); - }); - - it('returns data unchanged when both timestamp and frameId are 0', () => { - const payload = Uint8Array.from([1, 2, 3, 4]); - const result = appendPacketTrailer(payload, 0n, 0); - - expect(Array.from(result)).toEqual(Array.from(payload)); - }); - - it('passes frames through when there is no valid trailer', () => { - const payload = Uint8Array.from([1, 2, 3, 4, 5]); - const extracted = extractPacketTrailer(payload); - - expect(Array.from(extracted.data)).toEqual(Array.from(payload)); - expect(extracted.metadata).toBeUndefined(); - }); -}); diff --git a/src/e2ee/packetTrailer.ts b/src/e2ee/packetTrailer.ts deleted file mode 100644 index fce18a0196..0000000000 --- a/src/e2ee/packetTrailer.ts +++ /dev/null @@ -1,198 +0,0 @@ -export const PACKET_TRAILER_MAGIC = Uint8Array.from([ - 'L'.charCodeAt(0), - 'K'.charCodeAt(0), - 'T'.charCodeAt(0), - 'S'.charCodeAt(0), -]); - -export const PACKET_TRAILER_TIMESTAMP_TAG = 0x01; -export const PACKET_TRAILER_FRAME_ID_TAG = 0x02; -export const PACKET_TRAILER_ENVELOPE_SIZE = 5; - -const TIMESTAMP_TLV_SIZE = 10; -const FRAME_ID_TLV_SIZE = 6; - -export interface PacketTrailerMetadata { - userTimestamp: bigint; - frameId: number; -} - -export interface ExtractPacketTrailerResult { - data: Uint8Array; - metadata?: PacketTrailerMetadata; -} - -export function appendPacketTrailer( - data: Uint8Array, - userTimestamp: bigint, - frameId: number, -): Uint8Array { - const hasTimestamp = userTimestamp !== BigInt(0); - const hasFrameId = frameId !== 0; - - if (!hasTimestamp && !hasFrameId) { - return data; - } - - const trailerLength = - (hasTimestamp ? TIMESTAMP_TLV_SIZE : 0) + - (hasFrameId ? FRAME_ID_TLV_SIZE : 0) + - PACKET_TRAILER_ENVELOPE_SIZE; - const result = new Uint8Array(data.length + trailerLength); - let offset = 0; - - result.set(data, offset); - offset += data.length; - - if (hasTimestamp) { - result[offset++] = PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff; - result[offset++] = 8 ^ 0xff; - writeUint64Xor(result, offset, userTimestamp); - offset += 8; - } - - if (hasFrameId) { - result[offset++] = PACKET_TRAILER_FRAME_ID_TAG ^ 0xff; - result[offset++] = 4 ^ 0xff; - writeUint32Xor(result, offset, frameId); - offset += 4; - } - - result[offset++] = trailerLength ^ 0xff; - result.set(PACKET_TRAILER_MAGIC, offset); - - return result; -} - -export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPacketTrailerResult { - const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); - if (bytes.length < PACKET_TRAILER_ENVELOPE_SIZE) { - return { data: bytes }; - } - - const magicOffset = bytes.length - PACKET_TRAILER_MAGIC.length; - if (!matchesMagic(bytes, magicOffset)) { - return { data: bytes }; - } - - const trailerLength = bytes[bytes.length - PACKET_TRAILER_ENVELOPE_SIZE] ^ 0xff; - if (trailerLength < PACKET_TRAILER_ENVELOPE_SIZE || trailerLength > bytes.length) { - return { data: bytes }; - } - - const trailerStart = bytes.length - trailerLength; - const trailerEnd = bytes.length - PACKET_TRAILER_ENVELOPE_SIZE; - const strippedData = bytes.subarray(0, trailerStart); - let offset = trailerStart; - let foundAny = false; - const metadata: PacketTrailerMetadata = { - userTimestamp: BigInt(0), - frameId: 0, - }; - - while (offset + 2 <= trailerEnd) { - const tag = bytes[offset++] ^ 0xff; - const length = bytes[offset++] ^ 0xff; - - if (offset + length > trailerEnd) { - break; - } - - if (tag === PACKET_TRAILER_TIMESTAMP_TAG && length === 8) { - metadata.userTimestamp = readUint64Xor(bytes, offset); - foundAny = true; - } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { - metadata.frameId = readUint32Xor(bytes, offset, length); - foundAny = true; - } - - offset += length; - } - - if (!foundAny) { - return { data: bytes }; - } - - return { data: strippedData, metadata }; -} - -function matchesMagic(data: Uint8Array, offset: number) { - for (let index = 0; index < PACKET_TRAILER_MAGIC.length; index += 1) { - if (data[offset + index] !== PACKET_TRAILER_MAGIC[index]) { - return false; - } - } - return true; -} - -function readUint64Xor(data: Uint8Array, offset: number): bigint { - const hi = BigInt( - (((data[offset] ^ 0xff) << 24) | - ((data[offset + 1] ^ 0xff) << 16) | - ((data[offset + 2] ^ 0xff) << 8) | - (data[offset + 3] ^ 0xff)) >>> - 0, - ); - const lo = BigInt( - (((data[offset + 4] ^ 0xff) << 24) | - ((data[offset + 5] ^ 0xff) << 16) | - ((data[offset + 6] ^ 0xff) << 8) | - (data[offset + 7] ^ 0xff)) >>> - 0, - ); - return (hi << BigInt(32)) | lo; -} - -function readUint32Xor(data: Uint8Array, offset: number, length: number) { - let value = 0; - for (let index = 0; index < length; index += 1) { - value = (value << 8) | (data[offset + index] ^ 0xff); - } - return value >>> 0; -} - -function writeUint64Xor(target: Uint8Array, offset: number, value: bigint) { - const hi = Number((value >> BigInt(32)) & BigInt(0xffffffff)); - const lo = Number(value & BigInt(0xffffffff)); - target[offset] = (hi >>> 24) ^ 0xff; - target[offset + 1] = ((hi >>> 16) & 0xff) ^ 0xff; - target[offset + 2] = ((hi >>> 8) & 0xff) ^ 0xff; - target[offset + 3] = (hi & 0xff) ^ 0xff; - target[offset + 4] = (lo >>> 24) ^ 0xff; - target[offset + 5] = ((lo >>> 16) & 0xff) ^ 0xff; - target[offset + 6] = ((lo >>> 8) & 0xff) ^ 0xff; - target[offset + 7] = (lo & 0xff) ^ 0xff; -} - -function writeUint32Xor(target: Uint8Array, offset: number, value: number) { - for (let index = 3; index >= 0; index -= 1) { - target[offset + (3 - index)] = ((value >> (index * 8)) & 0xff) ^ 0xff; - } -} - -export function getFrameRtpTimestamp( - frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, -): number | undefined { - try { - const metadata = frame.getMetadata() as Record; - if (typeof metadata.rtpTimestamp === 'number') { - return metadata.rtpTimestamp; - } - if (typeof metadata.timestamp === 'number') { - return metadata.timestamp; - } - } catch { - // getMetadata() might not be available - } - return undefined; -} - -export function getFrameSsrc(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): number { - try { - const metadata = frame.getMetadata() as Record; - if (typeof metadata.synchronizationSource === 'number') { - return metadata.synchronizationSource; - } - } catch {} - return 0; -} diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index ce1664bf4e..28aad76098 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -6,7 +6,11 @@ import type { VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { type CryptorCallbacks, CryptorEvent } from '../events'; -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../packetTrailer'; +import { + extractPacketTrailer, + getFrameRtpTimestamp, + getFrameSsrc, +} from '../../packetTrailer/packetTrailer'; import type { DecodeRatchetOptions, KeyProviderOptions, diff --git a/src/index.ts b/src/index.ts index 7ffa39fa4e..c7f374bc96 100644 --- a/src/index.ts +++ b/src/index.ts @@ -63,7 +63,7 @@ import { import { getBrowser } from './utils/browserParser'; export { RpcError, type RpcInvocationData, type PerformRpcParams } from './room/rpc'; -export type { PacketTrailerMetadata } from './e2ee/packetTrailer'; +export type { PacketTrailerMetadata } from './packetTrailer/types'; export { PacketTrailerManager, type PacketTrailerOptions, diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 3f37e91552..4dae2b8305 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -1,5 +1,5 @@ import type { TrackInfo } from '@livekit/protocol'; -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../e2ee/packetTrailer'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from './packetTrailer'; import log from '../logger'; import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts index 63fd5910e3..4fa0394273 100644 --- a/src/packetTrailer/types.ts +++ b/src/packetTrailer/types.ts @@ -1,4 +1,7 @@ -import type { PacketTrailerMetadata } from '../e2ee/packetTrailer'; +export interface PacketTrailerMetadata { + userTimestamp: bigint; + frameId: number; +} export interface PTBaseMessage { kind: string; diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index aee1771840..1fcabaf860 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -1,4 +1,4 @@ -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../../e2ee/packetTrailer'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../packetTrailer'; import type { PTMetadataMessage, PTWorkerMessage } from '../types'; /** diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index a24944c0f0..f9de6b2cbf 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -1,4 +1,4 @@ -import type { PacketTrailerMetadata } from '../../e2ee/packetTrailer'; +import type { PacketTrailerMetadata } from '../../packetTrailer/types'; const MAX_ENTRIES = 300; diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index acfa70a216..6eea5726fa 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -1,4 +1,4 @@ -import type { PacketTrailerMetadata } from '../../e2ee/packetTrailer'; +import type { PacketTrailerMetadata } from '../../packetTrailer/types'; import { debounce } from '../debounce'; import { TrackEvent } from '../events'; import type { VideoReceiverStats } from '../stats'; From b730f813697b9c9f26a1a1ae5893343a1cc2618e Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 16:20:46 -0700 Subject: [PATCH 22/31] refactor parsing into helper, clean up framecryptor --- src/e2ee/E2eeManager.ts | 8 +++ src/e2ee/types.ts | 23 ++++---- src/e2ee/worker/FrameCryptor.ts | 54 ++++++++++--------- src/e2ee/worker/e2ee.worker.ts | 4 +- src/packetTrailer/types.ts | 9 ++-- .../worker/packetTrailer.worker.ts | 29 +++------- 6 files changed, 64 insertions(+), 63 deletions(-) diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index d4db0ec455..b378101a0d 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -479,11 +479,16 @@ export class E2EEManager if (!trackInfo?.mimeType || trackInfo.mimeType === '') { throw new TypeError('MimeType missing from trackInfo, cannot set up E2EE cryptor'); } + const hasPacketTrailer = + track.kind === 'video' && + !!trackInfo.packetTrailerFeatures && + trackInfo.packetTrailerFeatures.length > 0; this.handleReceiver( track.receiver, track.mediaStreamID, remoteId, track.kind === 'video' ? mimeTypeToVideoCodecString(trackInfo.mimeType) : undefined, + hasPacketTrailer, ); } @@ -505,6 +510,7 @@ export class E2EEManager trackId: string, participantIdentity: string, codec?: VideoCodec, + hasPacketTrailer?: boolean, ) { if (!this.worker) { return; @@ -521,6 +527,7 @@ export class E2EEManager participantIdentity, trackId, codec, + hasPacketTrailer, }; // @ts-ignore receiver.transform = new RTCRtpScriptTransform(this.worker, options); @@ -563,6 +570,7 @@ export class E2EEManager codec, participantIdentity: participantIdentity, isReuse: E2EE_FLAG in receiver, + hasPacketTrailer, }, }; this.worker.postMessage(msg, [readable, writable]); diff --git a/src/e2ee/types.ts b/src/e2ee/types.ts index b36727ff3c..a3ff0b712d 100644 --- a/src/e2ee/types.ts +++ b/src/e2ee/types.ts @@ -1,4 +1,5 @@ import type { LogLevel } from '../logger'; +import type { PacketTrailerFramePayload } from '../packetTrailer/packetTrailer'; import type { VideoCodec } from '../room/track/options'; import type { BaseE2EEManager } from './E2eeManager'; import type { BaseKeyProvider } from './KeyProvider'; @@ -51,6 +52,12 @@ export interface EncodeMessage extends BaseMessage { trackId: string; codec?: VideoCodec; isReuse: boolean; + /** + * Whether the published track advertises packet trailer features. + * When false, the cryptor skips the per-frame trailer extraction path + * entirely on decode. + */ + hasPacketTrailer?: boolean; }; } @@ -152,15 +159,7 @@ export interface EncryptDataResponseMessage extends BaseMessage { export interface PTMetadataFromE2EEMessage extends BaseMessage { kind: 'packetTrailerMetadata'; - data: { - trackId: string; - rtpTimestamp: number; - ssrc: number; - metadata: { - userTimestamp: bigint; - frameId: number; - }; - }; + data: PacketTrailerFramePayload; } export type E2EEWorkerMessage = @@ -235,4 +234,10 @@ export type ScriptTransformOptions = { participantIdentity: string; trackId: string; codec?: VideoCodec; + /** + * Whether the published track advertises packet trailer features. + * When false, the cryptor skips the per-frame trailer extraction path + * entirely on decode. + */ + hasPacketTrailer?: boolean; }; diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index 28aad76098..af4089919c 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -6,11 +6,7 @@ import type { VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { type CryptorCallbacks, CryptorEvent } from '../events'; -import { - extractPacketTrailer, - getFrameRtpTimestamp, - getFrameSsrc, -} from '../../packetTrailer/packetTrailer'; +import { processPacketTrailer } from '../../packetTrailer/packetTrailer'; import type { DecodeRatchetOptions, KeyProviderOptions, @@ -81,6 +77,13 @@ export class FrameCryptor extends BaseFrameCryptor { private currentTransform?: TransformerInfo; + /** + * Whether the subscribed track advertises packet trailer features. + * When false, we skip the per-frame trailer extraction path entirely + * on decode to avoid unnecessary work on tracks that don't use it. + */ + private hasPacketTrailer: boolean = false; + /** * Throttling mechanism for decryption errors to prevent memory leaks */ @@ -188,6 +191,15 @@ export class FrameCryptor extends BaseFrameCryptor { this.rtpMap = map; } + /** + * Sets whether the track associated with this cryptor carries packet + * trailer data. When false, {@link decodeFunction} skips the per-frame + * trailer extraction branch entirely. + */ + setHasPacketTrailer(hasPacketTrailer: boolean) { + this.hasPacketTrailer = hasPacketTrailer; + } + setupTransform( operation: 'encode' | 'decode', readable: ReadableStream, @@ -464,28 +476,18 @@ export class FrameCryptor extends BaseFrameCryptor { encodedFrame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, controller: TransformStreamDefaultController, ) { - if (isVideoFrame(encodedFrame) && encodedFrame.data.byteLength > 0) { + if (this.hasPacketTrailer && isVideoFrame(encodedFrame)) { try { - const ptResult = extractPacketTrailer(encodedFrame.data); - if (ptResult.metadata) { - const rtpTimestamp = getFrameRtpTimestamp(encodedFrame); - const ssrc = getFrameSsrc(encodedFrame); - if (rtpTimestamp !== undefined && this.trackId && this.participantIdentity) { - const msg: PTMetadataFromE2EEMessage = { - kind: 'packetTrailerMetadata', - data: { - trackId: this.trackId, - rtpTimestamp, - ssrc, - metadata: ptResult.metadata, - }, - }; - postMessage(msg); - } - encodedFrame.data = (ptResult.data.buffer as ArrayBuffer).slice( - ptResult.data.byteOffset, - ptResult.data.byteOffset + ptResult.data.byteLength, - ); + const ptResult = processPacketTrailer(encodedFrame, this.trackId); + if (ptResult.data) { + encodedFrame.data = ptResult.data; + } + if (ptResult.payload && this.participantIdentity) { + const msg: PTMetadataFromE2EEMessage = { + kind: 'packetTrailerMetadata', + data: ptResult.payload, + }; + postMessage(msg); } } catch { // best-effort: never break the media pipeline if trailer parsing fails diff --git a/src/e2ee/worker/e2ee.worker.ts b/src/e2ee/worker/e2ee.worker.ts index f9c642576c..b4529a2853 100644 --- a/src/e2ee/worker/e2ee.worker.ts +++ b/src/e2ee/worker/e2ee.worker.ts @@ -64,6 +64,7 @@ onmessage = (ev) => { break; case 'decode': let cryptor = getTrackCryptor(data.participantIdentity, data.trackId); + cryptor.setHasPacketTrailer(!!data.hasPacketTrailer); cryptor.setupTransform( kind, data.readableStream, @@ -333,10 +334,11 @@ if (self.RTCTransformEvent) { self.onrtctransform = (event: RTCTransformEvent) => { // @ts-ignore const transformer = event.transformer; - const { kind, participantIdentity, trackId, codec } = + const { kind, participantIdentity, trackId, codec, hasPacketTrailer } = transformer.options as ScriptTransformOptions; messageQueue.run(async () => { const cryptor = getTrackCryptor(participantIdentity, trackId); + cryptor.setHasPacketTrailer(!!hasPacketTrailer); workerLogger.debug('onrtctransform setup', { participantIdentity, trackId, codec }); cryptor.setupTransform( kind, diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts index 4fa0394273..ed71063fe2 100644 --- a/src/packetTrailer/types.ts +++ b/src/packetTrailer/types.ts @@ -1,3 +1,5 @@ +import type { PacketTrailerFramePayload } from './packetTrailer'; + export interface PacketTrailerMetadata { userTimestamp: bigint; frameId: number; @@ -27,12 +29,7 @@ export interface PTDecodeMessage extends PTBaseMessage { export interface PTMetadataMessage extends PTBaseMessage { kind: 'metadata'; - data: { - trackId: string; - rtpTimestamp: number; - ssrc: number; - metadata: PacketTrailerMetadata; - }; + data: PacketTrailerFramePayload; } export interface PTUpdateTrackIdMessage extends PTBaseMessage { diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 1fcabaf860..4bae7aaa0e 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -1,4 +1,4 @@ -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from '../packetTrailer'; +import { processPacketTrailer } from '../packetTrailer'; import type { PTMetadataMessage, PTWorkerMessage } from '../types'; /** @@ -43,26 +43,13 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream controller: TransformStreamDefaultController, ) { try { - const result = extractPacketTrailer(frame.data); - if (result.metadata) { - const rtpTimestamp = getFrameRtpTimestamp(frame); - const ssrc = getFrameSsrc(frame); - if (rtpTimestamp !== undefined) { - const msg: PTMetadataMessage = { - kind: 'metadata', - data: { - trackId: state.trackId, - rtpTimestamp, - ssrc, - metadata: result.metadata, - }, - }; - postMessage(msg); - } - frame.data = result.data.buffer.slice( - result.data.byteOffset, - result.data.byteOffset + result.data.byteLength, - ); + const result = processPacketTrailer(frame, state.trackId); + if (result.data) { + frame.data = result.data; + } + if (result.payload) { + const msg: PTMetadataMessage = { kind: 'metadata', data: result.payload }; + postMessage(msg); } } catch { // Never drop frames on trailer-extraction failure — pass through so From e0d173cbb390abf4cf9586440f67437aaea490a0 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 16:24:55 -0700 Subject: [PATCH 23/31] add the moved files --- src/packetTrailer/packetTrailer.test.ts | 55 ++++++ src/packetTrailer/packetTrailer.ts | 247 ++++++++++++++++++++++++ 2 files changed, 302 insertions(+) create mode 100644 src/packetTrailer/packetTrailer.test.ts create mode 100644 src/packetTrailer/packetTrailer.ts diff --git a/src/packetTrailer/packetTrailer.test.ts b/src/packetTrailer/packetTrailer.test.ts new file mode 100644 index 0000000000..29ee031a98 --- /dev/null +++ b/src/packetTrailer/packetTrailer.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from 'vitest'; +import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; + +describe('packetTrailer', () => { + it('extracts user timestamp and frame id from packet trailer', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestamp: 1_744_249_600_123_456n, + frameId: 42, + }); + }); + + it('extracts timestamp-only trailer when frameId is 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 0); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestamp: 1_744_249_600_123_456n, + frameId: 0, + }); + }); + + it('extracts frameId-only trailer when timestamp is 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 0n, 42); + const extracted = extractPacketTrailer(trailer); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toEqual({ + userTimestamp: 0n, + frameId: 42, + }); + }); + + it('returns data unchanged when both timestamp and frameId are 0', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const result = appendPacketTrailer(payload, 0n, 0); + + expect(Array.from(result)).toEqual(Array.from(payload)); + }); + + it('passes frames through when there is no valid trailer', () => { + const payload = Uint8Array.from([1, 2, 3, 4, 5]); + const extracted = extractPacketTrailer(payload); + + expect(Array.from(extracted.data)).toEqual(Array.from(payload)); + expect(extracted.metadata).toBeUndefined(); + }); +}); diff --git a/src/packetTrailer/packetTrailer.ts b/src/packetTrailer/packetTrailer.ts new file mode 100644 index 0000000000..db430e9f70 --- /dev/null +++ b/src/packetTrailer/packetTrailer.ts @@ -0,0 +1,247 @@ +import type { PacketTrailerMetadata } from './types'; + +export const PACKET_TRAILER_MAGIC = Uint8Array.from([ + 'L'.charCodeAt(0), + 'K'.charCodeAt(0), + 'T'.charCodeAt(0), + 'S'.charCodeAt(0), +]); + +export const PACKET_TRAILER_TIMESTAMP_TAG = 0x01; +export const PACKET_TRAILER_FRAME_ID_TAG = 0x02; +export const PACKET_TRAILER_ENVELOPE_SIZE = 5; + +const TIMESTAMP_TLV_SIZE = 10; +const FRAME_ID_TLV_SIZE = 6; + +export interface ExtractPacketTrailerResult { + data: Uint8Array; + metadata?: PacketTrailerMetadata; +} + +export function appendPacketTrailer( + data: Uint8Array, + userTimestamp: bigint, + frameId: number, +): Uint8Array { + const hasTimestamp = userTimestamp !== BigInt(0); + const hasFrameId = frameId !== 0; + + if (!hasTimestamp && !hasFrameId) { + return data; + } + + const trailerLength = + (hasTimestamp ? TIMESTAMP_TLV_SIZE : 0) + + (hasFrameId ? FRAME_ID_TLV_SIZE : 0) + + PACKET_TRAILER_ENVELOPE_SIZE; + const result = new Uint8Array(data.length + trailerLength); + let offset = 0; + + result.set(data, offset); + offset += data.length; + + if (hasTimestamp) { + result[offset++] = PACKET_TRAILER_TIMESTAMP_TAG ^ 0xff; + result[offset++] = 8 ^ 0xff; + writeUint64Xor(result, offset, userTimestamp); + offset += 8; + } + + if (hasFrameId) { + result[offset++] = PACKET_TRAILER_FRAME_ID_TAG ^ 0xff; + result[offset++] = 4 ^ 0xff; + writeUint32Xor(result, offset, frameId); + offset += 4; + } + + result[offset++] = trailerLength ^ 0xff; + result.set(PACKET_TRAILER_MAGIC, offset); + + return result; +} + +export function extractPacketTrailer(data: ArrayBuffer | Uint8Array): ExtractPacketTrailerResult { + const bytes = data instanceof Uint8Array ? data : new Uint8Array(data); + if (bytes.length < PACKET_TRAILER_ENVELOPE_SIZE) { + return { data: bytes }; + } + + const magicOffset = bytes.length - PACKET_TRAILER_MAGIC.length; + if (!matchesMagic(bytes, magicOffset)) { + return { data: bytes }; + } + + const trailerLength = bytes[bytes.length - PACKET_TRAILER_ENVELOPE_SIZE] ^ 0xff; + if (trailerLength < PACKET_TRAILER_ENVELOPE_SIZE || trailerLength > bytes.length) { + return { data: bytes }; + } + + const trailerStart = bytes.length - trailerLength; + const trailerEnd = bytes.length - PACKET_TRAILER_ENVELOPE_SIZE; + const strippedData = bytes.subarray(0, trailerStart); + let offset = trailerStart; + let foundAny = false; + const metadata: PacketTrailerMetadata = { + userTimestamp: BigInt(0), + frameId: 0, + }; + + while (offset + 2 <= trailerEnd) { + const tag = bytes[offset++] ^ 0xff; + const length = bytes[offset++] ^ 0xff; + + if (offset + length > trailerEnd) { + break; + } + + if (tag === PACKET_TRAILER_TIMESTAMP_TAG && length === 8) { + metadata.userTimestamp = readUint64Xor(bytes, offset); + foundAny = true; + } else if (tag === PACKET_TRAILER_FRAME_ID_TAG && length === 4) { + metadata.frameId = readUint32Xor(bytes, offset, length); + foundAny = true; + } + + offset += length; + } + + if (!foundAny) { + return { data: bytes }; + } + + return { data: strippedData, metadata }; +} + +function matchesMagic(data: Uint8Array, offset: number) { + for (let index = 0; index < PACKET_TRAILER_MAGIC.length; index += 1) { + if (data[offset + index] !== PACKET_TRAILER_MAGIC[index]) { + return false; + } + } + return true; +} + +function readUint64Xor(data: Uint8Array, offset: number): bigint { + const hi = BigInt( + (((data[offset] ^ 0xff) << 24) | + ((data[offset + 1] ^ 0xff) << 16) | + ((data[offset + 2] ^ 0xff) << 8) | + (data[offset + 3] ^ 0xff)) >>> + 0, + ); + const lo = BigInt( + (((data[offset + 4] ^ 0xff) << 24) | + ((data[offset + 5] ^ 0xff) << 16) | + ((data[offset + 6] ^ 0xff) << 8) | + (data[offset + 7] ^ 0xff)) >>> + 0, + ); + return (hi << BigInt(32)) | lo; +} + +function readUint32Xor(data: Uint8Array, offset: number, length: number) { + let value = 0; + for (let index = 0; index < length; index += 1) { + value = (value << 8) | (data[offset + index] ^ 0xff); + } + return value >>> 0; +} + +function writeUint64Xor(target: Uint8Array, offset: number, value: bigint) { + const hi = Number((value >> BigInt(32)) & BigInt(0xffffffff)); + const lo = Number(value & BigInt(0xffffffff)); + target[offset] = (hi >>> 24) ^ 0xff; + target[offset + 1] = ((hi >>> 16) & 0xff) ^ 0xff; + target[offset + 2] = ((hi >>> 8) & 0xff) ^ 0xff; + target[offset + 3] = (hi & 0xff) ^ 0xff; + target[offset + 4] = (lo >>> 24) ^ 0xff; + target[offset + 5] = ((lo >>> 16) & 0xff) ^ 0xff; + target[offset + 6] = ((lo >>> 8) & 0xff) ^ 0xff; + target[offset + 7] = (lo & 0xff) ^ 0xff; +} + +function writeUint32Xor(target: Uint8Array, offset: number, value: number) { + for (let index = 3; index >= 0; index -= 1) { + target[offset + (3 - index)] = ((value >> (index * 8)) & 0xff) ^ 0xff; + } +} + +export function getFrameRtpTimestamp( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, +): number | undefined { + try { + const metadata = frame.getMetadata() as Record; + if (typeof metadata.rtpTimestamp === 'number') { + return metadata.rtpTimestamp; + } + if (typeof metadata.timestamp === 'number') { + return metadata.timestamp; + } + } catch { + // getMetadata() might not be available + } + return undefined; +} + +export function getFrameSsrc(frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame): number { + try { + const metadata = frame.getMetadata() as Record; + if (typeof metadata.synchronizationSource === 'number') { + return metadata.synchronizationSource; + } + } catch {} + return 0; +} + +export interface PacketTrailerFramePayload { + trackId: string; + rtpTimestamp: number; + ssrc: number; + metadata: PacketTrailerMetadata; +} + +export interface ProcessPacketTrailerResult { + data?: ArrayBuffer; + payload?: PacketTrailerFramePayload; +} + +/** + * Extracts a packet trailer from an encoded frame and returns the stripped + * frame data (if any) along with a ready-to-post metadata payload. Returns an + * empty object when no trailer is present, an RTP timestamp can't be read, or + * a trackId isn't available. + */ +export function processPacketTrailer( + frame: RTCEncodedVideoFrame | RTCEncodedAudioFrame, + trackId: string | undefined, +): ProcessPacketTrailerResult { + if (frame.data.byteLength === 0) { + return {}; + } + + const result = extractPacketTrailer(frame.data); + if (!result.metadata) { + return {}; + } + + const strippedData = (result.data.buffer as ArrayBuffer).slice( + result.data.byteOffset, + result.data.byteOffset + result.data.byteLength, + ); + + const rtpTimestamp = getFrameRtpTimestamp(frame); + if (rtpTimestamp === undefined || !trackId) { + return { data: strippedData }; + } + + return { + data: strippedData, + payload: { + trackId, + rtpTimestamp, + ssrc: getFrameSsrc(frame), + metadata: result.metadata, + }, + }; +} From 6db94bd4d2c9355f48a72864fae8326107394e31 Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 16:26:37 -0700 Subject: [PATCH 24/31] revert unintended change --- src/api/WebSocketStream.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/src/api/WebSocketStream.test.ts b/src/api/WebSocketStream.test.ts index 08de033427..3445348042 100644 --- a/src/api/WebSocketStream.test.ts +++ b/src/api/WebSocketStream.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { WebSocketStream } from './WebSocketStream'; From 49942ef65a5345a92ea8ab86dee73e00ff56a03e Mon Sep 17 00:00:00 2001 From: David Chen Date: Thu, 16 Apr 2026 16:34:02 -0700 Subject: [PATCH 25/31] lint --- src/api/WebSocketStream.test.ts | 1 - src/e2ee/worker/FrameCryptor.ts | 2 +- src/packetTrailer/PacketTrailerManager.ts | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/api/WebSocketStream.test.ts b/src/api/WebSocketStream.test.ts index 3445348042..08de033427 100644 --- a/src/api/WebSocketStream.test.ts +++ b/src/api/WebSocketStream.test.ts @@ -1,4 +1,3 @@ -/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { WebSocketStream } from './WebSocketStream'; diff --git a/src/e2ee/worker/FrameCryptor.ts b/src/e2ee/worker/FrameCryptor.ts index af4089919c..00858f3265 100644 --- a/src/e2ee/worker/FrameCryptor.ts +++ b/src/e2ee/worker/FrameCryptor.ts @@ -2,11 +2,11 @@ import { EventEmitter } from 'events'; import type TypedEventEmitter from 'typed-emitter'; import { workerLogger } from '../../logger'; +import { processPacketTrailer } from '../../packetTrailer/packetTrailer'; import type { VideoCodec } from '../../room/track/options'; import { ENCRYPTION_ALGORITHM, IV_LENGTH, UNENCRYPTED_BYTES } from '../constants'; import { CryptorError, CryptorErrorReason } from '../errors'; import { type CryptorCallbacks, CryptorEvent } from '../events'; -import { processPacketTrailer } from '../../packetTrailer/packetTrailer'; import type { DecodeRatchetOptions, KeyProviderOptions, diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 4dae2b8305..4361637ee6 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -1,11 +1,11 @@ import type { TrackInfo } from '@livekit/protocol'; -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from './packetTrailer'; import log from '../logger'; import type Room from '../room/Room'; import { RoomEvent } from '../room/events'; import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; +import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from './packetTrailer'; import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; export interface PacketTrailerOptions { From ae7c8f555bc327c77a358069b3d5e4f098d08bd3 Mon Sep 17 00:00:00 2001 From: David Chen Date: Mon, 27 Apr 2026 19:35:41 -0700 Subject: [PATCH 26/31] update packet trailer to utilize client capabilities --- package.json | 2 +- pnpm-lock.yaml | 10 +- src/api/SignalClient.test.ts | 112 +++++++++++++++-- src/api/SignalClient.ts | 4 +- src/packetTrailer/PacketTrailerManager.ts | 147 ++++------------------ src/room/Room.ts | 9 +- src/room/track/PacketTrailerExtractor.ts | 2 +- src/room/track/RemoteVideoTrack.ts | 5 +- src/room/utils.test.ts | 15 ++- src/room/utils.ts | 4 +- 10 files changed, 160 insertions(+), 150 deletions(-) diff --git a/package.json b/package.json index 0bc2ebd95a..7fb6a1c3ab 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,7 @@ }, "dependencies": { "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.45.3", + "@livekit/protocol": "1.45.6", "events": "^3.3.0", "jose": "^6.1.0", "loglevel": "^1.9.2", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 74ec405a1d..cad6c445da 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -12,8 +12,8 @@ importers: specifier: 1.1.1 version: 1.1.1 '@livekit/protocol': - specifier: 1.45.3 - version: 1.45.3 + specifier: 1.45.6 + version: 1.45.6 '@types/dom-mediacapture-record': specifier: ^1 version: 1.0.22 @@ -1128,8 +1128,8 @@ packages: '@livekit/mutex@1.1.1': resolution: {integrity: sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==} - '@livekit/protocol@1.45.3': - resolution: {integrity: sha512-WmMxBTsy4dRBqcrswFwUUlgq3Z0nnhOqKR6tX749Rb/PcB1yBMUtrHxZvcsS6qi3/5+86zHeVG+exmu1sZqfJg==} + '@livekit/protocol@1.45.6': + resolution: {integrity: sha512-YPDmrUiVe1EY/q/2bD+Fp+69DWq6LZgeH+G/KEbz07OIVf8hgAYzfb1FgiOdWLRpSj06+SuTmrOY604fWNuD3w==} '@livekit/throws-transformer@0.1.3': resolution: {integrity: sha512-PBttE6W6g/2ALGu6kWOunZ5qdrXwP9Ge1An2/62OfE6Rhc0Abd4yp6ex2pWhwUfGxDsSZvFgoB1Ia/5mWAMuKQ==} @@ -5300,7 +5300,7 @@ snapshots: '@livekit/mutex@1.1.1': {} - '@livekit/protocol@1.45.3': + '@livekit/protocol@1.45.6': dependencies: '@bufbuild/protobuf': 1.10.1 diff --git a/src/api/SignalClient.test.ts b/src/api/SignalClient.test.ts index bffa5b8268..968c7c92b1 100644 --- a/src/api/SignalClient.test.ts +++ b/src/api/SignalClient.test.ts @@ -1,10 +1,14 @@ import { + ClientInfo_Capability, DisconnectReason, + JoinRequest, JoinResponse, LeaveRequest, ReconnectResponse, SignalRequest, SignalResponse, + WrappedJoinRequest, + WrappedJoinRequest_Compression, } from '@livekit/protocol'; import { beforeEach, describe, expect, it, vi } from 'vitest'; import { ConnectionError, ConnectionErrorReason } from '../room/errors'; @@ -60,6 +64,7 @@ interface MockWebSocketStreamOptions { connection?: WebSocketConnection; opened?: Promise; closed?: Promise; + onUrl?: (url: string) => void; readyState?: number; } @@ -69,18 +74,19 @@ function mockWebSocketStream(options: MockWebSocketStreamOptions = {}) { opened = connection ? Promise.resolve(connection) : new Promise(() => {}), closed = new Promise(() => {}), readyState = 1, + onUrl, } = options; - return vi.mocked(WebSocketStream).mockImplementationOnce( - () => - ({ - url: 'wss://test.livekit.io', - opened, - closed, - close: vi.fn(), - readyState, - }) as any, - ); + return vi.mocked(WebSocketStream).mockImplementationOnce((url) => { + onUrl?.(url); + return { + url: 'wss://test.livekit.io', + opened, + closed, + close: vi.fn(), + readyState, + } as any; + }); } describe('SignalClient.connect', () => { @@ -99,6 +105,47 @@ describe('SignalClient.connect', () => { signalClient = new SignalClient(false); }); + async function decodeJoinRequestFromUrl(url: string): Promise { + const joinRequestParam = new URL(url).searchParams.get('join_request'); + expect(joinRequestParam).toBeTruthy(); + + const paddedBase64Url = joinRequestParam!.padEnd( + joinRequestParam!.length + ((4 - (joinRequestParam!.length % 4)) % 4), + '=', + ); + const binaryString = atob(paddedBase64Url.replace(/-/g, '+').replace(/_/g, '/')); + const wrappedBytes = new Uint8Array(binaryString.length); + for (let i = 0; i < binaryString.length; i += 1) { + wrappedBytes[i] = binaryString.charCodeAt(i); + } + + const wrappedJoinRequest = WrappedJoinRequest.fromBinary(wrappedBytes); + if (wrappedJoinRequest.compression === WrappedJoinRequest_Compression.NONE) { + return JoinRequest.fromBinary(wrappedJoinRequest.joinRequest); + } + + const stream = new DecompressionStream('gzip'); + const writer = stream.writable.getWriter(); + writer.write(wrappedJoinRequest.joinRequest); + writer.close(); + + const chunks: Uint8Array[] = []; + const reader = stream.readable.getReader(); + while (true) { + const { done, value } = await reader.read(); + if (done) break; + chunks.push(value); + } + const totalLength = chunks.reduce((acc, chunk) => acc + chunk.length, 0); + const bytes = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + bytes.set(chunk, offset); + offset += chunk.length; + } + return JoinRequest.fromBinary(bytes); + } + describe('Happy Path - Initial Join', () => { it('should successfully connect and receive join response', async () => { const joinResponse = createJoinResponse(); @@ -113,6 +160,51 @@ describe('SignalClient.connect', () => { expect(result).toEqual(joinResponse); expect(signalClient.currentState).toBe(SignalConnectionState.CONNECTED); }); + + it('does not advertise packet trailer capability by default', async () => { + const joinResponse = createJoinResponse(); + const signalResponse = createSignalResponse('join', joinResponse); + const mockReadable = createMockReadableStream([signalResponse]); + const mockConnection = createMockConnection(mockReadable); + let capturedUrl = ''; + + mockWebSocketStream({ + connection: mockConnection, + onUrl: (url) => { + capturedUrl = url; + }, + }); + + await signalClient.join('wss://test.livekit.io', 'test-token', defaultOptions); + + const joinRequest = await decodeJoinRequestFromUrl(capturedUrl); + expect(joinRequest.clientInfo?.capabilities).toEqual([]); + }); + + it('advertises packet trailer capability when provided', async () => { + const joinResponse = createJoinResponse(); + const signalResponse = createSignalResponse('join', joinResponse); + const mockReadable = createMockReadableStream([signalResponse]); + const mockConnection = createMockConnection(mockReadable); + let capturedUrl = ''; + + mockWebSocketStream({ + connection: mockConnection, + onUrl: (url) => { + capturedUrl = url; + }, + }); + + await signalClient.join('wss://test.livekit.io', 'test-token', { + ...defaultOptions, + clientInfoCapabilities: [ClientInfo_Capability.CAP_PACKET_TRAILER], + }); + + const joinRequest = await decodeJoinRequestFromUrl(capturedUrl); + expect(joinRequest.clientInfo?.capabilities).toEqual([ + ClientInfo_Capability.CAP_PACKET_TRAILER, + ]); + }); }); describe('Happy Path - Reconnect', () => { diff --git a/src/api/SignalClient.ts b/src/api/SignalClient.ts index 31ffa57832..39c427a6be 100644 --- a/src/api/SignalClient.ts +++ b/src/api/SignalClient.ts @@ -3,6 +3,7 @@ import { AddTrackRequest, AudioTrackFeature, ClientInfo, + ClientInfo_Capability, ConnectionQualityUpdate, ConnectionSettings, DataTrackSubscriberHandles, @@ -83,6 +84,7 @@ interface ConnectOpts extends SignalOptions { export interface SignalOptions { autoSubscribe: boolean; adaptiveStream?: boolean; + clientInfoCapabilities?: ClientInfo_Capability[]; maxRetries: number; e2eeEnabled: boolean; websocketTimeout: number; @@ -323,7 +325,7 @@ export class SignalClient { this.connectOptions = opts; this.useV0SignalPath = useV0Path; - const clientInfo = getClientInfo(); + const clientInfo = getClientInfo(opts.clientInfoCapabilities); const params = useV0Path ? createConnectionParams(token, clientInfo, opts) : await createJoinRequestConnectionParams(token, clientInfo, opts, publisherOffer); diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 4361637ee6..097b6539f0 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -5,31 +5,16 @@ import { RoomEvent } from '../room/events'; import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; -import { extractPacketTrailer, getFrameRtpTimestamp, getFrameSsrc } from './packetTrailer'; import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; export interface PacketTrailerOptions { /** - * Optional dedicated worker for extracting packet trailers off the main thread. + * Dedicated worker for extracting packet trailers off the main thread. * - * When provided, encoded video streams are transferred to the worker for - * processing, which avoids any per-frame work on the main thread and is the - * recommended configuration for production. - * - * When omitted, the manager falls back to an inline `TransformStream` on the - * main thread. This is a safety net so video still decodes correctly if the - * worker wasn't wired up but a publishing track advertises packet trailer - * features — at a small CPU cost on the subscriber. + * Encoded video streams are transferred to the worker for processing, which + * avoids per-frame work on the main thread. */ - worker?: Worker; -} - -/** @internal */ -const PACKET_TRAILER_FLAG = 'lk_pkt_trailer'; - -/** Mutable pipeline state so the transform closure can be remapped when a receiver is reused. */ -interface MainThreadPipelineState { - extractor: PacketTrailerExtractor; + worker: Worker; } /** @@ -39,12 +24,8 @@ interface MainThreadPipelineState { * wires up an Insertable Streams pipeline to strip the trailer from encoded * frames and cache the metadata for lookup. * - * Two processing modes are supported: - * - **Worker** (preferred): encoded streams are transferred to a dedicated - * worker. No per-frame work happens on the main thread. - * - **Main-thread fallback**: if no worker was supplied, the transform runs - * inline. Slightly higher main-thread cost, but ensures video still - * decodes correctly when the worker was not wired up. + * Packet trailer extraction is worker-only. If no worker is configured, the + * SDK does not advertise packet trailer support and skips extraction. * * When E2EE is active, the E2EE FrameCryptor worker handles trailer * extraction directly (before decryption), so this manager only creates @@ -67,19 +48,8 @@ export class PacketTrailerManager { */ private workerPipelines = new Map(); - /** - * Tracks the active main-thread pipeline state for each receiver. When a - * receiver is reused for a new track, the state's extractor is swapped - * in-place so the existing `TransformStream` keeps feeding the correct - * extractor without re-calling `createEncodedStreams` (which would throw). - */ - private mainThreadPipelines = new Map(); - - /** Ensures the "no worker, using main-thread fallback" warning only fires once per room. */ - private mainThreadFallbackWarned = false; - - constructor(options: PacketTrailerOptions = {}) { - this.worker = options.worker; + constructor(options?: PacketTrailerOptions) { + this.worker = options?.worker; } /** @internal */ @@ -125,6 +95,15 @@ export class PacketTrailerManager { return; } + if (!this.worker && !this.room?.hasE2EESetup) { + log.warn( + 'subscribed to a track with packet trailer features but no packet trailer worker ' + + 'is configured; skipping packet trailer extraction. Pass `packetTrailer: { worker }` ' + + 'in RoomOptions to enable packet trailer support.', + ); + return; + } + const extractor = new PacketTrailerExtractor(); const trackId = track.mediaStreamID; @@ -139,22 +118,10 @@ export class PacketTrailerManager { log.debug('PacketTrailerManager: installing pipeline', { trackSid: track.sid, - mode: this.worker ? 'worker' : 'main-thread', + mode: 'worker', }); - if (this.worker) { - this.setupWorkerReceiver(receiver, trackId); - } else { - if (!this.mainThreadFallbackWarned) { - log.warn( - 'subscribed to a track with packet trailer features but no packet trailer worker ' + - 'is configured — falling back to main-thread frame processing. For best performance ' + - 'pass `packetTrailer: { worker }` in RoomOptions.', - ); - this.mainThreadFallbackWarned = true; - } - this.setupMainThreadReceiver(receiver, extractor); - } + this.setupWorkerReceiver(receiver, trackId); } private setupWorkerReceiver(receiver: RTCRtpReceiver, newTrackId: string) { @@ -200,71 +167,6 @@ export class PacketTrailerManager { this.workerPipelines.set(receiver, newTrackId); } - private setupMainThreadReceiver(receiver: RTCRtpReceiver, extractor: PacketTrailerExtractor) { - const existingState = this.mainThreadPipelines.get(receiver); - if (existingState) { - // Receiver was reused for a new track. The pipeline is still running - // on the already-transferred encoded streams — just swap the extractor - // the transform feeds into. - existingState.extractor = extractor; - return; - } - - if (PACKET_TRAILER_FLAG in receiver) { - return; - } - if (!('createEncodedStreams' in receiver)) { - log.debug('createEncodedStreams not supported, packet trailer extraction unavailable'); - return; - } - - let streams: { readable: ReadableStream; writable: WritableStream }; - try { - // @ts-ignore — createEncodedStreams is not in standard typings - streams = receiver.createEncodedStreams(); - } catch (err) { - log.warn('failed to create encoded streams for packet trailer extraction', { error: err }); - return; - } - - const state: MainThreadPipelineState = { extractor }; - this.mainThreadPipelines.set(receiver, state); - - const transform = new TransformStream({ - transform: (frame, controller) => { - try { - const result = extractPacketTrailer(frame.data); - if (result.metadata) { - const rtpTimestamp = getFrameRtpTimestamp(frame); - const ssrc = getFrameSsrc(frame); - if (rtpTimestamp !== undefined) { - state.extractor.storeMetadata(rtpTimestamp, ssrc, result.metadata); - } - frame.data = result.data.buffer.slice( - result.data.byteOffset, - result.data.byteOffset + result.data.byteLength, - ) as ArrayBuffer; - } - } catch (err) { - // Never drop frames on trailer-extraction failure — pass through so - // video keeps decoding even if metadata is lost for this frame. - log.debug('packet trailer extraction failed, passing frame through', { error: err }); - } - controller.enqueue(frame); - }, - }); - - streams.readable - .pipeThrough(transform) - .pipeTo(streams.writable) - .catch((err) => { - log.debug('packet trailer pipeline ended', { error: err }); - }); - - // @ts-ignore - receiver[PACKET_TRAILER_FLAG] = true; - } - private teardownTrack(track: RemoteTrack) { const trackId = track.mediaStreamID; const extractor = this.extractors.get(trackId); @@ -277,11 +179,11 @@ export class PacketTrailerManager { track.packetTrailerExtractor = undefined; } - // The receiver pipeline (worker or main-thread) is intentionally left - // running. If the receiver is reused for a new track, `setupReceiver` will - // remap it. If the room disconnects, `cleanup` drops all state. Any - // metadata produced in the meantime is harmless — the extractor above has - // already been disposed and is no longer reachable from any track. + // The receiver pipeline is intentionally left running. If the receiver is + // reused for a new track, `setupReceiver` will remap it. If the room + // disconnects, `cleanup` drops all state. Any metadata produced in the + // meantime is harmless — the extractor above has already been disposed and + // is no longer reachable from any track. } private cleanup() { @@ -290,7 +192,6 @@ export class PacketTrailerManager { } this.extractors.clear(); this.workerPipelines.clear(); - this.mainThreadPipelines.clear(); this.worker?.terminate(); } diff --git a/src/room/Room.ts b/src/room/Room.ts index 9c0ab9a649..bbb9b65018 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -1,6 +1,7 @@ import { Mutex } from '@livekit/mutex'; import { ChatMessage as ChatMessageModel, + ClientInfo_Capability, ConnectionQualityUpdate, type DataPacket, DataPacket_Kind, @@ -470,10 +471,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) private setupPacketTrailer() { // The manager is always created so tracks that advertise packet trailer - // features always have their trailers stripped — even if the app didn't - // pass `packetTrailer` in RoomOptions. A worker is used when provided for - // best performance; otherwise the manager falls back to a main-thread - // TransformStream so video still decodes correctly. + // features can be wired up when the app passes a packet trailer worker. this.packetTrailerManager = new PacketTrailerManager(this.options.packetTrailer); this.packetTrailerManager.setup(this); } @@ -927,6 +925,9 @@ class Room extends (EventEmitter as new () => TypedEmitter) autoSubscribe: connectOptions.autoSubscribe, adaptiveStream: typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream, + clientInfoCapabilities: roomOptions.packetTrailer?.worker + ? [ClientInfo_Capability.CAP_PACKET_TRAILER] + : undefined, maxRetries: connectOptions.maxRetries, e2eeEnabled: !!this.e2eeManager, websocketTimeout: connectOptions.websocketTimeout, diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index f9de6b2cbf..db4d8363a7 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -6,7 +6,7 @@ const MAX_ENTRIES = 300; * Caches packet trailer metadata extracted from received video frames, * keyed by RTP timestamp so it can be looked up when the frame is displayed. * - * Metadata is populated either by the main-thread pipeline installed by + * Metadata is populated either by the packet trailer worker managed by * `PacketTrailerManager` (non-E2EE) or by the E2EE FrameCryptor worker * after decryption (E2EE). * diff --git a/src/room/track/RemoteVideoTrack.ts b/src/room/track/RemoteVideoTrack.ts index 6eea5726fa..54335161ee 100644 --- a/src/room/track/RemoteVideoTrack.ts +++ b/src/room/track/RemoteVideoTrack.ts @@ -48,9 +48,8 @@ export default class RemoteVideoTrack extends RemoteTrack { * Use with the `TrackEvent.TimeSyncUpdate` event to correlate displayed frames * with their capture-time metadata. * - * Requires the room to be configured with the `packetTrailer` option - * (ideally with a dedicated `worker` for performance) and the publishing - * track to have packet trailer features enabled. + * Requires the room to be configured with the `packetTrailer` worker option + * and the publishing track to have packet trailer features enabled. * */ lookupFrameMetadata({ diff --git a/src/room/utils.test.ts b/src/room/utils.test.ts index 7d404c2170..c7d099b3b3 100644 --- a/src/room/utils.test.ts +++ b/src/room/utils.test.ts @@ -1,5 +1,6 @@ +import { ClientInfo_Capability } from '@livekit/protocol'; import { describe, expect, it } from 'vitest'; -import { extractMaxAgeFromRequestHeaders, splitUtf8, toWebsocketUrl } from './utils'; +import { extractMaxAgeFromRequestHeaders, getClientInfo, splitUtf8, toWebsocketUrl } from './utils'; describe('toWebsocketUrl', () => { it('leaves wss urls alone', () => { @@ -15,6 +16,18 @@ describe('toWebsocketUrl', () => { }); }); +describe('getClientInfo', () => { + it('does not advertise packet trailer capability by default', () => { + expect(getClientInfo().capabilities).toEqual([]); + }); + + it('advertises packet trailer capability when provided', () => { + expect(getClientInfo([ClientInfo_Capability.CAP_PACKET_TRAILER]).capabilities).toEqual([ + ClientInfo_Capability.CAP_PACKET_TRAILER, + ]); + }); +}); + describe('splitUtf8', () => { it('splits a string into chunks of the given size', () => { expect(splitUtf8('hello world', 5)).toEqual([ diff --git a/src/room/utils.ts b/src/room/utils.ts index 49df9b6fca..d8e414c2f9 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -1,6 +1,7 @@ import { ChatMessage as ChatMessageModel, ClientInfo, + ClientInfo_Capability, ClientInfo_SDK, DisconnectReason, Transcription as TranscriptionModel, @@ -364,8 +365,9 @@ export interface ObservableMediaElement extends HTMLMediaElement { handleVisibilityChanged: (entry: IntersectionObserverEntry) => void; } -export function getClientInfo(): ClientInfo { +export function getClientInfo(capabilities?: ClientInfo_Capability[]): ClientInfo { const info = new ClientInfo({ + capabilities, sdk: ClientInfo_SDK.JS, protocol: protocolVersion, version, From 125464ff05338dc0e96211005ab0e48558ee8891 Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 28 Apr 2026 00:02:23 -0700 Subject: [PATCH 27/31] dont attach transform if worker is not initialized --- examples/demo/demo.ts | 2 +- src/room/RTCEngine.test.ts | 96 ++++++++++++++++++++++++++++++++++++++ src/room/RTCEngine.ts | 16 ++++--- 3 files changed, 106 insertions(+), 8 deletions(-) create mode 100644 src/room/RTCEngine.test.ts diff --git a/examples/demo/demo.ts b/examples/demo/demo.ts index e6d0b0b6ba..5c2c1b125a 100644 --- a/examples/demo/demo.ts +++ b/examples/demo/demo.ts @@ -267,7 +267,7 @@ const appActions = { } const fmt = (d: Date) => { const pad = (n: number, w = 2) => String(n).padStart(w, '0'); - return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}:${pad(d.getMilliseconds(), 4)}`; + return `${d.getFullYear()}-${pad(d.getMonth() + 1)}-${pad(d.getDate())} ${pad(d.getHours())}:${pad(d.getMinutes())}:${pad(d.getSeconds())}:${pad(d.getMilliseconds(), 3)}`; }; text += `\nPublish: ${fmt(publishTime)}` + diff --git a/src/room/RTCEngine.test.ts b/src/room/RTCEngine.test.ts new file mode 100644 index 0000000000..56faff5e57 --- /dev/null +++ b/src/room/RTCEngine.test.ts @@ -0,0 +1,96 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import RTCEngine from './RTCEngine'; +import { roomOptionDefaults } from './defaults'; + +describe('RTCEngine', () => { + const originalRTCRtpSender = window.RTCRtpSender; + + afterEach(() => { + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: originalRTCRtpSender, + writable: true, + }); + }); + + function stubInsertableStreamsSupport() { + class MockRTCRtpSender { + createEncodedStreams() {} + } + + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: MockRTCRtpSender, + writable: true, + }); + } + + function makeRTCConfiguration(engine: RTCEngine) { + return ( + engine as unknown as { makeRTCConfiguration: () => RTCConfiguration } + ).makeRTCConfiguration(); + } + + function setupSenderPassthrough(engine: RTCEngine, sender: RTCRtpSender) { + ( + engine as unknown as { setupSenderPassthrough: (sender: RTCRtpSender) => void } + ).setupSenderPassthrough(sender); + } + + it('does not enable encoded insertable streams without E2EE or a packet trailer worker', () => { + stubInsertableStreamsSupport(); + + const engine = new RTCEngine(roomOptionDefaults); + + expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined(); + }); + + it('enables encoded insertable streams when a packet trailer worker is configured', () => { + stubInsertableStreamsSupport(); + + const engine = new RTCEngine({ + ...roomOptionDefaults, + packetTrailer: { worker: {} as Worker }, + }); + + expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true); + }); + + it('enables encoded insertable streams for E2EE', () => { + stubInsertableStreamsSupport(); + + const engine = new RTCEngine(roomOptionDefaults); + ( + engine as unknown as { + signalOpts: { + autoSubscribe: boolean; + maxRetries: number; + e2eeEnabled: boolean; + websocketTimeout: number; + }; + } + ).signalOpts = { + autoSubscribe: true, + maxRetries: 1, + e2eeEnabled: true, + websocketTimeout: 15_000, + }; + + expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true); + }); + + it('does not create sender encoded streams when packetTrailer has no worker', () => { + const engine = new RTCEngine({ + ...roomOptionDefaults, + packetTrailer: {} as never, + }); + const createEncodedStreams = vi.fn(); + const sender = { + createEncodedStreams, + } as unknown as RTCRtpSender; + + setupSenderPassthrough(engine, sender); + + expect(createEncodedStreams).not.toHaveBeenCalled(); + }); +}); diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index dbc72874a1..ca0d742d5a 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -745,12 +745,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit ): RTCConfiguration { const rtcConfig = { ...this.rtcConfig }; - // Always enable encoded insertable streams when supported. E2EE and packet - // trailer both rely on it, and enabling the flag is a no-op when nothing - // calls `createEncodedStreams()`. Having it always on means subscribers - // can automatically handle publishers that advertise packet trailer - // features without any explicit RoomOptions configuration. - if (isInsertableStreamSupported()) { + // E2EE and packet trailer extraction both rely on encoded insertable + // streams. Only opt in when a transform will actually be installed; if no + // packet trailer worker is configured, packet trailer support is not + // advertised and the SFU strips trailers before forwarding media. + if ( + (this.signalOpts?.e2eeEnabled || this.options.packetTrailer?.worker) && + isInsertableStreamSupported() + ) { this.log.debug('setting up transports with insertable streams', this.logContext); // this makes sure that no data is sent before the transforms are ready // @ts-ignore @@ -1030,7 +1032,7 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } private setupSenderPassthrough(sender: RTCRtpSender) { - if (!this.options.packetTrailer || this.signalOpts?.e2eeEnabled) { + if (!this.options.packetTrailer?.worker || this.signalOpts?.e2eeEnabled) { return; } if ('createEncodedStreams' in sender) { From b124e145dc7d1fdd5b85b2d307fd06796feebcea Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 28 Apr 2026 10:11:50 -0700 Subject: [PATCH 28/31] cleanup --- .size-limit.cjs | 2 +- src/packetTrailer/PacketTrailerManager.ts | 15 +++--------- src/room/track/PacketTrailerExtractor.ts | 29 +++-------------------- 3 files changed, 7 insertions(+), 39 deletions(-) diff --git a/.size-limit.cjs b/.size-limit.cjs index c9f5ffe6dc..19c0ae8cf2 100644 --- a/.size-limit.cjs +++ b/.size-limit.cjs @@ -2,7 +2,7 @@ module.exports = [ { path: 'dist/livekit-client.esm.mjs', import: '{ Room }', - limit: '100 kB', + limit: '101 kB', }, { path: 'dist/livekit-client.umd.js', diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 097b6539f0..85767944b0 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -96,11 +96,7 @@ export class PacketTrailerManager { } if (!this.worker && !this.room?.hasE2EESetup) { - log.warn( - 'subscribed to a track with packet trailer features but no packet trailer worker ' + - 'is configured; skipping packet trailer extraction. Pass `packetTrailer: { worker }` ' + - 'in RoomOptions to enable packet trailer support.', - ); + log.warn('packet trailer worker not configured; skipping extraction'); return; } @@ -116,11 +112,6 @@ export class PacketTrailerManager { return; } - log.debug('PacketTrailerManager: installing pipeline', { - trackSid: track.sid, - mode: 'worker', - }); - this.setupWorkerReceiver(receiver, trackId); } @@ -142,7 +133,7 @@ export class PacketTrailerManager { } if (!('createEncodedStreams' in receiver)) { - log.warn('createEncodedStreams not supported, packet trailer extraction unavailable'); + log.warn('createEncodedStreams not supported'); return; } @@ -151,7 +142,7 @@ export class PacketTrailerManager { // @ts-ignore — createEncodedStreams is not in standard typings streams = receiver.createEncodedStreams(); } catch (err) { - log.warn('failed to create encoded streams for packet trailer extraction', { error: err }); + log.warn('failed to create encoded streams', { error: err }); return; } diff --git a/src/room/track/PacketTrailerExtractor.ts b/src/room/track/PacketTrailerExtractor.ts index db4d8363a7..fbd55d48ae 100644 --- a/src/room/track/PacketTrailerExtractor.ts +++ b/src/room/track/PacketTrailerExtractor.ts @@ -15,37 +15,20 @@ const MAX_ENTRIES = 300; export class PacketTrailerExtractor { private metadataMap = new Map(); - private insertionOrder: number[] = []; - private activeSsrc: number = 0; storeMetadata(rtpTimestamp: number, ssrc: number, metadata: PacketTrailerMetadata) { // Simulcast layer switch: SSRC changed, flush stale entries from old layer. if (this.activeSsrc !== 0 && this.activeSsrc !== ssrc) { - const keep: number[] = []; - for (const ts of this.insertionOrder) { - const m = this.metadataMap.get(ts); - if (!m || (m as PacketTrailerMetadataInternal).ssrc !== ssrc) { - this.metadataMap.delete(ts); - } else { - keep.push(ts); - } - } - this.insertionOrder = keep; + this.metadataMap.clear(); } this.activeSsrc = ssrc; - const collision = this.metadataMap.has(rtpTimestamp); - - while (this.metadataMap.size >= MAX_ENTRIES && this.insertionOrder.length > 0) { - const evicted = this.insertionOrder.shift()!; + while (this.metadataMap.size >= MAX_ENTRIES) { + const evicted = this.metadataMap.keys().next().value!; this.metadataMap.delete(evicted); } - if (!collision) { - this.insertionOrder.push(rtpTimestamp); - } - (metadata as PacketTrailerMetadataInternal).ssrc = ssrc; this.metadataMap.set(rtpTimestamp, metadata); } @@ -55,12 +38,6 @@ export class PacketTrailerExtractor { dispose() { this.metadataMap.clear(); - this.insertionOrder.length = 0; this.activeSsrc = 0; } } - -/** @internal */ -interface PacketTrailerMetadataInternal extends PacketTrailerMetadata { - ssrc?: number; -} From d791bc07051f3fb37288086151df96a33d78925b Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 28 Apr 2026 15:17:50 -0700 Subject: [PATCH 29/31] ensure safari works --- src/e2ee/E2eeManager.ts | 24 +++--- src/e2ee/utils.ts | 3 +- .../PacketTrailerManager.test.ts | 79 +++++++++++++++++++ src/packetTrailer/PacketTrailerManager.ts | 22 +++++- src/packetTrailer/packetTrailer.test.ts | 26 +++++- src/packetTrailer/packetTrailer.ts | 3 + src/packetTrailer/types.ts | 5 ++ src/packetTrailer/utils.test.ts | 67 ++++++++++++++++ src/packetTrailer/utils.ts | 13 +++ .../worker/packetTrailer.worker.ts | 14 +++- src/room/RTCEngine.test.ts | 54 +++++++++++++ src/room/RTCEngine.ts | 23 +++--- src/room/Room.ts | 3 +- src/room/utils.ts | 12 +++ 14 files changed, 317 insertions(+), 31 deletions(-) create mode 100644 src/packetTrailer/PacketTrailerManager.test.ts create mode 100644 src/packetTrailer/utils.test.ts create mode 100644 src/packetTrailer/utils.ts diff --git a/src/e2ee/E2eeManager.ts b/src/e2ee/E2eeManager.ts index b378101a0d..3be1ebab5d 100644 --- a/src/e2ee/E2eeManager.ts +++ b/src/e2ee/E2eeManager.ts @@ -12,7 +12,13 @@ import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { Track } from '../room/track/Track'; import type { VideoCodec } from '../room/track/options'; import { mimeTypeToVideoCodecString } from '../room/track/utils'; -import { Future, isChromiumBased, isLocalTrack, isSafariBased, isVideoTrack } from '../room/utils'; +import { + Future, + isLocalTrack, + isSafariBased, + isScriptTransformSupportedForWorker, + isVideoTrack, +} from '../room/utils'; import type { BaseKeyProvider } from './KeyProvider'; import { E2EE_FLAG } from './constants'; import { type E2EEManagerCallbacks, EncryptionEvent, KeyProviderEvent } from './events'; @@ -35,7 +41,7 @@ import type { SifTrailerMessage, UpdateCodecMessage, } from './types'; -import { isE2EESupported, isScriptTransformSupported } from './utils'; +import { isE2EESupported } from './utils'; export interface BaseE2EEManager { setup(room: Room): void; @@ -516,12 +522,7 @@ export class E2EEManager return; } - if ( - isScriptTransformSupported() && - // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141. - // Disabling it for Chrome based browsers until the API has stabilized - !isChromiumBased() - ) { + if (isScriptTransformSupportedForWorker()) { const options: ScriptTransformOptions = { kind: 'decode', participantIdentity, @@ -594,12 +595,7 @@ export class E2EEManager throw TypeError('local identity needs to be known in order to set up encrypted sender'); } - if ( - isScriptTransformSupported() && - // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141. - // Disabling it for Chrome based browsers until the API has stabilized - !isChromiumBased() - ) { + if (isScriptTransformSupportedForWorker()) { log.info('initialize script transform'); const options = { kind: 'encode', diff --git a/src/e2ee/utils.ts b/src/e2ee/utils.ts index b7f78b94a7..f51c33cd7b 100644 --- a/src/e2ee/utils.ts +++ b/src/e2ee/utils.ts @@ -8,11 +8,12 @@ export function isE2EESupported() { export function isScriptTransformSupported() { // @ts-ignore - return typeof window.RTCRtpScriptTransform !== 'undefined'; + return typeof window !== 'undefined' && typeof window.RTCRtpScriptTransform !== 'undefined'; } export function isInsertableStreamSupported() { return ( + typeof window !== 'undefined' && typeof window.RTCRtpSender !== 'undefined' && // @ts-ignore typeof window.RTCRtpSender.prototype.createEncodedStreams !== 'undefined' diff --git a/src/packetTrailer/PacketTrailerManager.test.ts b/src/packetTrailer/PacketTrailerManager.test.ts new file mode 100644 index 0000000000..80bbb91e92 --- /dev/null +++ b/src/packetTrailer/PacketTrailerManager.test.ts @@ -0,0 +1,79 @@ +import { afterEach, describe, expect, it, vi } from 'vitest'; +import { PacketTrailerManager } from './PacketTrailerManager'; + +describe('PacketTrailerManager', () => { + const originalUserAgent = navigator.userAgent; + const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) + .RTCRtpScriptTransform; + + afterEach(() => { + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: originalUserAgent, + }); + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: originalRTCRtpScriptTransform, + writable: true, + }); + Object.defineProperty(globalThis, 'RTCRtpScriptTransform', { + configurable: true, + value: originalRTCRtpScriptTransform, + writable: true, + }); + }); + + function useSafariUserAgent() { + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + }); + } + + function setScriptTransform(mock: unknown) { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: mock, + writable: true, + }); + Object.defineProperty(globalThis, 'RTCRtpScriptTransform', { + configurable: true, + value: mock, + writable: true, + }); + } + + function setupWorkerReceiver(manager: PacketTrailerManager, receiver: RTCRtpReceiver) { + ( + manager as unknown as { + setupWorkerReceiver: (receiver: RTCRtpReceiver, newTrackId: string) => void; + } + ).setupWorkerReceiver(receiver, 'track-id'); + } + + it('uses RTCRtpScriptTransform for packet trailer extraction when supported', () => { + useSafariUserAgent(); + const transform = {}; + const RTCRtpScriptTransform = vi.fn(() => transform); + setScriptTransform(RTCRtpScriptTransform); + + const worker = {} as Worker; + const manager = new PacketTrailerManager({ worker }); + const receiver = { + createEncodedStreams: vi.fn(), + } as unknown as RTCRtpReceiver; + + setupWorkerReceiver(manager, receiver); + + expect(RTCRtpScriptTransform).toHaveBeenCalledWith(worker, { + kind: 'decode', + trackId: 'track-id', + }); + expect((receiver as unknown as { transform: unknown }).transform).toBe(transform); + expect( + (receiver as unknown as { createEncodedStreams: ReturnType }) + .createEncodedStreams, + ).not.toHaveBeenCalled(); + }); +}); diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 85767944b0..37555ced5a 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -6,6 +6,7 @@ import { PacketTrailerExtractor } from '../room/track/PacketTrailerExtractor'; import type RemoteTrack from '../room/track/RemoteTrack'; import RemoteVideoTrack from '../room/track/RemoteVideoTrack'; import type { PTDecodeMessage, PTUpdateTrackIdMessage, PTWorkerMessage } from './types'; +import { isPacketTrailerSupported, shouldUsePacketTrailerScriptTransform } from './utils'; export interface PacketTrailerOptions { /** @@ -21,8 +22,8 @@ export interface PacketTrailerOptions { * Manages packet trailer extraction for received video tracks. * * When a track's TrackInfo indicates packet trailer features, the manager - * wires up an Insertable Streams pipeline to strip the trailer from encoded - * frames and cache the metadata for lookup. + * wires up an encoded frame transform to strip the trailer from encoded frames + * and cache the metadata for lookup. * * Packet trailer extraction is worker-only. If no worker is configured, the * SDK does not advertise packet trailer support and skips extraction. @@ -95,8 +96,11 @@ export class PacketTrailerManager { return; } - if (!this.worker && !this.room?.hasE2EESetup) { - log.warn('packet trailer worker not configured; skipping extraction'); + if ( + !isPacketTrailerSupported(this.worker ? { worker: this.worker } : undefined) && + !this.room?.hasE2EESetup + ) { + log.warn('packet trailer transform not supported; skipping extraction'); return; } @@ -117,6 +121,16 @@ export class PacketTrailerManager { private setupWorkerReceiver(receiver: RTCRtpReceiver, newTrackId: string) { const worker = this.worker!; + + if (shouldUsePacketTrailerScriptTransform()) { + // @ts-ignore + receiver.transform = new RTCRtpScriptTransform(worker, { + kind: 'decode', + trackId: newTrackId, + }); + return; + } + const existingTrackId = this.workerPipelines.get(receiver); if (existingTrackId) { diff --git a/src/packetTrailer/packetTrailer.test.ts b/src/packetTrailer/packetTrailer.test.ts index 29ee031a98..ba2cd7a18c 100644 --- a/src/packetTrailer/packetTrailer.test.ts +++ b/src/packetTrailer/packetTrailer.test.ts @@ -1,5 +1,5 @@ import { describe, expect, it } from 'vitest'; -import { appendPacketTrailer, extractPacketTrailer } from './packetTrailer'; +import { appendPacketTrailer, extractPacketTrailer, processPacketTrailer } from './packetTrailer'; describe('packetTrailer', () => { it('extracts user timestamp and frame id from packet trailer', () => { @@ -52,4 +52,28 @@ describe('packetTrailer', () => { expect(Array.from(extracted.data)).toEqual(Array.from(payload)); expect(extracted.metadata).toBeUndefined(); }); + + it('uses the encoded frame timestamp when metadata does not include an RTP timestamp', () => { + const payload = Uint8Array.from([1, 2, 3, 4]); + const trailer = appendPacketTrailer(payload, 1_744_249_600_123_456n, 42); + const frame = { + data: trailer.buffer, + timestamp: 1234, + getMetadata() { + return {}; + }, + } as unknown as RTCEncodedVideoFrame; + + const result = processPacketTrailer(frame, 'track-id'); + + expect(result.payload).toEqual({ + trackId: 'track-id', + rtpTimestamp: 1234, + ssrc: 0, + metadata: { + userTimestamp: 1_744_249_600_123_456n, + frameId: 42, + }, + }); + }); }); diff --git a/src/packetTrailer/packetTrailer.ts b/src/packetTrailer/packetTrailer.ts index db430e9f70..f33edc83d4 100644 --- a/src/packetTrailer/packetTrailer.ts +++ b/src/packetTrailer/packetTrailer.ts @@ -181,6 +181,9 @@ export function getFrameRtpTimestamp( } catch { // getMetadata() might not be available } + if (typeof frame.timestamp === 'number') { + return frame.timestamp; + } return undefined; } diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts index ed71063fe2..b021a9bdda 100644 --- a/src/packetTrailer/types.ts +++ b/src/packetTrailer/types.ts @@ -27,6 +27,11 @@ export interface PTDecodeMessage extends PTBaseMessage { }; } +export type PTScriptTransformOptions = { + kind: 'decode'; + trackId: string; +}; + export interface PTMetadataMessage extends PTBaseMessage { kind: 'metadata'; data: PacketTrailerFramePayload; diff --git a/src/packetTrailer/utils.test.ts b/src/packetTrailer/utils.test.ts new file mode 100644 index 0000000000..6cde3fdf68 --- /dev/null +++ b/src/packetTrailer/utils.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, it } from 'vitest'; +import { isPacketTrailerSupported } from './utils'; + +describe('packet trailer support', () => { + const originalRTCRtpSender = window.RTCRtpSender; + const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) + .RTCRtpScriptTransform; + const originalUserAgent = navigator.userAgent; + + afterEach(() => { + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: originalRTCRtpSender, + writable: true, + }); + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: originalRTCRtpScriptTransform, + writable: true, + }); + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: originalUserAgent, + }); + }); + + function stubScriptTransformSupport(userAgent: string) { + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: undefined, + writable: true, + }); + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: class MockRTCRtpScriptTransform {}, + writable: true, + }); + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: userAgent, + }); + } + + it('supports packet trailers with RTCRtpScriptTransform on Safari', () => { + stubScriptTransformSupport( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + ); + + expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true); + }); + + it('supports packet trailers with RTCRtpScriptTransform on Firefox', () => { + stubScriptTransformSupport( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14.0; rv:144.0) Gecko/20100101 Firefox/144.0', + ); + + expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(true); + }); + + it('does not use RTCRtpScriptTransform support on Chromium-based browsers', () => { + stubScriptTransformSupport( + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/141.0.0.0 Safari/537.36', + ); + + expect(isPacketTrailerSupported({ worker: {} as Worker })).toBe(false); + }); +}); diff --git a/src/packetTrailer/utils.ts b/src/packetTrailer/utils.ts new file mode 100644 index 0000000000..e0c184d7ec --- /dev/null +++ b/src/packetTrailer/utils.ts @@ -0,0 +1,13 @@ +import { isInsertableStreamSupported } from '../e2ee/utils'; +import { isScriptTransformSupportedForWorker } from '../room/utils'; +import type { PacketTrailerOptions } from './PacketTrailerManager'; + +export function shouldUsePacketTrailerScriptTransform() { + return isScriptTransformSupportedForWorker(); +} + +export function isPacketTrailerSupported(options?: PacketTrailerOptions) { + return ( + !!options?.worker && (isInsertableStreamSupported() || shouldUsePacketTrailerScriptTransform()) + ); +} diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 4bae7aaa0e..0ea428f84e 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -1,5 +1,5 @@ import { processPacketTrailer } from '../packetTrailer'; -import type { PTMetadataMessage, PTWorkerMessage } from '../types'; +import type { PTMetadataMessage, PTScriptTransformOptions, PTWorkerMessage } from '../types'; /** * Holds the trackId currently associated with a pipeline. A mutable @@ -75,3 +75,15 @@ function updateTrackId(oldTrackId: string, newTrackId: string) { pipelines.set(newTrackId, state); } } + +// Operations using RTCRtpScriptTransform. +// @ts-ignore +if (self.RTCTransformEvent) { + // @ts-ignore + self.onrtctransform = (event: RTCTransformEvent) => { + // @ts-ignore + const transformer = event.transformer; + const { trackId } = transformer.options as PTScriptTransformOptions; + setupDecodeTransform(transformer.readable, transformer.writable, trackId); + }; +} diff --git a/src/room/RTCEngine.test.ts b/src/room/RTCEngine.test.ts index 56faff5e57..616cf9db8b 100644 --- a/src/room/RTCEngine.test.ts +++ b/src/room/RTCEngine.test.ts @@ -4,6 +4,9 @@ import { roomOptionDefaults } from './defaults'; describe('RTCEngine', () => { const originalRTCRtpSender = window.RTCRtpSender; + const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) + .RTCRtpScriptTransform; + const originalUserAgent = navigator.userAgent; afterEach(() => { Object.defineProperty(window, 'RTCRtpSender', { @@ -11,6 +14,15 @@ describe('RTCEngine', () => { value: originalRTCRtpSender, writable: true, }); + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: originalRTCRtpScriptTransform, + writable: true, + }); + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: originalUserAgent, + }); }); function stubInsertableStreamsSupport() { @@ -25,6 +37,19 @@ describe('RTCEngine', () => { }); } + function stubScriptTransformSupport() { + Object.defineProperty(window, 'RTCRtpScriptTransform', { + configurable: true, + value: class MockRTCRtpScriptTransform {}, + writable: true, + }); + Object.defineProperty(window.navigator, 'userAgent', { + configurable: true, + value: + 'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_0) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15', + }); + } + function makeRTCConfiguration(engine: RTCEngine) { return ( engine as unknown as { makeRTCConfiguration: () => RTCConfiguration } @@ -56,6 +81,18 @@ describe('RTCEngine', () => { expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBe(true); }); + it('does not enable encoded insertable streams for packet trailers when script transforms are supported', () => { + stubInsertableStreamsSupport(); + stubScriptTransformSupport(); + + const engine = new RTCEngine({ + ...roomOptionDefaults, + packetTrailer: { worker: {} as Worker }, + }); + + expect(makeRTCConfiguration(engine).encodedInsertableStreams).toBeUndefined(); + }); + it('enables encoded insertable streams for E2EE', () => { stubInsertableStreamsSupport(); @@ -93,4 +130,21 @@ describe('RTCEngine', () => { expect(createEncodedStreams).not.toHaveBeenCalled(); }); + + it('does not create sender passthrough streams for packet trailers when script transforms are supported', () => { + stubScriptTransformSupport(); + + const engine = new RTCEngine({ + ...roomOptionDefaults, + packetTrailer: { worker: {} as Worker }, + }); + const createEncodedStreams = vi.fn(); + const sender = { + createEncodedStreams, + } as unknown as RTCRtpSender; + + setupSenderPassthrough(engine, sender); + + expect(createEncodedStreams).not.toHaveBeenCalled(); + }); }); diff --git a/src/room/RTCEngine.ts b/src/room/RTCEngine.ts index 34be377041..f97fedde20 100644 --- a/src/room/RTCEngine.ts +++ b/src/room/RTCEngine.ts @@ -57,6 +57,7 @@ import type { BaseE2EEManager } from '../e2ee/E2eeManager'; import { asEncryptablePacket, isInsertableStreamSupported } from '../e2ee/utils'; import log, { LoggerNames, getLogger } from '../logger'; import type { InternalRoomOptions } from '../options'; +import { shouldUsePacketTrailerScriptTransform } from '../packetTrailer/utils'; import TypedPromise from '../utils/TypedPromise'; import { DataPacketBuffer } from '../utils/dataPacketBuffer'; import { TTLMap } from '../utils/ttlmap'; @@ -771,14 +772,14 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit ): RTCConfiguration { const rtcConfig = { ...this.rtcConfig }; - // E2EE and packet trailer extraction both rely on encoded insertable - // streams. Only opt in when a transform will actually be installed; if no - // packet trailer worker is configured, packet trailer support is not - // advertised and the SFU strips trailers before forwarding media. - if ( - (this.signalOpts?.e2eeEnabled || this.options.packetTrailer?.worker) && - isInsertableStreamSupported() - ) { + // E2EE and packet trailer extraction both rely on encoded frame transforms. + // Only opt into the createEncodedStreams flavor when that path will be + // used; RTCRtpScriptTransform does not need the PeerConnection flag. + const needsInsertableStreams = + this.signalOpts?.e2eeEnabled || + (this.options.packetTrailer?.worker && !shouldUsePacketTrailerScriptTransform()); + + if (needsInsertableStreams && isInsertableStreamSupported()) { this.log.debug('setting up transports with insertable streams', this.logContext); // this makes sure that no data is sent before the transforms are ready // @ts-ignore @@ -1058,7 +1059,11 @@ export default class RTCEngine extends (EventEmitter as new () => TypedEventEmit } private setupSenderPassthrough(sender: RTCRtpSender) { - if (!this.options.packetTrailer?.worker || this.signalOpts?.e2eeEnabled) { + if ( + !this.options.packetTrailer?.worker || + this.signalOpts?.e2eeEnabled || + shouldUsePacketTrailerScriptTransform() + ) { return; } if ('createEncodedStreams' in sender) { diff --git a/src/room/Room.ts b/src/room/Room.ts index 812d49de11..6db435dd75 100644 --- a/src/room/Room.ts +++ b/src/room/Room.ts @@ -45,6 +45,7 @@ import type { RoomOptions, } from '../options'; import { PacketTrailerManager } from '../packetTrailer/PacketTrailerManager'; +import { isPacketTrailerSupported } from '../packetTrailer/utils'; import TypedPromise from '../utils/TypedPromise'; import { getBrowser } from '../utils/browserParser'; import { BackOffStrategy } from './BackOffStrategy'; @@ -928,7 +929,7 @@ class Room extends (EventEmitter as new () => TypedEmitter) autoSubscribe: connectOptions.autoSubscribe, adaptiveStream: typeof roomOptions.adaptiveStream === 'object' ? true : roomOptions.adaptiveStream, - clientInfoCapabilities: roomOptions.packetTrailer?.worker + clientInfoCapabilities: isPacketTrailerSupported(roomOptions.packetTrailer) ? [ClientInfo_Capability.CAP_PACKET_TRAILER] : undefined, maxRetries: connectOptions.maxRetries, diff --git a/src/room/utils.ts b/src/room/utils.ts index d8e414c2f9..45f4a210b0 100644 --- a/src/room/utils.ts +++ b/src/room/utils.ts @@ -180,6 +180,18 @@ export function isChromiumBased(): boolean { return !!browser && browser.name === 'Chrome' && browser.os !== 'iOS'; } +export function isScriptTransformSupportedForWorker(): boolean { + // Chrome occasionally throws an `InvalidState` error when using script transforms directly after introducing this API in 141. + // Disabling it for Chrome based browsers until the API has stabilized. + // @ts-ignore + return ( + typeof window !== 'undefined' && + // @ts-ignore + typeof window.RTCRtpScriptTransform !== 'undefined' && + !isChromiumBased() + ); +} + export function isSafari(): boolean { return getBrowser()?.name === 'Safari'; } From afc0fee5a84c7890efc6da09f2b17d011f1a285b Mon Sep 17 00:00:00 2001 From: David Chen Date: Tue, 28 Apr 2026 15:41:23 -0700 Subject: [PATCH 30/31] revert unneded changes --- src/room/data-track/depacketizer.test.ts | 1 + src/room/data-track/handle.test.ts | 1 + src/room/data-track/incoming/IncomingDataTrackManager.test.ts | 1 + src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts | 1 + src/room/data-track/packet/index.test.ts | 1 + src/room/data-track/packetizer.test.ts | 1 + src/room/data-track/utils.test.ts | 1 + src/room/token-source/types.ts | 2 +- 8 files changed, 8 insertions(+), 1 deletion(-) diff --git a/src/room/data-track/depacketizer.test.ts b/src/room/data-track/depacketizer.test.ts index 7d1de0b5fd..9a35ec9069 100644 --- a/src/room/data-track/depacketizer.test.ts +++ b/src/room/data-track/depacketizer.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import DataTrackDepacketizer from './depacketizer'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/handle.test.ts b/src/room/data-track/handle.test.ts index 34e46408ee..c486c4a8c0 100644 --- a/src/room/data-track/handle.test.ts +++ b/src/room/data-track/handle.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts index 11dc931949..6f24e2dbd4 100644 --- a/src/room/data-track/incoming/IncomingDataTrackManager.test.ts +++ b/src/room/data-track/incoming/IncomingDataTrackManager.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { subscribeToEvents } from '../../../utils/subscribeToEvents'; import { type DataTrackFrame } from '../frame'; diff --git a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts index a2e8378607..4ef1bb8b08 100644 --- a/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts +++ b/src/room/data-track/outgoing/OutgoingDataTrackManager.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { type DecryptDataResponseMessage, diff --git a/src/room/data-track/packet/index.test.ts b/src/room/data-track/packet/index.test.ts index daddfa8316..f918acb721 100644 --- a/src/room/data-track/packet/index.test.ts +++ b/src/room/data-track/packet/index.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackPacket, DataTrackPacketHeader, FrameMarker } from '.'; import { DataTrackHandle } from '../handle'; diff --git a/src/room/data-track/packetizer.test.ts b/src/room/data-track/packetizer.test.ts index 7e6e333a32..452f765bc2 100644 --- a/src/room/data-track/packetizer.test.ts +++ b/src/room/data-track/packetizer.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { DataTrackFrameInternal } from './frame'; import { DataTrackHandle } from './handle'; diff --git a/src/room/data-track/utils.test.ts b/src/room/data-track/utils.test.ts index ce951a4cc8..dedf0e41bd 100644 --- a/src/room/data-track/utils.test.ts +++ b/src/room/data-track/utils.test.ts @@ -1,3 +1,4 @@ +/* eslint-disable @typescript-eslint/no-unused-vars */ import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; import { U16_MAX_SIZE, WrapAroundUnsignedInt } from './utils'; diff --git a/src/room/token-source/types.ts b/src/room/token-source/types.ts index 7ca2bacd26..b60e864af0 100644 --- a/src/room/token-source/types.ts +++ b/src/room/token-source/types.ts @@ -3,7 +3,7 @@ import type { JWTPayload } from 'jose'; import type { ValueToSnakeCase } from '../../utils/camelToSnakeCase'; // The below imports are being linked in tsdoc comments, so they have to be imported even if they // aren't being used. - +// eslint-disable-next-line @typescript-eslint/no-unused-vars import type { TokenSourceCustom, TokenSourceEndpoint, TokenSourceLiteral } from './TokenSource'; export type TokenSourceRequestObject = Required< From e51ca727c7492628dee64ed40006e1fff31b7b8b Mon Sep 17 00:00:00 2001 From: David Chen Date: Wed, 29 Apr 2026 08:00:11 -0700 Subject: [PATCH 31/31] ensure outgoing video doesnt have packet trailer transformer --- .../PacketTrailerManager.test.ts | 93 +++++++++++++++++++ src/packetTrailer/PacketTrailerManager.ts | 42 ++++++++- src/packetTrailer/types.ts | 2 + .../worker/packetTrailer.worker.ts | 40 +++++--- 4 files changed, 160 insertions(+), 17 deletions(-) diff --git a/src/packetTrailer/PacketTrailerManager.test.ts b/src/packetTrailer/PacketTrailerManager.test.ts index 80bbb91e92..108e74d5a6 100644 --- a/src/packetTrailer/PacketTrailerManager.test.ts +++ b/src/packetTrailer/PacketTrailerManager.test.ts @@ -1,12 +1,19 @@ import { afterEach, describe, expect, it, vi } from 'vitest'; +import type { TrackInfo } from '@livekit/protocol'; import { PacketTrailerManager } from './PacketTrailerManager'; describe('PacketTrailerManager', () => { + const originalRTCRtpSender = window.RTCRtpSender; const originalUserAgent = navigator.userAgent; const originalRTCRtpScriptTransform = (window as unknown as { RTCRtpScriptTransform?: unknown }) .RTCRtpScriptTransform; afterEach(() => { + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: originalRTCRtpSender, + writable: true, + }); Object.defineProperty(window.navigator, 'userAgent', { configurable: true, value: originalUserAgent, @@ -23,6 +30,18 @@ describe('PacketTrailerManager', () => { }); }); + function stubInsertableStreamsSupport() { + class MockRTCRtpSender { + createEncodedStreams() {} + } + + Object.defineProperty(window, 'RTCRtpSender', { + configurable: true, + value: MockRTCRtpSender, + writable: true, + }); + } + function useSafariUserAgent() { Object.defineProperty(window.navigator, 'userAgent', { configurable: true, @@ -52,6 +71,35 @@ describe('PacketTrailerManager', () => { ).setupWorkerReceiver(receiver, 'track-id'); } + function setupReceiver( + manager: PacketTrailerManager, + receiver: RTCRtpReceiver, + trackId: string, + trackInfo?: TrackInfo, + ) { + ( + manager as unknown as { + setupReceiver: ( + track: { receiver: RTCRtpReceiver; mediaStreamID: string }, + trackInfo?: TrackInfo, + ) => void; + } + ).setupReceiver({ receiver, mediaStreamID: trackId }, trackInfo); + } + + function makeReceiver() { + const readable = {} as ReadableStream; + const writable = {} as WritableStream; + const createEncodedStreams = vi.fn(() => ({ readable, writable })); + + return { + receiver: { createEncodedStreams } as unknown as RTCRtpReceiver, + readable, + writable, + createEncodedStreams, + }; + } + it('uses RTCRtpScriptTransform for packet trailer extraction when supported', () => { useSafariUserAgent(); const transform = {}; @@ -76,4 +124,49 @@ describe('PacketTrailerManager', () => { .createEncodedStreams, ).not.toHaveBeenCalled(); }); + + it('sets up a passthrough receiver pipeline when a subscribed track has no packet trailer features', () => { + stubInsertableStreamsSupport(); + + const worker = { postMessage: vi.fn() } as unknown as Worker; + const manager = new PacketTrailerManager({ worker }); + const { receiver, readable, writable, createEncodedStreams } = makeReceiver(); + + setupReceiver(manager, receiver, 'track-without-trailer'); + + expect(createEncodedStreams).toHaveBeenCalledTimes(1); + expect(worker.postMessage).toHaveBeenCalledWith( + { + kind: 'decode', + data: { + readableStream: readable, + writableStream: writable, + trackId: 'track-without-trailer', + hasPacketTrailer: false, + }, + }, + [readable, writable], + ); + }); + + it('updates a reused receiver from trailer extraction to passthrough for tracks without packet trailer features', () => { + stubInsertableStreamsSupport(); + + const worker = { postMessage: vi.fn() } as unknown as Worker; + const manager = new PacketTrailerManager({ worker }); + const { receiver } = makeReceiver(); + const trackInfo = { packetTrailerFeatures: [1] } as unknown as TrackInfo; + + setupReceiver(manager, receiver, 'track-with-trailer', trackInfo); + setupReceiver(manager, receiver, 'track-without-trailer'); + + expect(worker.postMessage).toHaveBeenLastCalledWith({ + kind: 'updateTrackId', + data: { + oldTrackId: 'track-with-trailer', + newTrackId: 'track-without-trailer', + hasPacketTrailer: false, + }, + }); + }); }); diff --git a/src/packetTrailer/PacketTrailerManager.ts b/src/packetTrailer/PacketTrailerManager.ts index 37555ced5a..7e4438f5e5 100644 --- a/src/packetTrailer/PacketTrailerManager.ts +++ b/src/packetTrailer/PacketTrailerManager.ts @@ -93,6 +93,9 @@ export class PacketTrailerManager { const hasFeatures = !!trackInfo?.packetTrailerFeatures && trackInfo.packetTrailerFeatures.length > 0; if (!hasFeatures) { + if (!this.room?.hasE2EESetup) { + this.setupPassthroughReceiver(receiver, track.mediaStreamID); + } return; } @@ -116,11 +119,41 @@ export class PacketTrailerManager { return; } - this.setupWorkerReceiver(receiver, trackId); + this.setupWorkerReceiver(receiver, trackId, true); + } + + private setupPassthroughReceiver(receiver: RTCRtpReceiver, trackId: string) { + if (shouldUsePacketTrailerScriptTransform()) { + if ('transform' in receiver) { + // @ts-ignore + receiver.transform = null; + } + return; + } + + if ( + this.worker && + isPacketTrailerSupported({ worker: this.worker }) && + !this.workerPipelines.has(receiver) + ) { + this.setupWorkerReceiver(receiver, trackId, false); + return; + } + + if (this.worker && this.workerPipelines.has(receiver)) { + this.setupWorkerReceiver(receiver, trackId, false); + } } - private setupWorkerReceiver(receiver: RTCRtpReceiver, newTrackId: string) { - const worker = this.worker!; + private setupWorkerReceiver( + receiver: RTCRtpReceiver, + newTrackId: string, + hasPacketTrailer = true, + ) { + const worker = this.worker; + if (!worker) { + return; + } if (shouldUsePacketTrailerScriptTransform()) { // @ts-ignore @@ -139,7 +172,7 @@ export class PacketTrailerManager { // correctly and re-activate processing. const msg: PTUpdateTrackIdMessage = { kind: 'updateTrackId', - data: { oldTrackId: existingTrackId, newTrackId }, + data: { oldTrackId: existingTrackId, newTrackId, hasPacketTrailer }, }; worker.postMessage(msg); this.workerPipelines.set(receiver, newTrackId); @@ -166,6 +199,7 @@ export class PacketTrailerManager { readableStream: streams.readable, writableStream: streams.writable, trackId: newTrackId, + hasPacketTrailer, }, }; worker.postMessage(msg, [streams.readable, streams.writable]); diff --git a/src/packetTrailer/types.ts b/src/packetTrailer/types.ts index b021a9bdda..1370ddf01b 100644 --- a/src/packetTrailer/types.ts +++ b/src/packetTrailer/types.ts @@ -24,6 +24,7 @@ export interface PTDecodeMessage extends PTBaseMessage { readableStream: ReadableStream; writableStream: WritableStream; trackId: string; + hasPacketTrailer: boolean; }; } @@ -42,6 +43,7 @@ export interface PTUpdateTrackIdMessage extends PTBaseMessage { data: { oldTrackId: string; newTrackId: string; + hasPacketTrailer: boolean; }; } diff --git a/src/packetTrailer/worker/packetTrailer.worker.ts b/src/packetTrailer/worker/packetTrailer.worker.ts index 0ea428f84e..79e439561b 100644 --- a/src/packetTrailer/worker/packetTrailer.worker.ts +++ b/src/packetTrailer/worker/packetTrailer.worker.ts @@ -8,6 +8,7 @@ import type { PTMetadataMessage, PTScriptTransformOptions, PTWorkerMessage } fro */ interface PipelineState { trackId: string; + hasPacketTrailer: boolean; } const pipelines = new Map(); @@ -21,11 +22,16 @@ onmessage = (ev: MessageEvent) => { break; case 'decode': - setupDecodeTransform(msg.data.readableStream, msg.data.writableStream, msg.data.trackId); + setupDecodeTransform( + msg.data.readableStream, + msg.data.writableStream, + msg.data.trackId, + msg.data.hasPacketTrailer, + ); break; case 'updateTrackId': - updateTrackId(msg.data.oldTrackId, msg.data.newTrackId); + updateTrackId(msg.data.oldTrackId, msg.data.newTrackId, msg.data.hasPacketTrailer); break; default: @@ -33,8 +39,13 @@ onmessage = (ev: MessageEvent) => { } }; -function setupDecodeTransform(readable: ReadableStream, writable: WritableStream, trackId: string) { - const state: PipelineState = { trackId }; +function setupDecodeTransform( + readable: ReadableStream, + writable: WritableStream, + trackId: string, + hasPacketTrailer: boolean, +) { + const state: PipelineState = { trackId, hasPacketTrailer }; pipelines.set(trackId, state); const transform = new TransformStream({ @@ -43,13 +54,15 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream controller: TransformStreamDefaultController, ) { try { - const result = processPacketTrailer(frame, state.trackId); - if (result.data) { - frame.data = result.data; - } - if (result.payload) { - const msg: PTMetadataMessage = { kind: 'metadata', data: result.payload }; - postMessage(msg); + if (state.hasPacketTrailer) { + const result = processPacketTrailer(frame, state.trackId); + if (result.data) { + frame.data = result.data; + } + if (result.payload) { + const msg: PTMetadataMessage = { kind: 'metadata', data: result.payload }; + postMessage(msg); + } } } catch { // Never drop frames on trailer-extraction failure — pass through so @@ -67,10 +80,11 @@ function setupDecodeTransform(readable: ReadableStream, writable: WritableStream }); } -function updateTrackId(oldTrackId: string, newTrackId: string) { +function updateTrackId(oldTrackId: string, newTrackId: string, hasPacketTrailer: boolean) { const state = pipelines.get(oldTrackId); if (state) { state.trackId = newTrackId; + state.hasPacketTrailer = hasPacketTrailer; pipelines.delete(oldTrackId); pipelines.set(newTrackId, state); } @@ -84,6 +98,6 @@ if (self.RTCTransformEvent) { // @ts-ignore const transformer = event.transformer; const { trackId } = transformer.options as PTScriptTransformOptions; - setupDecodeTransform(transformer.readable, transformer.writable, trackId); + setupDecodeTransform(transformer.readable, transformer.writable, trackId, true); }; }